From b5d0f8678eee4a194e303d5f18e3da3bf09685a1 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 00:35:32 +0100 Subject: [PATCH 01/35] feat(storage): inspect native huge-page allocation without host writes Part of #5791. Bind fixed memory counters to unchanged native hardware and boot identity while preserving all migration gates. --- .../workflows/prepare-longhorn-v2-node.yaml | 5 +- scripts/migrate-longhorn-v2/README.md | 9 + scripts/migrate-longhorn-v2/host_memory.go | 185 ++++++++++++++++++ .../migrate-longhorn-v2/host_memory_test.go | 156 +++++++++++++++ scripts/migrate-longhorn-v2/inventory.go | 4 +- .../inventory_temporary_test.go | 5 + scripts/migrate-longhorn-v2/main.go | 9 + 7 files changed, 371 insertions(+), 2 deletions(-) create mode 100644 scripts/migrate-longhorn-v2/host_memory.go create mode 100644 scripts/migrate-longhorn-v2/host_memory_test.go diff --git a/.github/workflows/prepare-longhorn-v2-node.yaml b/.github/workflows/prepare-longhorn-v2-node.yaml index c0cbafd64a..65b8103619 100644 --- a/.github/workflows/prepare-longhorn-v2-node.yaml +++ b/.github/workflows/prepare-longhorn-v2-node.yaml @@ -17,7 +17,7 @@ on: description: The guarded migration step to run required: true type: choice - options: [prepare-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas] + options: [prepare-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas, inspect-host-memory] node: description: Permanent worker to prepare, or a fixed read-only inventory selection required: false @@ -83,6 +83,9 @@ jobs: inventory-replicas) [[ "${LONGHORN_V2_NODE}" =~ ^(prod-worker-[123]|temporary-v1-[12]|temporary-v2-1)$ && "${LONGHORN_V2_CONFIRM}" == "inventory-longhorn-replicas-${LONGHORN_V2_NODE}" ]] ;; + inspect-host-memory) + [[ "${LONGHORN_V2_NODE}" =~ ^prod-worker-[123]$ && "${LONGHORN_V2_CONFIRM}" == "inspect-longhorn-v2-host-memory-${LONGHORN_V2_NODE}" ]] + ;; refresh-clients) [[ "${LONGHORN_V2_CONFIRM}" == refresh-longhorn-v2-clients ]] ;; diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index c3550acd92..78a6a062ec 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -1,5 +1,14 @@ # Prepare Longhorn V2 workers +`inspect-host-memory` is a read-only diagnostic for a fixed permanent worker, +confirmed with `inspect-longhorn-v2-host-memory-`. It reads only the native +huge-page allocation and selected `/proc/meminfo` counters, bracketed by unchanged +Kubernetes, storage, hardware and kernel-boot identity and the current reviewed +source. Its observation-only receipt contains counters and commitments, never raw +host text or private identities. An allocation shortfall is reported as observed; +it grants no reset, reboot, restart or V2-activation clearance. Incomplete reads, +changing allocation, identity changes and close failures produce no success receipt. + The `Prepare Longhorn V2 Migration` workflow runs guarded steps towards platform#5582. It does not migrate a claim, enable V2, disable V1 or remove iSCSI. diff --git a/scripts/migrate-longhorn-v2/host_memory.go b/scripts/migrate-longhorn-v2/host_memory.go new file mode 100644 index 0000000000..6e2b232401 --- /dev/null +++ b/scripts/migrate-longhorn-v2/host_memory.go @@ -0,0 +1,185 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/netip" + "os" + "regexp" + "strconv" + "strings" + "time" +) + +type hostMemoryReceipt struct { + Schema int `json:"schema"` + ObservationOnly bool `json:"observationOnly"` + ResetClearance bool `json:"resetClearance"` + APIProjectionDigest string `json:"apiProjectionDigest"` + Report hostMemoryReport `json:"report"` +} + +func hostMemoryReadAllowed(program string, args ...string) bool { + if program == "talosctl" && len(args) == 3 && strings.HasPrefix(args[0], "--nodes=") && args[1] == "read" && + (args[2] == "/proc/sys/vm/nr_hugepages" || args[2] == "/proc/meminfo") { + target := strings.TrimPrefix(args[0], "--nodes=") + address, err := netip.ParseAddr(target) + return err == nil && address.String() == target && netip.MustParsePrefix("10.0.0.0/16").Contains(address) + } + return inventoryReadAllowed(program, args...) +} + +func hostMemoryReadCommand(ctx context.Context, input []byte, program string, args ...string) ([]byte, error) { + if input != nil || !hostMemoryReadAllowed(program, args...) { + return nil, errHostMemory + } + return inventoryExecuteCommand(ctx, program, args...) +} + +func observeHostMemory(ctx context.Context, c client, target string, factory inventoryReaderFactory) (receipt hostMemoryReceipt, err error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + expires, _ := ctx.Deadline() + active := func() bool { return ctx.Err() == nil && time.Now().Before(expires) } + if !active() || !inventoryTarget.MatchString(target) { + return hostMemoryReceipt{}, stageDeadline + } + before, node, err := c.inventoryProjection(ctx, target) + if err != nil { + return hostMemoryReceipt{}, stageProjectionBefore + } + reader, closeReader, err := factory(ctx, before.Address) + if err != nil || reader == nil || closeReader == nil { + return hostMemoryReceipt{}, stageClient + } + defer func() { + if closeReader() != nil { + receipt = hostMemoryReceipt{} + err = errors.Join(err, stageClose) + } + if !active() { + receipt = hostMemoryReceipt{} + err = errors.Join(err, stageDeadline) + } + }() + if c.verifyHost(ctx, before.Address, node) != nil { + return hostMemoryReceipt{}, stageHostBefore + } + boot, err := reader.BootID(ctx) + if err != nil || boot != before.BootID { + return hostMemoryReceipt{}, stageBootBefore + } + pages, err := c.talosCommand(ctx, talosReadLimit, before.Address, "read", "/proc/sys/vm/nr_hugepages") + if err != nil { + return hostMemoryReceipt{}, stageHostMemoryCounters + } + memory, err := c.talosCommand(ctx, talosReadLimit, before.Address, "read", "/proc/meminfo") + if err != nil { + return hostMemoryReceipt{}, stageHostMemoryCounters + } + report, err := parseHostMemory(pages, memory) + if err != nil { + return hostMemoryReceipt{}, stageHostMemoryCounters + } + second, err := c.talosCommand(ctx, talosReadLimit, before.Address, "read", "/proc/sys/vm/nr_hugepages") + if err != nil || !bytes.Equal(pages, second) { + return hostMemoryReceipt{}, stageHostMemoryAllocationChanged + } + after, node, err := c.inventoryProjection(ctx, target) + if err != nil || before.Digest != after.Digest { + return hostMemoryReceipt{}, stageAPIConsistency + } + if c.verifyHost(ctx, after.Address, node) != nil { + return hostMemoryReceipt{}, stageHostAfter + } + boot, err = reader.BootID(ctx) + if err != nil || boot != before.BootID { + return hostMemoryReceipt{}, stageBootAfter + } + return hostMemoryReceipt{Schema: 1, ObservationOnly: true, ResetClearance: false, APIProjectionDigest: before.Digest, Report: report}, nil +} + +func authorizeHostMemory(getenv func(string) string, target string) error { + if !inventoryTarget.MatchString(target) || getenv("LONGHORN_V2_NODE") != target || getenv("LONGHORN_V2_OPERATION") != "inspect-host-memory" { + return errHostMemory + } + return authorizeWorkflow(getenv, "inspect-longhorn-v2-host-memory-"+target) +} + +func runHostMemory(ctx context.Context, target, source, runID string) error { + c := client{run: hostMemoryReadCommand, context: "admin@prod"} + if sourceCurrent(ctx, c.run, source) != nil { + return stageSourceBefore + } + receipt, err := observeHostMemory(ctx, c, target, newInventoryReader) + if err != nil { + return err + } + if sourceCurrent(ctx, c.run, source) != nil { + return stageSourceAfter + } + if json.NewEncoder(os.Stdout).Encode(struct { + Source string `json:"source"` + Run string `json:"run"` + Observation hostMemoryReceipt `json:"observation"` + }{source, runID, receipt}) != nil { + return stageOutput + } + return nil +} + +var errHostMemory = errors.New("native host memory observation is incomplete or changed") +var hostPageCount = regexp.MustCompile(`^[0-9]{1,10}\n$`) + +type hostMemoryReport struct { + PersistentPages, TotalPages, FreePages, ReservedPages, SurplusPages uint64 + TotalKiB, AvailableKiB, PageSizeKiB uint64 +} + +func parseHostMemory(pages, memory []byte) (hostMemoryReport, error) { + if !hostPageCount.Match(pages) || len(memory) == 0 || len(memory) > 16384 || !bytes.HasSuffix(memory, []byte("\n")) { + return hostMemoryReport{}, errHostMemory + } + report := hostMemoryReport{} + var err error + report.PersistentPages, err = strconv.ParseUint(strings.TrimSuffix(string(pages), "\n"), 10, 32) + if err != nil { + return hostMemoryReport{}, errHostMemory + } + fields := map[string]*uint64{ + "MemTotal:": &report.TotalKiB, "MemAvailable:": &report.AvailableKiB, + "HugePages_Total:": &report.TotalPages, "HugePages_Free:": &report.FreePages, + "HugePages_Rsvd:": &report.ReservedPages, "HugePages_Surp:": &report.SurplusPages, + "Hugepagesize:": &report.PageSizeKiB, + } + seen := make(map[string]bool, len(fields)) + for line := range strings.SplitSeq(string(memory), "\n") { + parts := strings.Fields(line) + if len(parts) == 0 { + continue + } + destination, selected := fields[parts[0]] + if !selected { + continue + } + kilobytes := parts[0] == "MemTotal:" || parts[0] == "MemAvailable:" || parts[0] == "Hugepagesize:" + if seen[parts[0]] || kilobytes && (len(parts) != 3 || parts[2] != "kB") || !kilobytes && len(parts) != 2 { + return hostMemoryReport{}, errHostMemory + } + if parts[1] == "" || strings.IndexFunc(parts[1], func(r rune) bool { return r < '0' || r > '9' }) >= 0 { + return hostMemoryReport{}, errHostMemory + } + *destination, err = strconv.ParseUint(parts[1], 10, 63) + if err != nil { + return hostMemoryReport{}, errHostMemory + } + seen[parts[0]] = true + } + if len(seen) != len(fields) || report.TotalKiB == 0 || report.PageSizeKiB == 0 || report.AvailableKiB > report.TotalKiB || + report.FreePages > report.TotalPages || report.ReservedPages > report.TotalPages || report.SurplusPages > report.TotalPages { + return hostMemoryReport{}, errHostMemory + } + return report, nil +} diff --git a/scripts/migrate-longhorn-v2/host_memory_test.go b/scripts/migrate-longhorn-v2/host_memory_test.go new file mode 100644 index 0000000000..699c385738 --- /dev/null +++ b/scripts/migrate-longhorn-v2/host_memory_test.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" +) + +const hostMemoryFixture = `MemTotal: 16384000 kB +MemAvailable: 6000000 kB +HugePages_Total: 317 +HugePages_Free: 317 +HugePages_Rsvd: 0 +HugePages_Surp: 0 +Hugepagesize: 2048 kB +PrivateIgnored: private-memory-canary +` + +func TestHostMemoryCountersPreserveAnObservedAllocationShortfall(t *testing.T) { + report, err := parseHostMemory([]byte("317\n"), []byte(hostMemoryFixture)) + if err != nil || report.PersistentPages != 317 || report.TotalPages != 317 || report.AvailableKiB != 6000000 || report.PageSizeKiB != 2048 { + t.Fatal("actual allocation shortfall was lost:", report, err) + } + b, _ := json.Marshal(report) + if strings.Contains(string(b), "private-memory-canary") { + t.Fatal("native memory text escaped the counter projection") + } +} + +func TestHostMemoryObserverBindsCountersToNativeHostAndBoot(t *testing.T) { + for _, scenario := range []string{"stable shortfall", "API changed", "boot changed", "close failed", "memory read failed", "allocation changed"} { + t.Run(scenario, func(t *testing.T) { + node, storage, replicas := inventoryFixture() + nodeReads, pageReads := 0, 0 + c := client{context: "admin@prod", run: func(_ context.Context, input []byte, program string, args ...string) ([]byte, error) { + if input != nil || !hostMemoryReadAllowed(program, args...) { + t.Fatal("host observer escaped fixed read-only scope") + } + if program == "talosctl" { + if args[1] == "get" { + return json.Marshal(object{"metadata": object{"id": "systeminformation", "namespace": "hardware"}, "spec": object{"uuid": "machine-uuid"}}) + } + if args[2] == "/proc/meminfo" { + if scenario == "memory read failed" { + return nil, errors.New("private-provider-canary") + } + return []byte(hostMemoryFixture), nil + } + pageReads++ + if scenario == "allocation changed" && pageReads > 1 { + return []byte("318\n"), nil + } + return []byte("317\n"), nil + } + api, kind, items := "v1", "NodeList", []any{node} + switch args[3] { + case "--raw=" + inventoryKubeNodes: + nodeReads++ + if scenario == "API changed" && nodeReads > 1 { + node["metadata"].(object)["uid"] = "replacement" + } + case "--raw=" + inventoryStorageNodes: + api, items = "longhorn.io/v1beta2", []any{storage} + case "--raw=" + inventoryReplicas: + api, kind, items = "longhorn.io/v1beta2", "ReplicaList", []any{replicas[0]} + default: + t.Fatal("unexpected query") + } + return json.Marshal(object{"apiVersion": api, "kind": kind, "metadata": object{"resourceVersion": "1"}, "items": items}) + }} + reader := &replicaReaderFixture{changeBoot: scenario == "boot changed"} + closed := false + receipt, err := observeHostMemory(context.Background(), c, "prod-worker-1", func(_ context.Context, address string) (nativeReplicaReader, func() error, error) { + if address != "10.0.99.4" { + t.Fatal("host observer selected another endpoint") + } + return reader, func() error { + closed = true + if scenario == "close failed" { + return errors.New("private-close-canary") + } + return nil + }, nil + }) + if !closed || reader.listCalls != 0 || (err == nil) != (scenario == "stable shortfall") { + t.Fatal("host counter observation lost its boundary:", err) + } + encoded, _ := json.Marshal(receipt) + if strings.Contains(string(encoded), "canary") || strings.Contains(string(encoded), "10.0.99.4") || strings.Contains(string(encoded), "machine-uuid") { + t.Fatal("counter receipt exposed private source text or identity") + } + if err != nil { + if receipt.Schema != 0 || strings.Contains(inventoryFailureMessage(err), "canary") { + t.Fatal("failed observer retained success or leaked provider text") + } + return + } + if receipt.Schema != 1 || !receipt.ObservationOnly || receipt.ResetClearance || receipt.Report.PersistentPages != 317 { + t.Fatal("counter observation claimed migration clearance or lost actual values") + } + }) + } +} + +func TestHostMemoryCountersRejectMissingMalformedOrAmbiguousObservations(t *testing.T) { + for _, test := range []struct{ name, pages, memory string }{ + {"missing allocation", "", hostMemoryFixture}, + {"partial allocation", "31", hostMemoryFixture}, + {"negative", "-1\n", hostMemoryFixture}, + {"allocation overflow", "999999999999999999999999\n", hostMemoryFixture}, + {"missing memory", "317\n", "MemTotal: 16384000 kB\n"}, + {"duplicate", "317\n", hostMemoryFixture + "HugePages_Total: 317\n"}, + {"wrong units", "317\n", strings.Replace(hostMemoryFixture, "2048 kB", "2048 MB", 1)}, + {"impossible available", "317\n", strings.Replace(hostMemoryFixture, "6000000", "99999999", 1)}, + {"impossible free", "317\n", strings.Replace(hostMemoryFixture, "HugePages_Free: 317", "HugePages_Free: 999", 1)}, + {"missing final newline", "317\n", strings.TrimSuffix(hostMemoryFixture, "\n")}, + {"oversized", "317\n", hostMemoryFixture + strings.Repeat("x", 16384)}, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := parseHostMemory([]byte(test.pages), []byte(test.memory)); err == nil { + t.Fatal("invalid native counters were accepted") + } + }) + } +} + +func TestHostMemoryObservationHasDistinctAuthorizationAndFixedReadScope(t *testing.T) { + env := dispatchFixture() + env["LONGHORN_V2_OPERATION"], env["LONGHORN_V2_NODE"], env["LONGHORN_V2_CONFIRM"] = "inspect-host-memory", "prod-worker-1", "inspect-longhorn-v2-host-memory-prod-worker-1" + getenv := func(key string) string { return env[key] } + if authorizeHostMemory(getenv, "prod-worker-1") != nil { + t.Fatal("the confirmed fixed host observation was refused") + } + if authorize(getenv, "prod-worker-1") == nil || authorizeRefresh(getenv) == nil || authorizeInventory(getenv, "prod-worker-1") == nil { + t.Fatal("a read-only confirmation authorized another operation") + } + for _, target := range []string{"temporary-v1-1", "prod-worker-4", "--help"} { + env["LONGHORN_V2_NODE"] = target + env["LONGHORN_V2_CONFIRM"] = "inspect-longhorn-v2-host-memory-" + target + if authorizeHostMemory(getenv, target) == nil { + t.Fatal("observation escaped permanent worker selection") + } + } + for _, args := range [][]string{ + {"--nodes=10.0.99.4", "service", "kubelet", "restart"}, + {"--nodes=10.0.99.4", "read", "/etc/private-canary"}, + {"--nodes=192.0.2.1", "read", "/proc/meminfo"}, + {"--nodes=10.0.99.4", "read", "/proc/meminfo", "--nodes=192.0.2.1"}, + } { + if hostMemoryReadAllowed("talosctl", args...) { + t.Fatal("host command wrapper authorized a write, path, or peer outside its fixed scope") + } + } +} diff --git a/scripts/migrate-longhorn-v2/inventory.go b/scripts/migrate-longhorn-v2/inventory.go index 4de6763990..dd3ec8138c 100644 --- a/scripts/migrate-longhorn-v2/inventory.go +++ b/scripts/migrate-longhorn-v2/inventory.go @@ -47,12 +47,14 @@ const ( stageSourceBefore stageSourceAfter stageOutput + stageHostMemoryCounters + stageHostMemoryAllocationChanged ) func (stage inventoryStage) Error() string { names := [...]string{"unknown", "deadline", "projection-before", "client", "host-before", "boot-before", "listing-before", "ownership-before", "listing-after", "projection-after", "api-consistency", "host-after", "boot-after", "ownership-after", - "listing-consistency", "close", "source-before", "source-after", "output"} + "listing-consistency", "close", "source-before", "source-after", "output", "host-memory-counters", "host-memory-allocation-changed"} if int(stage) >= len(names) { stage = stageUnknown } diff --git a/scripts/migrate-longhorn-v2/inventory_temporary_test.go b/scripts/migrate-longhorn-v2/inventory_temporary_test.go index 927e9889ec..8e98cbb66c 100644 --- a/scripts/migrate-longhorn-v2/inventory_temporary_test.go +++ b/scripts/migrate-longhorn-v2/inventory_temporary_test.go @@ -251,6 +251,11 @@ func TestTemporaryInventoryNativeWorkflowGuardPreservesResetBoundary(t *testing. operation, node, ref, attempt, confirm string allowed bool }{ + {"inspect-host-memory", "prod-worker-1", "refs/heads/main", "1", "inspect-longhorn-v2-host-memory-prod-worker-1", true}, + {"inspect-host-memory", "temporary-v1-1", "refs/heads/main", "1", "inspect-longhorn-v2-host-memory-temporary-v1-1", false}, + {"inspect-host-memory", "prod-worker-1", "refs/heads/feature", "1", "inspect-longhorn-v2-host-memory-prod-worker-1", false}, + {"inspect-host-memory", "prod-worker-1", "refs/heads/main", "2", "inspect-longhorn-v2-host-memory-prod-worker-1", false}, + {"inspect-host-memory", "prod-worker-1", "refs/heads/main", "1", "refresh-longhorn-v2-clients", false}, {"inventory-replicas", "temporary-v1-1", "refs/heads/main", "1", "inventory-longhorn-replicas-temporary-v1-1", true}, {"inventory-replicas", "temporary-v1-2", "refs/heads/main", "1", "inventory-longhorn-replicas-temporary-v1-2", true}, {"inventory-replicas", "temporary-v2-1", "refs/heads/main", "1", "inventory-longhorn-replicas-temporary-v2-1", true}, diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index 4cc9a67ea5..8c8f344f0c 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -18,6 +18,8 @@ func main() { authorization = authorizeBenchmark(os.Getenv, operation[len("benchmark-"):]) case "inventory-replicas": authorization = authorizeInventory(os.Getenv, target) + case "inspect-host-memory": + authorization = authorizeHostMemory(os.Getenv, target) default: authorization = authorizeRefresh(os.Getenv) } @@ -27,6 +29,13 @@ func main() { } ctx, cancel := context.WithTimeout(context.Background(), 110*time.Minute) defer cancel() + if operation == "inspect-host-memory" { + if err := runHostMemory(ctx, target, os.Getenv("GITHUB_SHA"), os.Getenv("GITHUB_RUN_ID")); err != nil { + fmt.Fprintln(os.Stderr, inventoryFailureMessage(err)) + os.Exit(1) + } + return + } if operation == "inventory-replicas" { if err := runReplicaInventory(ctx, target, os.Getenv("GITHUB_SHA"), os.Getenv("GITHUB_RUN_ID")); err != nil { fmt.Fprintln(os.Stderr, inventoryFailureMessage(err)) From ab20e77dd6df00f27d399f4c81f95460d18803b9 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 23:04:55 +0100 Subject: [PATCH 02/35] feat(storage): define evidence for unused temporary worker preparation --- go.mod | 4 +- go.sum | 35 ++---- scripts/migrate-longhorn-v2/journal.go | 65 +++++++--- .../temporary_prepare_test.go | 115 ++++++++++++++++++ .../migrate-longhorn-v2/temporary_storage.go | 62 ++++++++++ .../temporary_storage_test.go | 71 +++++++++++ 6 files changed, 306 insertions(+), 46 deletions(-) create mode 100644 scripts/migrate-longhorn-v2/temporary_prepare_test.go create mode 100644 scripts/migrate-longhorn-v2/temporary_storage.go create mode 100644 scripts/migrate-longhorn-v2/temporary_storage_test.go diff --git a/go.mod b/go.mod index 0717befd3e..58a89b553e 100644 --- a/go.mod +++ b/go.mod @@ -55,9 +55,9 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect - github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/cel-go v0.31.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/cronexpr v1.1.3 // indirect github.com/klauspost/compress v1.19.2 // indirect @@ -96,7 +96,7 @@ require ( github.com/aws/smithy-go v1.28.6 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudflare/circl v1.6.4 // indirect - github.com/cosi-project/runtime v1.16.3 // indirect + github.com/cosi-project/runtime v1.16.3 github.com/gertd/go-pluralize v0.2.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 3fd20e4b14..6f8f21e584 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,6 @@ github.com/frankban/quicktest v1.13.1 h1:xVm/f9seEhZFL9+n5kv5XLrGwy6elc4V9v/XFY2 github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlLgiA= github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -83,8 +81,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= -github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= +github.com/gofrs/flock v0.13.1 h1:jjREztyBeSKBZYAC+mgc1laB+xsgy4kYMf3FbKF2UBo= +github.com/gofrs/flock v0.13.1/go.mod h1:sf4BFiHwnvgxa25DlQoDqXQnwRMEOwqxRq37P6MzzmE= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= @@ -124,8 +122,6 @@ github.com/jsimonetti/rtnetlink/v2 v2.2.1-0.20260802200809-43bafec815b3/go.mod h github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= @@ -150,14 +146,10 @@ github.com/mdlayher/netlink v1.11.2 h1:HKh2jqe+omdSWcQ88nrT7INE61B0NXfiSPFdgL4Yb github.com/mdlayher/netlink v1.11.2/go.mod h1:uT2Yc/QLaZubzDpZIBi9d4GoeLwtp3x1AMeqSRrK2sA= github.com/mdlayher/socket v0.6.1 h1:M7uj2NtuujUY4mYr1C57NmfNiRHbkKpnBxO856lsc3A= github.com/mdlayher/socket v0.6.1/go.mod h1:+/SGtqc9V+5dAuRgQsU0fGBI+oRDiW7O2Obx10OIWfg= -github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= -github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= -github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -200,8 +192,8 @@ github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+L github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= -github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -227,10 +219,11 @@ github.com/siderolabs/protoenc v0.2.4/go.mod h1:i5XLHjfv5vyi7LhQrSEo19HCA+lYtDd7 github.com/siderolabs/talos/pkg/machinery v1.14.4 h1:reYNIV4XTbUcOSVzlAqkM6psdOlO9U+HYc2AeuvAzfI= github.com/siderolabs/talos/pkg/machinery v1.14.4/go.mod h1:PEzCUOIG+87NIqBsONm/eQEQQfgoOwWL+r5ANhCrQGU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -239,16 +232,14 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tg123/go-htpasswd v1.2.4 h1:HgH8KKCjdmo7jjXWN9k1nefPBd7Be3tFCTjc2jPraPU= github.com/tg123/go-htpasswd v1.2.4/go.mod h1:EKThQok9xHkun6NBMynNv6Jmu24A33XdZzzl4Q7H1+0= -github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= -github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8ua9s= github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI= -github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= -github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= @@ -261,10 +252,6 @@ go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -294,16 +281,14 @@ golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 h1:dWeMvEJ3JhYgqSCAHUZZJgMUyfniiiCvDc72x5EqJP0= google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 h1:k5CJw9e5ONCcA/u0webKt092npXuY+KeGh3Q8NAVf0g= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= diff --git a/scripts/migrate-longhorn-v2/journal.go b/scripts/migrate-longhorn-v2/journal.go index da4090d9cf..61ee1edac1 100644 --- a/scripts/migrate-longhorn-v2/journal.go +++ b/scripts/migrate-longhorn-v2/journal.go @@ -10,21 +10,25 @@ import ( ) type journal struct { - Schema int `json:"schema"` - Target string `json:"target"` - Source string `json:"source"` - OwnerRun string `json:"ownerRun"` - Phase string `json:"phase"` - NodeUID string `json:"nodeUID"` - ProviderID string `json:"providerID"` - MachineUUID string `json:"machineUUID"` - LonghornUID string `json:"longhornUID"` - EphemeralUUID string `json:"ephemeralUUID"` - OriginalDisks object `json:"originalDisks"` + Schema int `json:"schema"` + Target string `json:"target"` + Source string `json:"source"` + OwnerRun string `json:"ownerRun"` + Phase string `json:"phase"` + NodeUID string `json:"nodeUID"` + ProviderID string `json:"providerID"` + MachineUUID string `json:"machineUUID"` + LonghornUID string `json:"longhornUID"` + EphemeralUUID string `json:"ephemeralUUID"` + OriginalDisks object `json:"originalDisks"` + TemporarySelection string `json:"temporarySelection,omitempty"` + NativeEmptyDigest string `json:"nativeEmptyDigest,omitempty"` + OriginalBootID string `json:"originalBootID,omitempty"` + RawVolumeAbsent bool `json:"rawVolumeAbsent,omitempty"` } func validateJournal(j journal, target string) error { - if j.Schema != 1 || j.Target != target || !regexp.MustCompile(`^prod-worker-[123]$`).MatchString(target) || + if j.Target != target || !regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString(j.Source) || !regexp.MustCompile(`^[1-9][0-9]*$`).MatchString(j.OwnerRun) || j.NodeUID == "" || !regexp.MustCompile(`^hcloud://[1-9][0-9]*$`).MatchString(j.ProviderID) || j.MachineUUID == "" || j.LonghornUID == "" || j.EphemeralUUID == "" { return errors.New("journal does not bind the exact original storage identities") @@ -34,6 +38,18 @@ func validateJournal(j journal, target string) error { default: return errors.New("journal phase is outside the node-preparation procedure") } + if j.Schema == 2 { + if !regexp.MustCompile(`^autoscale-longhorn-v1-[0-9a-f]{16}$`).MatchString(target) || + (j.TemporarySelection != "temporary-v1-1" && j.TemporarySelection != "temporary-v1-2") || + !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(j.NativeEmptyDigest) || !inventoryBoot.MatchString(j.OriginalBootID) || + !j.RawVolumeAbsent || j.OriginalDisks == nil || len(j.OriginalDisks) != 0 { + return errors.New("temporary preparation lacks its separate empty-storage identity proof") + } + return nil + } + if j.Schema != 1 || !inventoryTarget.MatchString(target) || j.TemporarySelection != "" || j.NativeEmptyDigest != "" || j.OriginalBootID != "" || j.RawVolumeAbsent { + return errors.New("journal schema and preparation scope do not match") + } if len(j.OriginalDisks) != 1 { return errors.New("node preparation requires exactly the original V1 filesystem disk") } @@ -47,11 +63,22 @@ func validateJournal(j journal, target string) error { } type preparation struct { - c client - target, name string - source, owner string - j journal - pools map[string]migrationPool + c client + target, name string + source, owner string + j journal + pools map[string]migrationPool + temporarySelection string +} + +func (p preparation) validateRecord(j journal) error { + if err := validateJournal(j, p.target); err != nil { + return err + } + if p.temporarySelection != j.TemporarySelection || (j.Schema == 2 && p.name != "longhorn-v2-prepare-"+j.TemporarySelection) { + return errors.New("journal does not match the explicitly selected preparation profile") + } + return nil } func (p preparation) readRecord(ctx context.Context) (object, journal, error) { @@ -69,7 +96,7 @@ func (p preparation) readRecord(ctx context.Context) (object, journal, error) { if d.Decode(&extra) != io.EOF { return nil, journal{}, errors.New("node-preparation journal is not singular") } - return cm, j, validateJournal(j, p.target) + return cm, j, p.validateRecord(j) } // reviewedWorkflowPath accepts the bare path GitHub returns today and the suffixed @@ -85,7 +112,7 @@ func (p *preparation) claimRecord(ctx context.Context, initial journal) error { return err } if cm == nil { - if err = validateJournal(initial, p.target); err != nil { + if err = p.validateRecord(initial); err != nil { return err } if err = sourceCurrent(ctx, p.c.run, p.source); err != nil { diff --git a/scripts/migrate-longhorn-v2/temporary_prepare_test.go b/scripts/migrate-longhorn-v2/temporary_prepare_test.go new file mode 100644 index 0000000000..e2b4175560 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_prepare_test.go @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" +) + +func temporaryJournalDocument(t *testing.T) object { + t.Helper() + data, err := json.Marshal(journalFixture()) + if err != nil { + t.Fatal(err) + } + var document object + if err = json.Unmarshal(data, &document); err != nil { + t.Fatal(err) + } + document["schema"] = float64(2) + document["target"] = "autoscale-longhorn-v1-0123456789abcdef" + document["originalDisks"] = object{} + document["temporarySelection"] = "temporary-v1-1" + document["nativeEmptyDigest"] = strings.Repeat("a", 64) + document["originalBootID"] = "12345678-1234-4234-8234-123456789abc" + document["rawVolumeAbsent"] = true + return document +} + +func TestLegacyPreparationCannotAdoptAnEmptyTemporaryJournal(t *testing.T) { + document := temporaryJournalDocument(t) + data, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + storage := resource("longhorn-v2-prepare-autoscale-longhorn-v1-0123456789abcdef") + storage["data"] = object{"record": string(data)} + p := preparation{target: text(document, "target"), name: text(storage, "metadata", "name"), c: client{context: "admin@prod", run: func(_ context.Context, input []byte, program string, args ...string) ([]byte, error) { + if input != nil || program != "kubectl" || len(args) < 3 || args[2] != "get" { + t.Fatal("journal inspection escaped its read scope") + } + return json.Marshal(storage) + }}} + if _, _, err = p.readRecord(context.Background()); err == nil { + t.Fatal("the permanent-worker preparation adopted a temporary reset contract") + } +} + +func TestTemporaryJournalRequiresItsOwnExplicitCallerAndJournalName(t *testing.T) { + record := decodeTemporaryJournal(t, temporaryJournalDocument(t)) + p := preparation{target: record.Target, name: "longhorn-v2-prepare-temporary-v1-1", temporarySelection: "temporary-v1-1"} + if err := p.validateRecord(record); err != nil { + t.Fatal("matching temporary caller rejected:", err) + } + p.name = "longhorn-v2-prepare-temporary-v1-2" + if p.validateRecord(record) == nil { + t.Fatal("another slot's journal was adopted") + } + p.name, p.temporarySelection = "longhorn-v2-prepare-temporary-v1-1", "temporary-v1-2" + if p.validateRecord(record) == nil { + t.Fatal("another explicitly selected slot was adopted") + } + legacy := journalFixture() + p.target, p.name = legacy.Target, "longhorn-v2-prepare-"+legacy.Target + if p.validateRecord(legacy) == nil { + t.Fatal("a temporary caller adopted the permanent-worker reset profile") + } +} + +func decodeTemporaryJournal(t *testing.T, document object) journal { + t.Helper() + data, err := json.Marshal(document) + if err != nil { + t.Fatal(err) + } + var record journal + if err = json.Unmarshal(data, &record); err != nil { + t.Fatal(err) + } + return record +} + +func TestEmptyTemporaryJournalKeepsASeparateResetContract(t *testing.T) { + document := temporaryJournalDocument(t) + record := decodeTemporaryJournal(t, document) + if err := validateJournal(record, record.Target); err != nil { + t.Fatalf("positively bound empty temporary reset cannot be journaled: %v", err) + } + for name, mutate := range map[string]func(object){ + "legacy schema": func(d object) { d["schema"] = float64(1) }, + "permanent target": func(d object) { d["target"] = "prod-worker-1" }, + "fresh V2 pool": func(d object) { d["target"] = "autoscale-longhorn-v2-0123456789abcdef" }, + "ordinary compute": func(d object) { d["target"] = "autoscale-cx43-0123456789abcdef" }, + "unknown slot": func(d object) { d["temporarySelection"] = "temporary-v1-3" }, + "missing physical proof": func(d object) { delete(d, "nativeEmptyDigest") }, + "malformed physical proof": func(d object) { d["nativeEmptyDigest"] = "empty" }, + "missing boot identity": func(d object) { delete(d, "originalBootID") }, + "raw state unknown": func(d object) { delete(d, "rawVolumeAbsent") }, + "raw volume exists": func(d object) { d["rawVolumeAbsent"] = false }, + "registered old disk": func(d object) { d["originalDisks"] = journalFixture().OriginalDisks }, + } { + t.Run(name, func(t *testing.T) { + document := temporaryJournalDocument(t) + mutate(document) + record := decodeTemporaryJournal(t, document) + if err := validateJournal(record, record.Target); err == nil { + t.Fatal("an unbound or data-bearing reset was accepted") + } + }) + } + legacy := journalFixture() + if err := validateJournal(legacy, legacy.Target); err != nil { + t.Fatal("legacy permanent-worker contract changed:", err) + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_storage.go b/scripts/migrate-longhorn-v2/temporary_storage.go new file mode 100644 index 0000000000..30c71f6e76 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_storage.go @@ -0,0 +1,62 @@ +package main + +import ( + cosiresource "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/talos/pkg/machinery/resources/block" + "strings" +) + +// Check the complete native block-resource inventory, never a CLI error string. +func verifyUnusedTemporaryPartition(statuses, discovered cosiresource.List, ephemeralUUID string) error { + if ephemeralUUID == "" || len(statuses.Items) == 0 || len(statuses.Items) > 128 || len(discovered.Items) == 0 || len(discovered.Items) > 512 { + return errInventory + } + seen := map[string]bool{} + var original *block.VolumeStatusSpec + for _, item := range statuses.Items { + volume, ok := item.(*block.VolumeStatus) + if !ok || volume == nil || volume.Metadata().Namespace() != block.NamespaceName || volume.Metadata().Phase() != cosiresource.PhaseRunning || seen[volume.Metadata().ID()] { + return errInventory + } + id := volume.Metadata().ID() + seen[id] = true + spec := volume.TypedSpec() + if id == "EPHEMERAL" { + original = spec + } + if strings.HasPrefix(id, "r-") || strings.HasPrefix(id, "u-") { + if id != "r-longhorn" || spec.Phase != block.VolumePhaseWaiting || spec.Location != "" || spec.MountLocation != "" || spec.ParentLocation != "" || spec.UUID != "" || spec.PartitionUUID != "" || spec.PartitionIndex != 0 || spec.Size != 0 { + return errInventory + } + } + } + if original == nil || original.Phase != block.VolumePhaseReady || original.Type != block.VolumeTypePartition || original.UUID != ephemeralUUID || + original.Location == "" || original.ParentLocation == "" || !strings.HasPrefix(original.MountLocation, "/dev/dm-") || + original.EncryptionProvider != block.EncryptionProviderLUKS2 || !original.EncryptionLockedToState || + len(original.ConfiguredEncryptionKeys) != 1 || original.ConfiguredEncryptionKeys[0] != "nodeID" || original.Size < 136<<30 { + return errInventory + } + seen = map[string]bool{} + found := false + for _, item := range discovered.Items { + volume, ok := item.(*block.DiscoveredVolume) + if !ok || volume == nil || volume.Metadata().Namespace() != block.NamespaceName || volume.Metadata().Phase() != cosiresource.PhaseRunning || seen[volume.Metadata().ID()] { + return errInventory + } + seen[volume.Metadata().ID()] = true + spec := volume.TypedSpec() + if spec.DevPath == "" || spec.Size == 0 || strings.HasPrefix(spec.PartitionLabel, "r-") || strings.HasPrefix(spec.PartitionLabel, "u-") { + return errInventory + } + if spec.PartitionLabel == "EPHEMERAL" { + if found || spec.DevPath != original.Location || spec.ParentDevPath != original.ParentLocation || spec.Size != original.Size || spec.PartitionUUID == "" { + return errInventory + } + found = true + } + } + if !found { + return errInventory + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/temporary_storage_test.go b/scripts/migrate-longhorn-v2/temporary_storage_test.go new file mode 100644 index 0000000000..5ed4d6e70f --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_storage_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "testing" + + cosiresource "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/talos/pkg/machinery/resources/block" +) + +func temporaryPartitionFixture() (cosiresource.List, cosiresource.List, string) { + eph := block.NewVolumeStatus(block.NamespaceName, "EPHEMERAL") + *eph.TypedSpec() = block.VolumeStatusSpec{Phase: block.VolumePhaseReady, Type: block.VolumeTypePartition, + UUID: "ephemeral-original", Location: "/dev/sda6", ParentLocation: "/dev/sda", MountLocation: "/dev/dm-0", Size: 150 << 30, + EncryptionProvider: block.EncryptionProviderLUKS2, EncryptionLockedToState: true, ConfiguredEncryptionKeys: []string{"nodeID"}} + raw := block.NewVolumeStatus(block.NamespaceName, "r-longhorn") + raw.TypedSpec().Phase = block.VolumePhaseWaiting + physical := block.NewDiscoveredVolume(block.NamespaceName, "sda6") + *physical.TypedSpec() = block.DiscoveredVolumeSpec{DevPath: "/dev/sda6", ParentDevPath: "/dev/sda", PartitionLabel: "EPHEMERAL", PartitionUUID: "partition-original", Size: 150 << 30} + return cosiresource.List{Items: []cosiresource.Resource{eph, raw}}, cosiresource.List{Items: []cosiresource.Resource{physical}}, eph.TypedSpec().UUID +} + +func TestNativeTemporaryPartitionRequiresPositiveOriginalAndNoRawStorage(t *testing.T) { + statuses, discovered, uuid := temporaryPartitionFixture() + if err := verifyUnusedTemporaryPartition(statuses, discovered, uuid); err != nil { + t.Fatal("complete encrypted source partition with no raw storage was rejected:", err) + } + for name, mutate := range map[string]func(*cosiresource.List, *cosiresource.List){ + "missing original": func(s, _ *cosiresource.List) { s.Items = s.Items[1:] }, + "empty discovery": func(_, d *cosiresource.List) { d.Items = nil }, + "duplicate status": func(s, _ *cosiresource.List) { s.Items = append(s.Items, s.Items[0]) }, + "duplicate discovery": func(_, d *cosiresource.List) { d.Items = append(d.Items, d.Items[0]) }, + "wrong status type": func(s, d *cosiresource.List) { s.Items = append(s.Items, d.Items[0]) }, + "wrong namespace": func(s, _ *cosiresource.List) { + r := block.NewVolumeStatus("other", "EPHEMERAL") + *r.TypedSpec() = *s.Items[0].(*block.VolumeStatus).TypedSpec() + s.Items[0] = r + }, + "tearing down": func(s, _ *cosiresource.List) { s.Items[0].Metadata().SetPhase(cosiresource.PhaseTearingDown) }, + "changed partition": func(s, _ *cosiresource.List) { s.Items[0].(*block.VolumeStatus).TypedSpec().UUID = "replacement" }, + "unencrypted source": func(s, _ *cosiresource.List) { + s.Items[0].(*block.VolumeStatus).TypedSpec().EncryptionProvider = block.EncryptionProviderNone + }, + "unlocked to state": func(s, _ *cosiresource.List) { + s.Items[0].(*block.VolumeStatus).TypedSpec().EncryptionLockedToState = false + }, + "different physical source": func(_, d *cosiresource.List) { d.Items[0].(*block.DiscoveredVolume).TypedSpec().DevPath = "/dev/sdb6" }, + "raw allocated": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().Phase = block.VolumePhaseReady + }, + "raw location while waiting": func(s, _ *cosiresource.List) { s.Items[1].(*block.VolumeStatus).TypedSpec().Location = "/dev/sda7" }, + "raw missing phase": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().Phase = block.VolumePhaseMissing + }, + "foreign pending volume": func(s, _ *cosiresource.List) { + s.Items = append(s.Items, block.NewVolumeStatus(block.NamespaceName, "r-other")) + }, + "raw physical partition": func(_, d *cosiresource.List) { + r := block.NewDiscoveredVolume(block.NamespaceName, "sda7") + *r.TypedSpec() = block.DiscoveredVolumeSpec{DevPath: "/dev/sda7", ParentDevPath: "/dev/sda", PartitionLabel: "r-longhorn", Size: 96 << 30} + d.Items = append(d.Items, r) + }, + } { + t.Run(name, func(t *testing.T) { + s, d, uuid := temporaryPartitionFixture() + mutate(&s, &d) + if verifyUnusedTemporaryPartition(s, d, uuid) == nil { + t.Fatal("unknown or existing storage was accepted as empty") + } + }) + } +} From a8007213f79cfc11c0f89c14615eb14bd29e8270 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 23:15:54 +0100 Subject: [PATCH 03/35] feat(storage): bracket empty temporary observations with native block inventory --- .../migrate-longhorn-v2/inventory_runtime.go | 10 +- .../migrate-longhorn-v2/temporary_storage.go | 36 ++++++- .../temporary_storage_test.go | 102 ++++++++++++++++++ 3 files changed, 146 insertions(+), 2 deletions(-) diff --git a/scripts/migrate-longhorn-v2/inventory_runtime.go b/scripts/migrate-longhorn-v2/inventory_runtime.go index 720e69a940..f2c48b644b 100644 --- a/scripts/migrate-longhorn-v2/inventory_runtime.go +++ b/scripts/migrate-longhorn-v2/inventory_runtime.go @@ -121,6 +121,10 @@ func inventoryExecuteCommand(ctx context.Context, program string, args ...string } func newInventoryReader(ctx context.Context, address string) (nativeReplicaReader, func() error, error) { + return openInventoryReader(ctx, address, "") +} + +func openInventoryReader(ctx context.Context, address, unusedEphemeralUUID string) (nativeReplicaReader, func() error, error) { directory := os.Getenv("LONGHORN_V2_CREDENTIAL_DIR") if !regexp.MustCompile(`^/dev/shm/longhorn-v2\.[A-Za-z0-9]+$`).MatchString(directory) || os.Getenv("TALOSCONFIG") != filepath.Join(directory, "talosconfig") { return nil, nil, errInventory @@ -158,7 +162,11 @@ func newInventoryReader(ctx context.Context, address string) (nativeReplicaReade if err != nil { return nil, nil, errInventory } - return longhorninventory.Native{API: connection.MachineClient, Target: address}, connection.Close, nil + var reader nativeReplicaReader = longhorninventory.Native{API: connection.MachineClient, Target: address} + if unusedEphemeralUUID != "" { + reader = unusedTemporaryReader{nativeReplicaReader: reader, blocks: connection.COSI, address: address, ephemeralUUID: unusedEphemeralUUID} + } + return reader, connection.Close, nil } func runReplicaInventory(ctx context.Context, target, source, runID string) error { diff --git a/scripts/migrate-longhorn-v2/temporary_storage.go b/scripts/migrate-longhorn-v2/temporary_storage.go index 30c71f6e76..043f26eb7d 100644 --- a/scripts/migrate-longhorn-v2/temporary_storage.go +++ b/scripts/migrate-longhorn-v2/temporary_storage.go @@ -1,11 +1,45 @@ package main import ( + "context" + "net/netip" + "strings" + "time" + cosiresource "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/devantler-tech/platform/internal/longhorninventory" + talos "github.com/siderolabs/talos/pkg/machinery/client" "github.com/siderolabs/talos/pkg/machinery/resources/block" - "strings" ) +type unusedTemporaryReader struct { + nativeReplicaReader + blocks state.CoreStateReader + address, ephemeralUUID string +} + +// Both native block inventories are collected inside the existing hardware and +// boot bracket on EACH file observation. A partial resource list never clears +// a reset, even when its returned prefix happens to contain the original disk. +func (r unusedTemporaryReader) ListReplicas(ctx context.Context) (longhorninventory.Listing, error) { + address, addressErr := netip.ParseAddr(r.address) + if ctx.Err() != nil || r.blocks == nil || r.nativeReplicaReader == nil || r.ephemeralUUID == "" || addressErr != nil || address.String() != r.address || !netip.MustParsePrefix("10.0.0.0/16").Contains(address) { + return longhorninventory.Listing{}, errInventory + } + ctx, cancel := context.WithTimeout(talos.WithNode(ctx, r.address), 20*time.Second) + defer cancel() + statuses, err := r.blocks.List(ctx, block.NewVolumeStatus(block.NamespaceName, "").Metadata()) + if err != nil || ctx.Err() != nil { + return longhorninventory.Listing{}, errInventory + } + discovered, err := r.blocks.List(ctx, block.NewDiscoveredVolume(block.NamespaceName, "").Metadata()) + if err != nil || ctx.Err() != nil || verifyUnusedTemporaryPartition(statuses, discovered, r.ephemeralUUID) != nil { + return longhorninventory.Listing{}, errInventory + } + return r.nativeReplicaReader.ListReplicas(ctx) +} + // Check the complete native block-resource inventory, never a CLI error string. func verifyUnusedTemporaryPartition(statuses, discovered cosiresource.List, ephemeralUUID string) error { if ephemeralUUID == "" || len(statuses.Items) == 0 || len(statuses.Items) > 128 || len(discovered.Items) == 0 || len(discovered.Items) > 512 { diff --git a/scripts/migrate-longhorn-v2/temporary_storage_test.go b/scripts/migrate-longhorn-v2/temporary_storage_test.go index 5ed4d6e70f..9526df5d9b 100644 --- a/scripts/migrate-longhorn-v2/temporary_storage_test.go +++ b/scripts/migrate-longhorn-v2/temporary_storage_test.go @@ -1,10 +1,17 @@ package main import ( + "context" + "errors" + "strings" "testing" + "time" cosiresource "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/devantler-tech/platform/internal/longhorninventory" "github.com/siderolabs/talos/pkg/machinery/resources/block" + "google.golang.org/grpc/metadata" ) func temporaryPartitionFixture() (cosiresource.List, cosiresource.List, string) { @@ -69,3 +76,98 @@ func TestNativeTemporaryPartitionRequiresPositiveOriginalAndNoRawStorage(t *test }) } } + +type temporaryBlockReaderFixture struct { + t *testing.T + statuses, discovered cosiresource.List + calls, failAt int + allocateAfterFirst bool +} + +func (r *temporaryBlockReaderFixture) Get(context.Context, cosiresource.Pointer, ...state.GetOption) (cosiresource.Resource, error) { + r.t.Fatal("temporary storage proof attempted an unreviewed resource request") + return nil, errInventory +} + +func (r *temporaryBlockReaderFixture) List(ctx context.Context, kind cosiresource.Kind, options ...state.ListOption) (cosiresource.List, error) { + r.t.Helper() + r.calls++ + deadline, bounded := ctx.Deadline() + md, _ := metadata.FromOutgoingContext(ctx) + if !bounded || time.Until(deadline) > 20*time.Second || len(md.Get("node")) != 1 || md.Get("node")[0] != "10.0.99.4" || len(md.Get("nodes")) != 0 || kind.Namespace() != block.NamespaceName || len(options) != 0 { + r.t.Fatal("native resource proof lost its bounded exact-node scope") + } + var result cosiresource.List + if r.calls%2 == 1 { + if kind.Type() != block.VolumeStatusType { + r.t.Fatal("unexpected volume resource query") + } + if r.allocateAfterFirst && r.calls > 2 { + r.statuses.Items[1].(*block.VolumeStatus).TypedSpec().Phase = block.VolumePhaseReady + } + result = r.statuses + } else { + if kind.Type() != block.DiscoveredVolumeType { + r.t.Fatal("unexpected physical discovery query") + } + result = r.discovered + } + if r.calls == r.failAt { + return result, errors.New("partial-native-list-private-detail") + } + return result, nil +} + +type temporaryFileReaderFixture struct{ calls int } + +func (*temporaryFileReaderFixture) BootID(context.Context) (string, error) { return "unused", nil } +func (r *temporaryFileReaderFixture) ListReplicas(context.Context) (longhorninventory.Listing, error) { + r.calls++ + return longhorninventory.Listing{Complete: true}, nil +} + +func TestTemporaryNativeProofStopsBeforeFilesOnPartialOrChangedStorage(t *testing.T) { + for _, scenario := range []string{"complete", "partial status", "partial discovery", "raw allocated between observations", "canceled", "missing original binding"} { + t.Run(scenario, func(t *testing.T) { + s, d, uuid := temporaryPartitionFixture() + blocks := &temporaryBlockReaderFixture{t: t, statuses: s, discovered: d} + files := &temporaryFileReaderFixture{} + reader := unusedTemporaryReader{nativeReplicaReader: files, blocks: blocks, address: "10.0.99.4", ephemeralUUID: uuid} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + switch scenario { + case "partial status": + blocks.failAt = 1 + case "partial discovery": + blocks.failAt = 2 + case "raw allocated between observations": + blocks.allocateAfterFirst = true + case "canceled": + cancel() + case "missing original binding": + reader.ephemeralUUID = "" + } + first, err := reader.ListReplicas(ctx) + if scenario == "complete" || scenario == "raw allocated between observations" { + if err != nil || !first.Complete || files.calls != 1 { + t.Fatal("complete physical proof did not reach file inspection:", err) + } + second, secondErr := reader.ListReplicas(ctx) + if scenario == "complete" { + if secondErr != nil || !second.Complete || files.calls != 2 { + t.Fatal("second independent physical proof did not complete") + } + } else if secondErr == nil || second.Complete || files.calls != 1 { + t.Fatal("a raw partition appeared without invalidating the second observation") + } + return + } + if err == nil || first.Complete || files.calls != 0 || strings.Contains(err.Error(), "private-detail") { + t.Fatal("incomplete physical proof reached file inspection or leaked details") + } + if (scenario == "canceled" || scenario == "missing original binding") && blocks.calls != 0 { + t.Fatal("an unauthorized or expired observation started an API request") + } + }) + } +} From 55d7c8b476948af903a6408cfcc9ead1db750a38 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 23:18:52 +0100 Subject: [PATCH 04/35] feat(storage): bind temporary inventory to its owned preparation cordon --- scripts/migrate-longhorn-v2/api.go | 7 +- .../migrate-longhorn-v2/inventory_observe.go | 8 +++ .../migrate-longhorn-v2/temporary_cordon.go | 40 +++++++++++ .../temporary_cordon_test.go | 69 +++++++++++++++++++ 4 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 scripts/migrate-longhorn-v2/temporary_cordon.go create mode 100644 scripts/migrate-longhorn-v2/temporary_cordon_test.go diff --git a/scripts/migrate-longhorn-v2/api.go b/scripts/migrate-longhorn-v2/api.go index 92c1c76de7..99909253d2 100644 --- a/scripts/migrate-longhorn-v2/api.go +++ b/scripts/migrate-longhorn-v2/api.go @@ -35,9 +35,10 @@ func execute(ctx context.Context, input []byte, program string, args ...string) } type client struct { - run command - context string - inventoryPools map[string]migrationPool + run command + context string + inventoryPools map[string]migrationPool + inventoryPreparation *journal } var kubeName = regexp.MustCompile(`^[a-z0-9]([-a-z0-9.]*[a-z0-9])?$`) diff --git a/scripts/migrate-longhorn-v2/inventory_observe.go b/scripts/migrate-longhorn-v2/inventory_observe.go index eba00af3ed..e0570d9a7f 100644 --- a/scripts/migrate-longhorn-v2/inventory_observe.go +++ b/scripts/migrate-longhorn-v2/inventory_observe.go @@ -54,6 +54,11 @@ func (c client) inventoryProjection(ctx context.Context, target string) (invento } } selectionDigest := "" + var projectionErr error + nodes, projectionErr = temporaryInventoryNodes(nodes, c.inventoryPreparation) + if projectionErr != nil { + return inventoryProjection{}, nil, projectionErr + } if !inventoryTarget.MatchString(target) { var err error target, selectionDigest, err = resolveTemporaryInventoryTarget(target, nodes, c.inventoryPools) @@ -61,6 +66,9 @@ func (c client) inventoryProjection(ctx context.Context, target string) (invento return inventoryProjection{}, nil, err } } + if c.inventoryPreparation != nil && target != c.inventoryPreparation.Target { + return inventoryProjection{}, nil, errInventory + } var node, lh object for _, item := range nodes { if text(item, "metadata", "name") == target { diff --git a/scripts/migrate-longhorn-v2/temporary_cordon.go b/scripts/migrate-longhorn-v2/temporary_cordon.go new file mode 100644 index 0000000000..e5a800ef47 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_cordon.go @@ -0,0 +1,40 @@ +package main + +import "maps" + +// The ordinary pool resolver continues rejecting cordoned nodes. A preparation +// may inspect its own cordoned target only after binding the live hardware, +// original boot, node identity and fence to its separately validated journal. +// The analysis view never changes the observed object or sends a mutation. +func temporaryInventoryNodes(nodes []object, record *journal) ([]object, error) { + if record == nil { + return nodes, nil + } + if record.Schema != 2 || validateJournal(*record, record.Target) != nil || (record.Phase != "evacuating" && record.Phase != "draining") { + return nil, errInventory + } + result := append([]object(nil), nodes...) + found := false + for i, node := range nodes { + if text(node, "metadata", "name") != record.Target { + continue + } + if found || value(node, "spec", "unschedulable") != true || text(node, "metadata", "annotations", fenceAnnotation) != "longhorn-v2-prepare-"+record.TemporarySelection || + text(node, "status", "nodeInfo", "bootID") != record.OriginalBootID { + return nil, errInventory + } + if _, err := verifyNode(node, *record, true); err != nil { + return nil, errInventory + } + found = true + copy := maps.Clone(node) + spec := maps.Clone(valueObject(node, "spec")) + spec["unschedulable"] = false + copy["spec"] = spec + result[i] = copy + } + if !found { + return nil, errInventory + } + return result, nil +} diff --git a/scripts/migrate-longhorn-v2/temporary_cordon_test.go b/scripts/migrate-longhorn-v2/temporary_cordon_test.go new file mode 100644 index 0000000000..ab905e3047 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_cordon_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "testing" +) + +func temporaryOwnedCordonFixture(t *testing.T) ([]object, journal) { + t.Helper() + nodes := temporaryInventoryRoster() + j := decodeTemporaryJournal(t, temporaryJournalDocument(t)) + n := nodes[0] + j.Target, j.NodeUID = text(n, "metadata", "name"), text(n, "metadata", "uid") + j.ProviderID, j.MachineUUID = text(n, "spec", "providerID"), text(n, "status", "nodeInfo", "systemUUID") + j.OriginalBootID = text(n, "status", "nodeInfo", "bootID") + n["spec"].(object)["unschedulable"] = true + n["metadata"].(object)["annotations"].(object)[fenceAnnotation] = "longhorn-v2-prepare-temporary-v1-1" + return nodes, j +} + +func TestTemporaryInventoryRecognizesOnlyItsIdentityBoundOwnedCordon(t *testing.T) { + pools, err := configuredMigrationPools(poolConfigFixture()) + if err != nil { + t.Fatal(err) + } + nodes, j := temporaryOwnedCordonFixture(t) + if _, _, err = resolveTemporaryInventoryTarget(j.TemporarySelection, nodes, pools); err == nil { + t.Fatal("ordinary read-only selection accepted a cordoned temporary node") + } + view, err := temporaryInventoryNodes(nodes, &j) + if err != nil { + t.Fatal(err) + } + target, _, err := resolveTemporaryInventoryTarget(j.TemporarySelection, view, pools) + if err != nil || target != j.Target || value(nodes[0], "spec", "unschedulable") != true { + t.Fatal("owned cordon could not be observed without changing the original node:", err) + } + for name, mutate := range map[string]func([]object, *journal){ + "foreign fence": func(n []object, _ *journal) { + n[0]["metadata"].(object)["annotations"].(object)[fenceAnnotation] = "foreign" + }, + "missing fence": func(n []object, _ *journal) { + delete(n[0]["metadata"].(object)["annotations"].(object), fenceAnnotation) + }, + "replaced node": func(n []object, _ *journal) { n[0]["metadata"].(object)["uid"] = "replacement" }, + "changed hardware": func(n []object, _ *journal) { + n[0]["status"].(object)["nodeInfo"].(object)["systemUUID"] = "replacement" + }, + "changed boot": func(n []object, _ *journal) { + n[0]["status"].(object)["nodeInfo"].(object)["bootID"] = "12345678-1234-4234-8234-123456789abc" + }, + "changed provider": func(n []object, _ *journal) { n[0]["spec"].(object)["providerID"] = "hcloud://9999" }, + "another slot": func(_ []object, j *journal) { j.TemporarySelection = "temporary-v1-2" }, + "no physical proof": func(_ []object, j *journal) { j.NativeEmptyDigest = "" }, + "wrong schema": func(_ []object, j *journal) { j.Schema = 1 }, + "cordoned peer": func(n []object, _ *journal) { n[1]["spec"].(object)["unschedulable"] = true }, + } { + t.Run(name, func(t *testing.T) { + nodes, j := temporaryOwnedCordonFixture(t) + mutate(nodes, &j) + view, err := temporaryInventoryNodes(nodes, &j) + if err == nil { + _, _, err = resolveTemporaryInventoryTarget(j.TemporarySelection, view, pools) + } + if err == nil { + t.Fatal("an unowned, replaced or unproven cordon was accepted") + } + }) + } +} From d967e09f7f62d3be4c8b1b60790e5aad4ae6f996 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 2 Nov 2026 23:56:58 +0100 Subject: [PATCH 05/35] feat(storage): prepare empty temporary workers with a separate recovery journal --- .../workflows/prepare-longhorn-v2-node.yaml | 7 +- scripts/migrate-longhorn-v2/README.md | 35 +- scripts/migrate-longhorn-v2/inventory.go | 8 +- .../inventory_temporary.go | 10 +- .../inventory_temporary_test.go | 7 + scripts/migrate-longhorn-v2/main.go | 47 +++ .../temporary_authorization.go | 10 + .../temporary_authorization_test.go | 35 ++ .../migrate-longhorn-v2/temporary_prepare.go | 381 ++++++++++++++++++ .../temporary_prepare_flow_test.go | 264 ++++++++++++ .../migrate-longhorn-v2/temporary_recovery.go | 34 ++ .../temporary_recovery_test.go | 45 +++ 12 files changed, 871 insertions(+), 12 deletions(-) create mode 100644 scripts/migrate-longhorn-v2/temporary_authorization.go create mode 100644 scripts/migrate-longhorn-v2/temporary_authorization_test.go create mode 100644 scripts/migrate-longhorn-v2/temporary_prepare.go create mode 100644 scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go create mode 100644 scripts/migrate-longhorn-v2/temporary_recovery.go create mode 100644 scripts/migrate-longhorn-v2/temporary_recovery_test.go diff --git a/.github/workflows/prepare-longhorn-v2-node.yaml b/.github/workflows/prepare-longhorn-v2-node.yaml index c0cbafd64a..d1cf6503f2 100644 --- a/.github/workflows/prepare-longhorn-v2-node.yaml +++ b/.github/workflows/prepare-longhorn-v2-node.yaml @@ -17,9 +17,9 @@ on: description: The guarded migration step to run required: true type: choice - options: [prepare-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas] + options: [prepare-worker, prepare-unused-temporary-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas] node: - description: Permanent worker to prepare, or a fixed read-only inventory selection + description: Permanent worker or fixed temporary-worker selection for the chosen operation required: false type: choice options: [prod-worker-1, prod-worker-2, prod-worker-3, temporary-v1-1, temporary-v1-2, temporary-v2-1] @@ -80,6 +80,9 @@ jobs: prepare-worker) [[ "${LONGHORN_V2_NODE}" =~ ^prod-worker-[123]$ && "${LONGHORN_V2_CONFIRM}" == "prepare-longhorn-v2-${LONGHORN_V2_NODE}" ]] ;; + prepare-unused-temporary-worker) + [[ "${LONGHORN_V2_NODE}" =~ ^temporary-v1-[12]$ && "${LONGHORN_V2_CONFIRM}" == "prepare-unused-longhorn-v2-${LONGHORN_V2_NODE}" ]] + ;; inventory-replicas) [[ "${LONGHORN_V2_NODE}" =~ ^(prod-worker-[123]|temporary-v1-[12]|temporary-v2-1)$ && "${LONGHORN_V2_CONFIRM}" == "inventory-longhorn-replicas-${LONGHORN_V2_NODE}" ]] ;; diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index c3550acd92..828ac7253f 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -3,15 +3,40 @@ The `Prepare Longhorn V2 Migration` workflow runs guarded steps towards platform#5582. It does not migrate a claim, enable V2, disable V1 or remove iSCSI. -This tool lands in stages (platform#5655). Today it has `refresh-clients`, `prepare-worker` and the -benchmarks. `refresh-clients` holds until the fixed migration workers are Ready and the host +This tool lands in stages (platform#5655). It includes `refresh-clients`, `prepare-worker`, +`prepare-unused-temporary-worker`, read-only replica inventories and the benchmarks. +`refresh-clients` holds until the fixed migration workers are Ready and the host huge-page settings are loaded. Worker preparation is below and is dormant; enabling temporary capacity follows in its own change. +For the unused temporary-worker route (platform#5787), finish the client refresh and the +cleaned V1 baseline first. Select `prepare-unused-temporary-worker`, `temporary-v1-1` or +`temporary-v1-2`, and the matching confirmation `prepare-unused-longhorn-v2-temporary-v1-1` +or `prepare-unused-longhorn-v2-temporary-v1-2`. Each selection is resolved from the complete, +reviewed two-member V1 pool and one-member V2 pool. The fresh V2 pool has no reset operation. + +This operation wipes **only the selected temporary worker's EPHEMERAL partition** and reboots +it. Its separate journal binds the original node UID, provider and hardware identities, boot, +encrypted partition UUID, and native empty-storage receipt. It requires unregistered, fenced +Longhorn storage, complete native replica listings and block-resource inventories, and no +allocated raw partition. Unaccounted replica-root entries, incomplete reads or changed identities hold the run. +All three temporary workers receive owned scale-down protection before the selected worker is +cordoned. Drain uses eviction and respects disruption budgets. Two fresh storage observations +after drain precede the recorded reset intent. + +Recovery from `reset-requested` only observes post-state. An uncertain response never causes a +second reset. A new boot and encrypted EPHEMERAL UUID, the reviewed 96 GiB encrypted raw +partition, loaded configuration and huge-page readiness must be observed before releasing the +owned cordon. CPX42 workers retain their larger root capacity; the permanent-worker root-size +cap does not apply to this separate profile. The raw disk remains unregistered and fenced; +storage activation and volume migration follow separately. Original storage replicas remain on +the permanent workers throughout this preparation route. + Dispatch only the merged workflow on `main`. Run `refresh-clients` promptly after -`talos/workers/enable-longhorn-v2.yaml` deploys: the hosts then hold the pages, but kubelet -still counts them as free memory, so an unrelated kubelet restart could reject Pods. -Select `refresh-clients` with +`talos/workers/enable-longhorn-v2.yaml` deploys. That configuration requests the pages; +the refresh must prove the actual allocation before changing kubelet capacity. Until +capacity is refreshed, kubelet can still count reserved pages as free memory, so an +unrelated kubelet restart could reject Pods. Select `refresh-clients` with `refresh-longhorn-v2-clients`. It verifies the live host allocation and that each kernel module is usable, loaded or built in, restarts only kubelet where capacity is stale, and requires every worker to report 2 GiB of 2 MiB huge pages. Containers and disks are preserved. Kubelet would reject diff --git a/scripts/migrate-longhorn-v2/inventory.go b/scripts/migrate-longhorn-v2/inventory.go index 4de6763990..be40e37f34 100644 --- a/scripts/migrate-longhorn-v2/inventory.go +++ b/scripts/migrate-longhorn-v2/inventory.go @@ -14,9 +14,9 @@ import ( ) type inventoryProjection struct { - Address, BootID, Digest string - Replicas []longhorninventory.Replica - Registered bool + Address, BootID, Digest, StorageUID string + Replicas []longhorninventory.Replica + Registered bool } var errInventory = errors.New("replica inventory identity, placement or completeness is unproven") @@ -72,7 +72,7 @@ func inventoryFailureMessage(err error) string { } func projectInventory(target string, node, storage object, replicas []object) (inventoryProjection, error) { - p := inventoryProjection{BootID: text(node, "status", "nodeInfo", "bootID")} + p := inventoryProjection{BootID: text(node, "status", "nodeInfo", "bootID"), StorageUID: text(storage, "metadata", "uid")} if (!inventoryTarget.MatchString(target) && !inventoryTemporaryNode.MatchString(target)) || !inventoryBoot.MatchString(p.BootID) || text(node, "metadata", "uid") == "" || text(node, "metadata", "resourceVersion") == "" || text(node, "spec", "providerID") == "" || text(node, "status", "nodeInfo", "systemUUID") == "" || diff --git a/scripts/migrate-longhorn-v2/inventory_temporary.go b/scripts/migrate-longhorn-v2/inventory_temporary.go index 6ae5683e81..fa7300b451 100644 --- a/scripts/migrate-longhorn-v2/inventory_temporary.go +++ b/scripts/migrate-longhorn-v2/inventory_temporary.go @@ -11,6 +11,10 @@ import ( // Resolve an ordinal only from the complete reviewed pool roster. The roster // commitment binds the second observation even when another pool member changes. func resolveTemporaryInventoryTarget(selection string, nodes []object, pools map[string]migrationPool) (string, string, error) { + return resolveTemporarySelection(selection, nodes, pools, nil) +} + +func resolveTemporarySelection(selection string, nodes []object, pools map[string]migrationPool, postReset *journal) (string, string, error) { var role string var ordinal int switch selection { @@ -45,7 +49,11 @@ func resolveTemporaryInventoryTarget(selection string, nodes []object, pools map continue } pool, exists := pools[nodeRole] - if !exists || verifyMigrationNode(node, pool) != nil || !inventoryBoot.MatchString(text(node, "status", "nodeInfo", "bootID")) { + memberErr := verifyMigrationNode(node, pool) + if postReset != nil && name == postReset.Target { + memberErr = verifyTemporaryResetMember(node, pool, *postReset) + } + if !exists || memberErr != nil || !inventoryBoot.MatchString(text(node, "status", "nodeInfo", "bootID")) { return "", "", errInventory } identity := object{"name": name, "uid": text(node, "metadata", "uid"), "providerID": text(node, "spec", "providerID"), diff --git a/scripts/migrate-longhorn-v2/inventory_temporary_test.go b/scripts/migrate-longhorn-v2/inventory_temporary_test.go index 927e9889ec..11c37b38f9 100644 --- a/scripts/migrate-longhorn-v2/inventory_temporary_test.go +++ b/scripts/migrate-longhorn-v2/inventory_temporary_test.go @@ -251,6 +251,13 @@ func TestTemporaryInventoryNativeWorkflowGuardPreservesResetBoundary(t *testing. operation, node, ref, attempt, confirm string allowed bool }{ + {"prepare-unused-temporary-worker", "temporary-v1-1", "refs/heads/main", "1", "prepare-unused-longhorn-v2-temporary-v1-1", true}, + {"prepare-unused-temporary-worker", "temporary-v1-2", "refs/heads/main", "1", "prepare-unused-longhorn-v2-temporary-v1-2", true}, + {"prepare-unused-temporary-worker", "temporary-v2-1", "refs/heads/main", "1", "prepare-unused-longhorn-v2-temporary-v2-1", false}, + {"prepare-unused-temporary-worker", "prod-worker-1", "refs/heads/main", "1", "prepare-unused-longhorn-v2-prod-worker-1", false}, + {"prepare-unused-temporary-worker", "temporary-v1-1", "refs/heads/feature", "1", "prepare-unused-longhorn-v2-temporary-v1-1", false}, + {"prepare-unused-temporary-worker", "temporary-v1-1", "refs/heads/main", "2", "prepare-unused-longhorn-v2-temporary-v1-1", false}, + {"prepare-unused-temporary-worker", "temporary-v1-1", "refs/heads/main", "1", "inventory-longhorn-replicas-temporary-v1-1", false}, {"inventory-replicas", "temporary-v1-1", "refs/heads/main", "1", "inventory-longhorn-replicas-temporary-v1-1", true}, {"inventory-replicas", "temporary-v1-2", "refs/heads/main", "1", "inventory-longhorn-replicas-temporary-v1-2", true}, {"inventory-replicas", "temporary-v2-1", "refs/heads/main", "1", "inventory-longhorn-replicas-temporary-v2-1", true}, diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index 4cc9a67ea5..82d7f19862 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "time" @@ -14,6 +15,8 @@ func main() { switch operation { case "prepare-worker": authorization = authorize(os.Getenv, target) + case "prepare-unused-temporary-worker": + authorization = authorizeTemporaryPreparation(os.Getenv, target) case "benchmark-v1", "benchmark-v2": authorization = authorizeBenchmark(os.Getenv, operation[len("benchmark-"):]) case "inventory-replicas": @@ -52,6 +55,50 @@ func main() { fmt.Fprintln(os.Stderr, err) os.Exit(1) } + if operation == "prepare-unused-temporary-worker" { + c := client{run: execute, context: "admin@prod", inventoryPools: pools} + nodes, readErr := c.list(ctx, "nodes", "") + if readErr != nil { + fmt.Fprintln(os.Stderr, "Temporary preparation held: node inventory is unreadable.") + os.Exit(1) + } + resolved, _, resolveErr := resolveTemporaryInventoryTarget(target, nodes, pools) + if resolveErr != nil { + // Recovery may begin with the selected worker's owned cordon. A + // journal, not the ordinal alone, must prove that exception. + cm, cmErr := c.get(ctx, "configmap", "longhorn-system", "longhorn-v2-prepare-"+target, true) + if cmErr != nil || cm == nil { + fmt.Fprintln(os.Stderr, "Temporary preparation held: reviewed pool selection is unproven.") + os.Exit(1) + } + var record journal + if json.Unmarshal([]byte(text(cm, "data", "record")), &record) != nil || record.TemporarySelection != target { + fmt.Fprintln(os.Stderr, "Temporary preparation held: recovery identity is unproven.") + os.Exit(1) + } + if record.Phase == "reset-requested" { + resolved, resolveErr = resolveTemporaryResetRecovery(target, nodes, pools, record) + } else { + view, viewErr := temporaryInventoryNodes(nodes, &record) + if viewErr != nil { + fmt.Fprintln(os.Stderr, "Temporary preparation held: owned cordon is unproven.") + os.Exit(1) + } + resolved, _, resolveErr = resolveTemporaryInventoryTarget(target, view, pools) + } + if resolveErr != nil || resolved != record.Target { + fmt.Fprintln(os.Stderr, "Temporary preparation held: recovery roster changed.") + os.Exit(1) + } + } + p := temporaryPreparation{preparation: preparation{c: c, target: resolved, name: "longhorn-v2-prepare-" + target, temporarySelection: target, source: os.Getenv("GITHUB_SHA"), owner: os.Getenv("GITHUB_RUN_ID"), pools: pools}} + if err = p.prepare(ctx); err != nil { + fmt.Fprintln(os.Stderr, "Unused temporary preparation held:", err) + fmt.Fprintln(os.Stderr, "Preserve its journal and owned cordon; uncertain resets are never replayed.") + os.Exit(1) + } + return + } if operation == "prepare-worker" { p := preparation{c: client{run: execute, context: "admin@prod"}, target: target, name: "longhorn-v2-prepare-" + target, source: os.Getenv("GITHUB_SHA"), owner: os.Getenv("GITHUB_RUN_ID"), pools: pools} diff --git a/scripts/migrate-longhorn-v2/temporary_authorization.go b/scripts/migrate-longhorn-v2/temporary_authorization.go new file mode 100644 index 0000000000..deb529f221 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_authorization.go @@ -0,0 +1,10 @@ +package main + +import "errors" + +func authorizeTemporaryPreparation(getenv func(string) string, selection string) error { + if (selection != "temporary-v1-1" && selection != "temporary-v1-2") || getenv("LONGHORN_V2_NODE") != selection || getenv("LONGHORN_V2_OPERATION") != "prepare-unused-temporary-worker" { + return errors.New("the separate unused-temporary-worker preparation selection is required") + } + return authorizeWorkflow(getenv, "prepare-unused-longhorn-v2-"+selection) +} diff --git a/scripts/migrate-longhorn-v2/temporary_authorization_test.go b/scripts/migrate-longhorn-v2/temporary_authorization_test.go new file mode 100644 index 0000000000..87e00d08e2 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_authorization_test.go @@ -0,0 +1,35 @@ +package main + +import "testing" + +func TestUnusedTemporaryPreparationRequiresItsSeparateExactDispatch(t *testing.T) { + for _, selection := range []string{"temporary-v1-1", "temporary-v1-2"} { + env := dispatchFixture() + env["LONGHORN_V2_OPERATION"], env["LONGHORN_V2_NODE"] = "prepare-unused-temporary-worker", selection + env["LONGHORN_V2_CONFIRM"] = "prepare-unused-longhorn-v2-" + selection + getenv := func(key string) string { return env[key] } + if err := authorizeTemporaryPreparation(getenv, selection); err != nil { + t.Fatal("confirmed unused temporary preparation was rejected:", err) + } + if authorize(getenv, selection) == nil || authorizeInventory(getenv, selection) == nil || authorizeRefresh(getenv) == nil || authorizeBenchmark(getenv, "v1") == nil { + t.Fatal("temporary preparation authorized a different workflow operation") + } + for key, bad := range map[string]string{ + "LONGHORN_V2_NODE": "prod-worker-1", "LONGHORN_V2_CONFIRM": "prepare-longhorn-v2-prod-worker-1", + "LONGHORN_V2_OPERATION": "inventory-replicas", "GITHUB_REF": "refs/heads/candidate", "GITHUB_RUN_ATTEMPT": "2", "GITHUB_WORKFLOW_REF": "other-workflow", "GITHUB_SHA": "abbreviated", + } { + old := env[key] + env[key] = bad + if authorizeTemporaryPreparation(getenv, selection) == nil { + t.Fatal("foreign or unconfirmed temporary preparation accepted:", key) + } + env[key] = old + } + for _, bad := range []string{"temporary-v2-1", "temporary-v1-3", "prod-control-plane-1", "prod-worker-1", "autoscale-longhorn-v1-0123456789abcdef", "10.0.99.4"} { + env["LONGHORN_V2_NODE"], env["LONGHORN_V2_CONFIRM"] = bad, "prepare-unused-longhorn-v2-"+bad + if authorizeTemporaryPreparation(getenv, bad) == nil { + t.Fatal("a nonempty, fresh-V2, permanent or arbitrary target was admitted") + } + } + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_prepare.go b/scripts/migrate-longhorn-v2/temporary_prepare.go new file mode 100644 index 0000000000..56912d17d3 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_prepare.go @@ -0,0 +1,381 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "maps" +) + +type temporaryPreparation struct { + preparation + reader inventoryReaderFactory +} + +func (p *temporaryPreparation) prepare(ctx context.Context) error { + if err := p.beginUnused(ctx); err != nil { + return err + } + if p.j.Phase == "evacuating" { + if err := p.protectUnusedCapacity(ctx); err != nil { + return err + } + n, _, err := p.node(ctx, true) + if err != nil { + return err + } + fence := text(n, "metadata", "annotations", fenceAnnotation) + if (fence != "" && fence != p.name) || (value(n, "spec", "unschedulable") == true && fence != p.name) { + return errInventory + } + patch := append(annotationPatch(n, p.name), object{"op": "add", "path": "/spec/unschedulable", "value": true}) + if err = p.mutate(ctx, "node", "", n, patch...); err != nil { + return err + } + if err = p.advance(ctx, "draining"); err != nil { + return err + } + } + if p.j.Phase == "draining" { + if err := p.resetUnused(ctx); err != nil { + return err + } + } + if p.j.Phase == "reset-requested" { + if err := waitReady(ctx, func(ctx context.Context) error { _, err := p.unusedPostState(ctx); return err }); err != nil { + return err + } + n, err := p.unusedPostState(ctx) + if err != nil { + return err + } + fence := text(n, "metadata", "annotations", fenceAnnotation) + if fence == p.name && value(n, "spec", "unschedulable") == true { + if err = p.mutate(ctx, "node", "", n, object{"op": "add", "path": "/spec/unschedulable", "value": false}, object{"op": "remove", "path": "/metadata/annotations/storage.platform.devantler.tech~1longhorn-v2-preparation"}); err != nil { + return err + } + } else if fence != "" || value(n, "spec", "unschedulable") == true { + return errInventory + } + if err = p.advance(ctx, "prepared"); err != nil { + return err + } + } + if p.j.Phase != "prepared" { + return errInventory + } + n, err := p.unusedPostState(ctx) + if err != nil || value(n, "spec", "unschedulable") == true || text(n, "metadata", "annotations", fenceAnnotation) != "" { + return errInventory + } + s, err := p.c.snapshot(ctx) + if err != nil { + return err + } + if err = p.unusedCapacity(s, true); err != nil { + return err + } + if err = verifyRedundancy(s); err != nil { + return err + } + fmt.Println("Unused temporary worker recovered with encrypted raw storage; Longhorn storage remains fenced and unregistered.") + return nil +} + +func (p temporaryPreparation) unusedCapacity(s inventory, pinned bool) error { + nodes := s.nodes + for _, n := range nodes { + if text(n, "metadata", "name") == p.target && value(n, "spec", "unschedulable") == true { + var err error + nodes, err = temporaryInventoryNodes(nodes, &p.j) + if err != nil { + return err + } + break + } + } + if err := verifyCapacityRoster(nodes, p.pools, false); err != nil { + return err + } + for _, n := range nodes { + role := text(n, "metadata", "labels", migrationPoolLabel) + if role != "" { + owner, pin := text(n, "metadata", "annotations", capacityOwnerAnnotation), text(n, "metadata", "annotations", scaleDownDisabled) + if (owner != "" && owner != capacityOwnerValue) || (pin != "" && pin != "true") || (pinned && (owner != capacityOwnerValue || pin != "true")) { + return errInventory + } + } + if role != "" || inventoryTarget.MatchString(text(n, "metadata", "name")) { + if !hugepagesReady(n) { + return errInventory + } + } + } + return nil +} + +func (p *temporaryPreparation) beginUnused(ctx context.Context) error { + cm, _, err := p.readRecord(ctx) + if err != nil { + return err + } + if cm != nil { + if err = p.claimRecord(ctx, journal{}); err != nil { + return err + } + n, _, nodeErr := p.node(ctx, true) + if nodeErr != nil { + return nodeErr + } + lh, storageErr := p.storageNode(ctx) + if storageErr != nil || !unusedStorageFenced(lh) { + return errInventory + } + if (p.j.Phase == "evacuating" || p.j.Phase == "draining") && text(n, "status", "nodeInfo", "bootID") != p.j.OriginalBootID { + return errInventory + } + return nil + } + s, err := p.c.snapshot(ctx) + if err != nil { + return err + } + if err = p.unusedCapacity(s, false); err != nil { + return err + } + if err = verifyRedundancy(s); err != nil { + return err + } + if err = requireBenchmarkBaseline(ctx, p.c, s.nodes); err != nil { + return err + } + c := p.c + c.inventoryPools = p.pools + _, n, err := c.inventoryProjection(ctx, p.temporarySelection) + if err != nil || text(n, "metadata", "name") != p.target || value(n, "spec", "unschedulable") == true || text(n, "metadata", "annotations", fenceAnnotation) != "" { + return errInventory + } + lh, err := p.c.get(ctx, "nodes.longhorn.io", "longhorn-system", p.target, false) + if err != nil { + return err + } + p.j = journal{Schema: 2, Target: p.target, Source: p.source, OwnerRun: p.owner, Phase: "evacuating", NodeUID: text(n, "metadata", "uid"), ProviderID: text(n, "spec", "providerID"), MachineUUID: text(n, "status", "nodeInfo", "systemUUID"), LonghornUID: text(lh, "metadata", "uid"), OriginalDisks: object{}, TemporarySelection: p.temporarySelection, OriginalBootID: text(n, "status", "nodeInfo", "bootID")} + _, address, err := p.node(ctx, true) + if err != nil { + return err + } + if err = p.desiredLayout(ctx, address); err != nil { + return err + } + eph, err := p.c.talosVolume(ctx, address, "EPHEMERAL") + if err != nil || text(eph, "spec", "phase") != "ready" || text(eph, "spec", "encryptionProvider") != "luks2" { + return errInventory + } + p.j.EphemeralUUID = text(eph, "spec", "uuid") + digest, err := p.observeUnused(ctx, false) + if err != nil { + return err + } + p.j.NativeEmptyDigest, p.j.RawVolumeAbsent = digest, true + return p.claimRecord(ctx, p.j) +} + +func (p temporaryPreparation) observeUnused(ctx context.Context, owned bool) (string, error) { + if err := sourceCurrent(ctx, p.c.run, p.source); err != nil { + return "", err + } + c := p.c + c.inventoryPools = p.pools + if owned { + c.inventoryPreparation = &p.j + } + before, n, err := c.inventoryProjection(ctx, p.temporarySelection) + if err != nil { + return "", err + } + if _, err = verifyNode(n, p.j, true); err != nil || before.BootID != p.j.OriginalBootID || before.StorageUID != p.j.LonghornUID { + return "", errInventory + } + factory := p.reader + if factory == nil { + factory = func(ctx context.Context, address string) (nativeReplicaReader, func() error, error) { + return openInventoryReader(ctx, address, p.j.EphemeralUUID) + } + } + receipt, err := observeReplicaInventory(ctx, c, p.temporarySelection, factory) + if err != nil { + return "", err + } + if receipt.APIProjectionDigest != before.Digest || receipt.Registered || !receipt.Report.EmptyAndAccounted { + return "", errInventory + } + if err = sourceCurrent(ctx, p.c.run, p.source); err != nil { + return "", err + } + digest := sha256.Sum256([]byte(receipt.APIProjectionDigest + ":" + receipt.Report.Digest)) + return hex.EncodeToString(digest[:]), nil +} + +func (p temporaryPreparation) protectUnusedCapacity(ctx context.Context) error { + s, err := p.c.snapshot(ctx) + if err != nil { + return err + } + if err = p.unusedCapacity(s, false); err != nil { + return err + } + for _, expected := range s.nodes { + role := text(expected, "metadata", "labels", migrationPoolLabel) + if role == "" { + continue + } + n, err := p.c.get(ctx, "node", "", text(expected, "metadata", "name"), false) + if err != nil { + return err + } + if _, err = verifyNode(n, journal{Target: text(expected, "metadata", "name"), NodeUID: text(expected, "metadata", "uid"), ProviderID: text(expected, "spec", "providerID"), MachineUUID: text(expected, "status", "nodeInfo", "systemUUID")}, true); err != nil { + return err + } + view := []object{n} + if value(n, "spec", "unschedulable") == true { + view, err = temporaryInventoryNodes(view, &p.j) + if err != nil { + return err + } + } + if err = verifyMigrationNode(view[0], p.pools[role]); err != nil { + return err + } + annotations := maps.Clone(valueObject(n, "metadata", "annotations")) + if annotations == nil { + annotations = object{} + } + owner, pin := text(n, "metadata", "annotations", capacityOwnerAnnotation), text(n, "metadata", "annotations", scaleDownDisabled) + if owner != "" && owner != capacityOwnerValue || pin != "" && pin != "true" { + return errInventory + } + if owner == capacityOwnerValue && pin == "true" { + continue + } + annotations[capacityOwnerAnnotation], annotations[scaleDownDisabled] = capacityOwnerValue, "true" + if err = p.mutate(ctx, "node", "", n, object{"op": "add", "path": "/metadata/annotations", "value": annotations}); err != nil { + return err + } + } + current, err := p.c.snapshot(ctx) + if err != nil { + return err + } + return p.unusedCapacity(current, true) +} + +func (p *temporaryPreparation) resetUnused(ctx context.Context) error { + n, address, err := p.node(ctx, true) + if err != nil { + return err + } + if value(n, "spec", "unschedulable") != true || text(n, "metadata", "annotations", fenceAnnotation) != p.name { + return errInventory + } + if err = p.checkOwner(ctx); err != nil { + return err + } + if _, err = p.c.kubectl(ctx, nil, "drain", p.target, "--ignore-daemonsets", "--delete-emptydir-data", "--timeout=40m"); err != nil { + return errors.New("temporary worker drain was blocked; no reset was requested") + } + for range 2 { + s, err := p.c.snapshot(ctx) + if err != nil { + return err + } + if err = p.unusedCapacity(s, true); err != nil { + return err + } + if err = verifyEvacuated(p.target, s); err != nil { + return err + } + if err = requireBenchmarkBaseline(ctx, p.c, s.nodes); err != nil { + return err + } + if err = p.desiredLayout(ctx, address); err != nil { + return err + } + if _, err = p.observeUnused(ctx, true); err != nil { + return err + } + } + if _, _, err = p.node(ctx, true); err != nil { + return err + } + if err = p.advance(ctx, "reset-requested"); err != nil { + return err + } + _, resetErr := p.c.run(ctx, nil, "talosctl", "--nodes="+address, "reset", "--system-labels-to-wipe=EPHEMERAL", "--reboot", "--graceful=true", "--wait", "--timeout=15m") + if resetErr != nil { + fmt.Println("Temporary reset response was uncertain; only positive recovery can advance the journal.") + } + return nil +} + +func (p temporaryPreparation) unusedPostState(ctx context.Context) (node object, err error) { + n, address, err := p.node(ctx, true) + if err != nil { + return nil, err + } + status := valueObject(n, "status") + boot := text(n, "status", "nodeInfo", "bootID") + if !ready(status) || !hugepagesReady(n) || !inventoryBoot.MatchString(boot) || boot == p.j.OriginalBootID { + return nil, errInventory + } + factory := p.reader + if factory == nil { + factory = newInventoryReader + } + reader, closeReader, err := factory(ctx, address) + if err != nil || reader == nil || closeReader == nil { + return nil, errInventory + } + defer func() { + if closeReader() != nil { + node = nil + err = errInventory + } + }() + actualBoot, err := reader.BootID(ctx) + if err != nil || actualBoot != boot { + return nil, errInventory + } + if err = p.desiredLayout(ctx, address); err != nil { + return nil, err + } + raw, err := p.c.talosVolume(ctx, address, "r-longhorn") + if err != nil { + return nil, err + } + eph, err := p.c.talosVolume(ctx, address, "EPHEMERAL") + if err != nil { + return nil, err + } + if text(eph, "spec", "uuid") == p.j.EphemeralUUID { + return nil, errInventory + } + if err = verifyEncryptedLocalLayout(raw, eph); err != nil { + return nil, err + } + lh, err := p.storageNode(ctx) + if err != nil { + return nil, err + } + if !unusedStorageFenced(lh) { + return nil, errInventory + } + return n, nil +} + +func unusedStorageFenced(lh object) bool { + disks, ok := value(lh, "spec", "disks").(map[string]any) + diskStatus, statusOK := value(lh, "status", "diskStatus").(map[string]any) + return ok && statusOK && len(disks) == 0 && len(diskStatus) == 0 && value(lh, "spec", "allowScheduling") == false && value(lh, "spec", "evictionRequested") == true && text(lh, "metadata", "labels", migrationEnabledLabel) == "" +} diff --git a/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go new file mode 100644 index 0000000000..cf6f8e91d3 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go @@ -0,0 +1,264 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" +) + +type unusedPreparationFixture struct { + base *preparationAPI + baseline object + nativeCalls int + badNativeAfter int + missingBaseline, noRecovery bool + afterReset func() + dropPinWrites bool +} + +func unusedBaselineFixture(t *testing.T, s inventory) object { + t.Helper() + machines, err := benchmarkMachines(s, "v1") + if err != nil { + t.Fatal(err) + } + metrics, err := parseFIO(fioFixture(t)) + if err != nil { + t.Fatal(err) + } + record := benchmarkRecord{Version: 2, Phase: "v1", Source: strings.Repeat("1", 40), Run: "5678", Image: fioImage, Workload: benchmarkWorkload(), Namespace: "longhorn-benchmark-v1-5678", NamespaceUID: "scratch-uid", Machines: machines, Measurements: metrics, Cleaned: true, CompletedAt: "2026-10-29T18:00:00Z"} + compute, managers, probe := benchmarkComputeFixture() + record.CPU, err = benchmarkCompute(compute, managers, probe, "v1") + if err != nil { + t.Fatal(err) + } + record.Resources = map[string]benchmarkResource{ + "Namespace": {Name: record.Namespace, UID: "scratch-uid"}, "Pod": {Name: "fio", Namespace: record.Namespace, UID: "pod"}, + "NetworkPolicy": {Name: "fio-default-deny", Namespace: record.Namespace, UID: "policy"}, "PersistentVolumeClaim": {Name: "fio-data", Namespace: record.Namespace, UID: "claim"}, + "StorageClass": {Name: "lh-benchmark-v1-5678", UID: "class"}, "PersistentVolume": {Name: "scratch-pv", UID: "pv"}, "LonghornVolume": {Name: "scratch-volume", Namespace: "longhorn-system", UID: "volume"}, + } + encoded, err := json.Marshal(record) + if err != nil { + t.Fatal(err) + } + cm := resource("longhorn-v2-benchmark-v1") + cm["data"] = object{"record": string(encoded)} + return cm +} + +func unusedPreparationAPI(t *testing.T) (*unusedPreparationFixture, temporaryPreparation) { + t.Helper() + base, old := newPreparationAPI(t, "evacuating") + base.s = benchmarkInventoryFixture() + base.s.nodes = append(base.s.nodes, capacityFixture()[3:]...) + for i, n := range base.s.nodes { + n["metadata"].(object)["uid"] = "uid-" + text(n, "metadata", "name") + if value(n, "metadata", "annotations") == nil { + n["metadata"].(object)["annotations"] = object{} + } + if value(n, "spec") == nil { + n["spec"] = object{} + } + n["spec"].(object)["providerID"] = fmt.Sprintf("hcloud://%d", 500+i) + n["status"].(object)["nodeInfo"] = object{"systemUUID": "hardware-" + text(n, "metadata", "name"), "bootID": fmt.Sprintf("12345678-1234-4234-8234-%012x", i+1), "osImage": "Talos fixture", "architecture": "amd64"} + n["status"].(object)["addresses"] = []any{object{"type": "InternalIP", "address": fmt.Sprintf("10.0.99.%d", 4+i)}} + if i >= 3 && i < 6 { + delete(n["metadata"].(object)["annotations"].(object), scaleDownDisabled) + delete(n["metadata"].(object)["annotations"].(object), capacityOwnerAnnotation) + lh := resource(text(n, "metadata", "name")) + lh["spec"] = object{"allowScheduling": false, "evictionRequested": true, "disks": object{}} + lh["status"] = object{"diskStatus": object{}} + base.s.storageNodes = append(base.s.storageNodes, lh) + } + } + base.node, base.storage, base.cm = base.s.nodes[3], base.s.storageNodes[3], nil + base.v2Off = true + f := &unusedPreparationFixture{base: base} + f.baseline = unusedBaselineFixture(t, base.s) + p := temporaryPreparation{preparation: preparation{target: text(base.node, "metadata", "name"), name: "longhorn-v2-prepare-temporary-v1-1", temporarySelection: "temporary-v1-1", source: old.source, owner: old.owner, pools: old.pools}} + p.c = client{context: "admin@prod", run: f.run, inventoryPools: old.pools} + p.reader = func(_ context.Context, address string) (nativeReplicaReader, func() error, error) { + f.nativeCalls++ + if address != "10.0.99.7" { + t.Fatal("temporary inspection selected another endpoint") + } + return temporaryNativeReader{boot: text(base.node, "status", "nodeInfo", "bootID"), unmatched: f.badNativeAfter > 0 && f.nativeCalls >= f.badNativeAfter}, func() error { return nil }, nil + } + return f, p +} + +func (f *unusedPreparationFixture) run(ctx context.Context, input []byte, program string, args ...string) ([]byte, error) { + f.base.t.Helper() + if program == "gh" { + return json.Marshal(object{"status": "completed", "head_sha": strings.Repeat("1", 40), "head_branch": "main", "event": "workflow_dispatch", "run_attempt": 1, "path": ".github/workflows/prepare-longhorn-v2-node.yaml"}) + } + if program == "kubectl" && len(args) > 3 { + if args[2] == "get" && strings.HasPrefix(args[3], "--raw=") { + api, kind := "v1", "NodeList" + var items []object + switch args[3] { + case "--raw=" + inventoryKubeNodes: + items = f.base.s.nodes + case "--raw=" + inventoryStorageNodes: + api = "longhorn.io/v1beta2" + items = f.base.s.storageNodes + case "--raw=" + inventoryReplicas: + api, kind = "longhorn.io/v1beta2", "ReplicaList" + items = f.base.s.replicas + default: + f.base.t.Fatal("unexpected raw request") + } + return json.Marshal(object{"apiVersion": api, "kind": kind, "metadata": object{"resourceVersion": "1"}, "items": items}) + } + if len(args) > 4 && args[2] == "get" && args[3] == "configmap" && args[4] == "longhorn-v2-benchmark-v1" { + if f.missingBaseline { + return nil, nil + } + return json.Marshal(f.baseline) + } + if len(args) > 4 && (args[2] == "get" || args[2] == "patch") && args[3] == "node" { + if args[2] == "patch" && f.dropPinWrites { + var patch []object + if json.Unmarshal(input, &patch) != nil { + f.base.t.Fatal("malformed test patch") + } + for _, change := range patch { + if text(change, "path") == "/metadata/annotations" && text(valueObject(change, "value"), capacityOwnerAnnotation) == capacityOwnerValue { + return nil, nil + } + } + } + for _, node := range f.base.s.nodes { + if text(node, "metadata", "name") == args[4] { + original := f.base.node + f.base.node = node + defer func() { f.base.node = original }() + return f.base.run(ctx, input, program, args...) + } + } + f.base.t.Fatal("unknown node request") + } + } + if program == "talosctl" && len(args) > 1 && args[1] == "reset" { + want := []string{"--nodes=10.0.99.7", "reset", "--system-labels-to-wipe=EPHEMERAL", "--reboot", "--graceful=true", "--wait", "--timeout=15m"} + var j journal + if !reflect.DeepEqual(args, want) || json.Unmarshal([]byte(text(f.base.cm, "data", "record")), &j) != nil || j.Schema != 2 || j.Phase != "reset-requested" || f.nativeCalls < 3 { + f.base.t.Fatal("reset lacked its separate journal, fresh physical proof or exact EPHEMERAL scope") + } + f.base.resets++ + if !f.noRecovery { + f.base.postReset = true + f.base.node["status"].(object)["nodeInfo"].(object)["bootID"] = "12345678-1234-4234-8234-000000000099" + } + if f.afterReset != nil { + f.afterReset() + } + if f.base.resetError { + return nil, errors.New("accepted, response lost") + } + return nil, nil + } + data, err := f.base.run(ctx, input, program, args...) + if err == nil && f.base.postReset && program == "talosctl" && len(args) == 5 && args[2] == "volumestatus" && args[3] == "EPHEMERAL" { + var volume object + if json.Unmarshal(data, &volume) != nil { + f.base.t.Fatal("malformed fixture volume") + } + volume["spec"].(object)["size"] = float64(128 << 30) + return json.Marshal(volume) + } + return data, err +} + +func TestUnusedTemporaryPreparationPreservesV1AndLeavesRawStorageFenced(t *testing.T) { + f, p := unusedPreparationAPI(t) + before, _ := json.Marshal(f.base.s.storageNodes[:3]) + if err := p.prepare(context.Background()); err != nil { + t.Fatalf("empty temporary worker could not be prepared: %v (phase=%s native=%d resets=%d snapshots=%d)", err, p.j.Phase, f.nativeCalls, f.base.resets, f.base.snapshots) + } + after, _ := json.Marshal(f.base.s.storageNodes[:3]) + if f.base.resets != 1 || p.j.Phase != "prepared" || !reflect.DeepEqual(before, after) || value(f.base.storage, "spec", "allowScheduling") != false || value(f.base.storage, "spec", "evictionRequested") != true || len(valueObject(f.base.storage, "spec", "disks")) != 0 || value(f.base.node, "spec", "unschedulable") == true || text(f.base.node, "metadata", "annotations", fenceAnnotation) != "" { + t.Fatal("temporary preparation changed V1 storage, enabled raw storage, or failed to recover its node") + } + if err := p.prepare(context.Background()); err != nil || f.base.resets != 1 { + t.Fatal("prepared recovery replayed a reset:", err) + } +} + +func TestUnusedTemporaryPreparationHoldsBeforeResetOnLostEvidence(t *testing.T) { + for _, scenario := range []string{"missing baseline", "native data", "native data after drain", "foreign pin", "unready peer", "source changed", "layout changed"} { + t.Run(scenario, func(t *testing.T) { + f, p := unusedPreparationAPI(t) + switch scenario { + case "missing baseline": + f.missingBaseline = true + case "native data": + f.badNativeAfter = 1 + case "native data after drain": + f.badNativeAfter = 2 + case "foreign pin": + f.base.s.nodes[4]["metadata"].(object)["annotations"].(object)[capacityOwnerAnnotation] = "foreign" + case "unready peer": + f.base.s.nodes[5]["status"].(object)["conditions"] = []any{} + case "source changed": + f.base.staleSource = true + case "layout changed": + f.base.staleLayout = true + } + if p.prepare(context.Background()) == nil || f.base.resets != 0 { + t.Fatal("lost evidence authorized a temporary reset") + } + }) + } +} + +func TestUncertainTemporaryResetIsNeverReplayed(t *testing.T) { + f, p := unusedPreparationAPI(t) + f.base.resetError, f.noRecovery = true, true + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f.afterReset = cancel + if p.prepare(ctx) == nil || f.base.resets != 1 || p.j.Phase != "reset-requested" { + t.Fatal("uncertain reset lost its durable intent") + } + ctx2, cancel2 := context.WithTimeout(context.Background(), time.Second) + defer cancel2() + if p.prepare(ctx2) == nil || f.base.resets != 1 { + t.Fatal("recovery replayed the uncertain destructive request") + } +} + +func TestTemporaryPreparationRebindsIdentityAndStorageBeforeResumingWrites(t *testing.T) { + for _, scenario := range []string{"storage enabled", "node replaced", "unexpected reboot"} { + t.Run(scenario, func(t *testing.T) { + f, p := unusedPreparationAPI(t) + if err := p.beginUnused(context.Background()); err != nil { + t.Fatal(err) + } + switch scenario { + case "storage enabled": + f.base.storage["spec"].(object)["allowScheduling"] = true + case "node replaced": + f.base.node["metadata"].(object)["uid"] = "replacement" + case "unexpected reboot": + f.base.node["status"].(object)["nodeInfo"].(object)["bootID"] = "12345678-1234-4234-8234-000000000099" + } + if p.prepare(context.Background()) == nil || f.base.resets != 0 || value(f.base.node, "spec", "unschedulable") == true || text(f.base.node, "metadata", "annotations", capacityOwnerAnnotation) != "" { + t.Fatal("changed storage or node identity reached a preparation write") + } + }) + } +} + +func TestTemporaryCordonRequiresObservedPoolProtection(t *testing.T) { + f, p := unusedPreparationAPI(t) + f.dropPinWrites = true + if p.prepare(context.Background()) == nil || f.base.resets != 0 || value(f.base.node, "spec", "unschedulable") == true { + t.Fatal("an acknowledgement without live scale-down protection reached the worker cordon") + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_recovery.go b/scripts/migrate-longhorn-v2/temporary_recovery.go new file mode 100644 index 0000000000..bf67dc88ab --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_recovery.go @@ -0,0 +1,34 @@ +package main + +func resolveTemporaryResetRecovery(selection string, nodes []object, pools map[string]migrationPool, record journal) (string, error) { + if record.Schema != 2 || record.Phase != "reset-requested" || record.TemporarySelection != selection || validateJournal(record, record.Target) != nil { + return "", errInventory + } + target, _, err := resolveTemporarySelection(selection, nodes, pools, &record) + if err != nil || target != record.Target { + return "", errInventory + } + return target, nil +} + +// This exception selects only the already journaled physical node for observing +// recovery. It proves no readiness and grants no new reset. The state machine's +// reset-requested phase can only inspect post-state, never replay destruction. +func verifyTemporaryResetMember(n object, pool migrationPool, record journal) error { + if pool.role != "v1" || pool.name != "autoscale-longhorn-v1" || text(n, "metadata", "labels", migrationPoolLabel) != pool.role || + text(n, "metadata", "labels", "ksail.io/autoscaled") != "true" || text(n, "metadata", "labels", "node.kubernetes.io/instance-type") != pool.serverType || text(n, "metadata", "resourceVersion") == "" { + return errInventory + } + if _, err := verifyNode(n, record, true); err != nil { + return errInventory + } + fence := text(n, "metadata", "annotations", fenceAnnotation) + if value(n, "spec", "unschedulable") == true { + if fence != "longhorn-v2-prepare-"+record.TemporarySelection { + return errInventory + } + } else if fence != "" { + return errInventory + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/temporary_recovery_test.go b/scripts/migrate-longhorn-v2/temporary_recovery_test.go new file mode 100644 index 0000000000..e4316961fc --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_recovery_test.go @@ -0,0 +1,45 @@ +package main + +import "testing" + +func TestTemporaryResetRecoveryCanObserveItsBoundNodeBeforeItRejoins(t *testing.T) { + pools, err := configuredMigrationPools(poolConfigFixture()) + if err != nil { + t.Fatal(err) + } + nodes, j := temporaryOwnedCordonFixture(t) + j.Phase = "reset-requested" + nodes[0]["status"].(object)["conditions"] = []any{} + nodes[0]["status"].(object)["nodeInfo"].(object)["bootID"] = "12345678-1234-4234-8234-000000000099" + if _, _, err = resolveTemporaryInventoryTarget(j.TemporarySelection, nodes, pools); err == nil { + t.Fatal("ordinary inventory accepted an unready cordoned node") + } + if target, err := resolveTemporaryResetRecovery(j.TemporarySelection, nodes, pools, j); err != nil || target != j.Target { + t.Fatal("reset intent cannot reach read-only recovery:", err) + } + for _, scenario := range []string{"wrong phase", "missing proof", "wrong slot", "foreign fence", "replaced node", "changed provider", "unready peer"} { + t.Run(scenario, func(t *testing.T) { + nodes, j := temporaryOwnedCordonFixture(t) + j.Phase = "reset-requested" + switch scenario { + case "wrong phase": + j.Phase = "draining" + case "missing proof": + j.NativeEmptyDigest = "" + case "wrong slot": + j.TemporarySelection = "temporary-v1-2" + case "foreign fence": + nodes[0]["metadata"].(object)["annotations"].(object)[fenceAnnotation] = "foreign" + case "replaced node": + nodes[0]["metadata"].(object)["uid"] = "replacement" + case "changed provider": + nodes[0]["spec"].(object)["providerID"] = "hcloud://9999" + case "unready peer": + nodes[1]["status"].(object)["conditions"] = []any{} + } + if _, err := resolveTemporaryResetRecovery(j.TemporarySelection, nodes, pools, j); err == nil { + t.Fatal("recovery selected a different or unproven physical node") + } + }) + } +} From 2f4e81b6202f10786c9bc6405172c4d3bbeecdaa Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 00:49:23 +0100 Subject: [PATCH 06/35] docs(storage): distinguish huge-page requests from allocation proof --- scripts/migrate-longhorn-v2/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 78a6a062ec..50a8340d9d 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -18,8 +18,10 @@ huge-page settings are loaded. Worker preparation is below and is dormant; enabl capacity follows in its own change. Dispatch only the merged workflow on `main`. Run `refresh-clients` promptly after -`talos/workers/enable-longhorn-v2.yaml` deploys: the hosts then hold the pages, but kubelet -still counts them as free memory, so an unrelated kubelet restart could reject Pods. +`talos/workers/enable-longhorn-v2.yaml` deploys. That configuration requests the pages; +the refresh must prove the actual allocation before changing kubelet capacity. Until +capacity is refreshed, kubelet can still count reserved pages as free memory, so an +unrelated kubelet restart could reject Pods. Select `refresh-clients` with `refresh-longhorn-v2-clients`. It verifies the live host allocation and that each kernel module is usable, loaded or built in, restarts only kubelet where capacity is stale, and requires every worker to report From 09501cbd5a67777d6d3208189d6f975a87dfb7ee Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 01:49:33 +0100 Subject: [PATCH 07/35] feat(storage): inspect bounded native worker block layouts Part of #5782 and #5791. Keep the original root observation scope unchanged; require separately confirmed layout observations, clean bounded native streams and unchanged identities. No reset clearance is granted. --- .../workflows/prepare-longhorn-v2-node.yaml | 5 +- go.mod | 4 +- go.sum | 35 ++-- internal/longhorninventory/block_layout.go | 166 ++++++++++++++++++ .../longhorninventory/block_layout_test.go | 96 ++++++++++ internal/longhorninventory/block_wire.go | 82 +++++++++ internal/longhorninventory/block_wire_test.go | 98 +++++++++++ internal/longhorninventory/native.go | 1 + internal/longhorninventory/storage_root.go | 2 +- scripts/migrate-longhorn-v2/README.md | 9 + scripts/migrate-longhorn-v2/inventory.go | 3 +- .../migrate-longhorn-v2/inventory_runtime.go | 5 +- .../inventory_temporary_test.go | 6 + scripts/migrate-longhorn-v2/main.go | 9 + .../storage_root_inventory.go | 65 ++++++- .../storage_root_inventory_test.go | 47 ++++- 16 files changed, 589 insertions(+), 44 deletions(-) create mode 100644 internal/longhorninventory/block_layout.go create mode 100644 internal/longhorninventory/block_layout_test.go create mode 100644 internal/longhorninventory/block_wire.go create mode 100644 internal/longhorninventory/block_wire_test.go diff --git a/.github/workflows/prepare-longhorn-v2-node.yaml b/.github/workflows/prepare-longhorn-v2-node.yaml index 862959b0d5..e567fe1077 100644 --- a/.github/workflows/prepare-longhorn-v2-node.yaml +++ b/.github/workflows/prepare-longhorn-v2-node.yaml @@ -17,7 +17,7 @@ on: description: The guarded migration step to run required: true type: choice - options: [prepare-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas, inventory-storage-root, inspect-host-memory] + options: [prepare-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas, inventory-storage-root, inventory-storage-layout, inspect-host-memory] node: description: Permanent worker to prepare, or a fixed read-only inventory selection required: false @@ -89,6 +89,9 @@ jobs: inventory-storage-root) [[ "${LONGHORN_V2_NODE}" =~ ^(prod-worker-[123]|temporary-v1-[12]|temporary-v2-1)$ && "${LONGHORN_V2_CONFIRM}" == "inventory-longhorn-storage-root-${LONGHORN_V2_NODE}" ]] ;; + inventory-storage-layout) + [[ "${LONGHORN_V2_NODE}" =~ ^temporary-(v1-[12]|v2-1)$ && "${LONGHORN_V2_CONFIRM}" == "inventory-longhorn-storage-layout-${LONGHORN_V2_NODE}" ]] + ;; refresh-clients) [[ "${LONGHORN_V2_CONFIRM}" == refresh-longhorn-v2-clients ]] ;; diff --git a/go.mod b/go.mod index 0717befd3e..58a89b553e 100644 --- a/go.mod +++ b/go.mod @@ -55,9 +55,9 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chmduquesne/rollinghash v4.0.0+incompatible // indirect github.com/edsrzf/mmap-go v1.2.0 // indirect - github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/cel-go v0.31.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/cronexpr v1.1.3 // indirect github.com/klauspost/compress v1.19.2 // indirect @@ -96,7 +96,7 @@ require ( github.com/aws/smithy-go v1.28.6 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudflare/circl v1.6.4 // indirect - github.com/cosi-project/runtime v1.16.3 // indirect + github.com/cosi-project/runtime v1.16.3 github.com/gertd/go-pluralize v0.2.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 3fd20e4b14..6f8f21e584 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,6 @@ github.com/frankban/quicktest v1.13.1 h1:xVm/f9seEhZFL9+n5kv5XLrGwy6elc4V9v/XFY2 github.com/frankban/quicktest v1.13.1/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/gertd/go-pluralize v0.2.1 h1:M3uASbVjMnTsPb0PNqg+E/24Vwigyo/tvyMTtAlLgiA= github.com/gertd/go-pluralize v0.2.1/go.mod h1:rbYaKDbsXxmRfr8uygAEKhOWsjyrrqrkHVpZvoOp8zk= -github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= -github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -83,8 +81,8 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= -github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= +github.com/gofrs/flock v0.13.1 h1:jjREztyBeSKBZYAC+mgc1laB+xsgy4kYMf3FbKF2UBo= +github.com/gofrs/flock v0.13.1/go.mod h1:sf4BFiHwnvgxa25DlQoDqXQnwRMEOwqxRq37P6MzzmE= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= @@ -124,8 +122,6 @@ github.com/jsimonetti/rtnetlink/v2 v2.2.1-0.20260802200809-43bafec815b3/go.mod h github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= -github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= @@ -150,14 +146,10 @@ github.com/mdlayher/netlink v1.11.2 h1:HKh2jqe+omdSWcQ88nrT7INE61B0NXfiSPFdgL4Yb github.com/mdlayher/netlink v1.11.2/go.mod h1:uT2Yc/QLaZubzDpZIBi9d4GoeLwtp3x1AMeqSRrK2sA= github.com/mdlayher/socket v0.6.1 h1:M7uj2NtuujUY4mYr1C57NmfNiRHbkKpnBxO856lsc3A= github.com/mdlayher/socket v0.6.1/go.mod h1:+/SGtqc9V+5dAuRgQsU0fGBI+oRDiW7O2Obx10OIWfg= -github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q= -github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ= -github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk= github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -200,8 +192,8 @@ github.com/prometheus/common v0.67.4 h1:yR3NqWO1/UyO1w2PhUvXlGQs/PtFmoveVO0KZ4+L github.com/prometheus/common v0.67.4/go.mod h1:gP0fq6YjjNCLssJCQp0yk4M8W6ikLURwkdd/YKtTbyI= github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= -github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= -github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/rogpeppe/go-internal v1.16.0 h1:O9DK+vNMDVGLr2BeZqmpLeMjiMNkuXfcqntWbZV6S5g= +github.com/rogpeppe/go-internal v1.16.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -227,10 +219,11 @@ github.com/siderolabs/protoenc v0.2.4/go.mod h1:i5XLHjfv5vyi7LhQrSEo19HCA+lYtDd7 github.com/siderolabs/talos/pkg/machinery v1.14.4 h1:reYNIV4XTbUcOSVzlAqkM6psdOlO9U+HYc2AeuvAzfI= github.com/siderolabs/talos/pkg/machinery v1.14.4/go.mod h1:PEzCUOIG+87NIqBsONm/eQEQQfgoOwWL+r5ANhCrQGU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -239,16 +232,14 @@ github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWD github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tg123/go-htpasswd v1.2.4 h1:HgH8KKCjdmo7jjXWN9k1nefPBd7Be3tFCTjc2jPraPU= github.com/tg123/go-htpasswd v1.2.4/go.mod h1:EKThQok9xHkun6NBMynNv6Jmu24A33XdZzzl4Q7H1+0= -github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= -github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/zalando/go-keyring v0.2.6 h1:r7Yc3+H+Ux0+M72zacZoItR3UDxeWfKTcabvkI8ua9s= github.com/zalando/go-keyring v0.2.6/go.mod h1:2TCrxYrbUNYfNS/Kgy/LSrkSQzZ5UPVH85RwfczwvcI= -github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= -github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI= github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= @@ -261,10 +252,6 @@ go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= -go.opentelemetry.io/otel/sdk v1.45.0 h1:4VVSMgQ83dUgW2aoX5f6JgLvHwIvzcuLnF9lUdCSpCw= -go.opentelemetry.io/otel/sdk v1.45.0/go.mod h1:Sr40LgXV7DsKMMJMKOhUWOgMWTfAaqvm2kF0g7ilwuA= -go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= -go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -294,16 +281,14 @@ golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= -golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= -golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754 h1:dWeMvEJ3JhYgqSCAHUZZJgMUyfniiiCvDc72x5EqJP0= google.golang.org/genproto/googleapis/api v0.0.0-20260810153831-ec0a7760b754/go.mod h1:q/3oV3jAi5vwelxsVAprMBC8BcM2zmNe+IjRGd+9/ks= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754 h1:k5CJw9e5ONCcA/u0webKt092npXuY+KeGh3Q8NAVf0g= google.golang.org/genproto/googleapis/rpc v0.0.0-20260810153831-ec0a7760b754/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= -google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= diff --git a/internal/longhorninventory/block_layout.go b/internal/longhorninventory/block_layout.go new file mode 100644 index 0000000000..29d328e8bf --- /dev/null +++ b/internal/longhorninventory/block_layout.go @@ -0,0 +1,166 @@ +package longhorninventory + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "slices" + "strings" + + resource "github.com/cosi-project/runtime/pkg/resource" + "github.com/siderolabs/talos/pkg/machinery/resources/block" +) + +type BlockLayoutReport struct { + VolumeStatuses, DiscoveredVolumes int + EphemeralPhase string + EphemeralSizeBytes uint64 + EphemeralLUKS2, EphemeralLockedToState, EphemeralNodeIDOnly bool + RawPresent bool + RawPhase string + RawSizeBytes uint64 + RawLUKS2, RawLockedToState, RawNodeIDOnly bool + RawDiscoveredPartitions, OtherUserVolumes int + Digest string +} + +func (n Native) ListBlockLayout(ctx context.Context) (BlockLayoutReport, error) { + ctx, cancel, err := n.requestContext(ctx) + if err != nil { + return BlockLayoutReport{}, nativeFailure{code: "request"} + } + defer cancel() + if n.Blocks == nil { + return BlockLayoutReport{}, nativeFailure{code: "block-client"} + } + first, _, err := n.readBlockLayout(ctx) + if err != nil { + return BlockLayoutReport{}, err + } + second, report, err := n.readBlockLayout(ctx) + if err != nil { + return BlockLayoutReport{}, err + } + if !active(ctx) || !bytes.Equal(first, second) { + return BlockLayoutReport{}, nativeFailure{code: "block-change"} + } + encoded, err := json.Marshal(report) + if err != nil { + return BlockLayoutReport{}, nativeFailure{code: "block-shape"} + } + digest := sha256.Sum256(encoded) + report.Digest = hex.EncodeToString(digest[:]) + return report, nil +} + +// Selected layout metadata is compared privately. The public commitment covers +// only reported counters and properties, without private device identities, +// key material, configuration contents, filesystem contents or provider errors. +func (n Native) readBlockLayout(ctx context.Context) ([]byte, BlockLayoutReport, error) { + if !active(ctx) { + return nil, BlockLayoutReport{}, nativeFailure{code: "block-list"} + } + statuses, err := n.Blocks.List(ctx, block.NewVolumeStatus(block.NamespaceName, "").Metadata()) + if err != nil || !active(ctx) { + return nil, BlockLayoutReport{}, nativeFailure{code: "block-list"} + } + discovered, err := n.Blocks.List(ctx, block.NewDiscoveredVolume(block.NamespaceName, "").Metadata()) + if err != nil || !active(ctx) { + return nil, BlockLayoutReport{}, nativeFailure{code: "block-list"} + } + invalid := nativeFailure{code: "block-shape"} + if len(statuses.Items) == 0 || len(statuses.Items) > 128 || len(discovered.Items) == 0 || len(discovered.Items) > 512 { + return nil, BlockLayoutReport{}, invalid + } + report := BlockLayoutReport{VolumeStatuses: len(statuses.Items), DiscoveredVolumes: len(discovered.Items), RawPhase: "absent"} + rows := make([][]byte, 0, len(statuses.Items)+len(discovered.Items)) + metadataBytes := 0 + appendRow := func(row any) bool { + b, err := json.Marshal(row) + metadataBytes += len(b) + if err != nil || metadataBytes > MaxMetadataBytes { + return false + } + rows = append(rows, b) + return true + } + seen := map[string]bool{} + ephFound := false + for _, item := range statuses.Items { + v, ok := item.(*block.VolumeStatus) + if !ok || v == nil || v.Metadata().Namespace() != block.NamespaceName || v.Metadata().Phase() != resource.PhaseRunning || v.Metadata().ID() == "" || len(v.Metadata().ID()) > 256 || seen[v.Metadata().ID()] { + return nil, BlockLayoutReport{}, invalid + } + id, s := v.Metadata().ID(), v.TypedSpec() + seen[id] = true + if s == nil { + return nil, BlockLayoutReport{}, invalid + } + phase := "other" + switch s.Phase { + case block.VolumePhaseReady: + phase = "ready" + case block.VolumePhaseWaiting: + phase = "waiting" + } + nodeIDOnly := len(s.ConfiguredEncryptionKeys) == 1 && s.ConfiguredEncryptionKeys[0] == "nodeID" + switch id { + case "EPHEMERAL": + ephFound = true + report.EphemeralPhase = phase + report.EphemeralSizeBytes = s.Size + report.EphemeralLUKS2 = s.EncryptionProvider == block.EncryptionProviderLUKS2 + report.EphemeralLockedToState = s.EncryptionLockedToState + report.EphemeralNodeIDOnly = nodeIDOnly + case "r-longhorn": + report.RawPresent = true + report.RawPhase = phase + report.RawSizeBytes = s.Size + report.RawLUKS2 = s.EncryptionProvider == block.EncryptionProviderLUKS2 + report.RawLockedToState = s.EncryptionLockedToState + report.RawNodeIDOnly = nodeIDOnly + default: + if strings.HasPrefix(id, "r-") || strings.HasPrefix(id, "u-") { + report.OtherUserVolumes++ + } + } + if !appendRow(struct { + Kind, ID string + Metadata any + }{"status", id, []any{s.Phase, s.Type, s.Location, s.MountLocation, s.ParentLocation, s.UUID, s.PartitionUUID, s.PartitionIndex, s.Size, s.EncryptionProvider, s.EncryptionLockedToState, s.ConfiguredEncryptionKeys}}) { + return nil, BlockLayoutReport{}, invalid + } + } + if !ephFound { + return nil, BlockLayoutReport{}, invalid + } + seen = map[string]bool{} + for _, item := range discovered.Items { + v, ok := item.(*block.DiscoveredVolume) + if !ok || v == nil || v.Metadata().Namespace() != block.NamespaceName || v.Metadata().Phase() != resource.PhaseRunning || v.Metadata().ID() == "" || len(v.Metadata().ID()) > 256 || seen[v.Metadata().ID()] { + return nil, BlockLayoutReport{}, invalid + } + id, s := v.Metadata().ID(), v.TypedSpec() + seen[id] = true + if s == nil || s.DevPath == "" || s.Size == 0 { + return nil, BlockLayoutReport{}, invalid + } + if s.PartitionLabel == "r-longhorn" { + report.RawDiscoveredPartitions++ + } + if !appendRow(struct { + Kind, ID string + Metadata any + }{"discovered", id, []any{s.DevPath, s.ParentDevPath, s.PartitionLabel, s.PartitionUUID, s.Size}}) { + return nil, BlockLayoutReport{}, invalid + } + } + slices.SortFunc(rows, bytes.Compare) + encoded, err := json.Marshal(rows) + if err != nil || len(encoded) > MaxMetadataBytes || !active(ctx) { + return nil, BlockLayoutReport{}, invalid + } + return encoded, report, nil +} diff --git a/internal/longhorninventory/block_layout_test.go b/internal/longhorninventory/block_layout_test.go new file mode 100644 index 0000000000..5d07cf606c --- /dev/null +++ b/internal/longhorninventory/block_layout_test.go @@ -0,0 +1,96 @@ +package longhorninventory + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + resource "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/talos/pkg/machinery/resources/block" + "google.golang.org/grpc/metadata" +) + +type blockLayoutFixture struct { + state.CoreStateReader + t *testing.T + calls int + mode string +} + +func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, options ...state.ListOption) (resource.List, error) { + f.t.Helper() + f.calls++ + deadline, bounded := ctx.Deadline() + md, _ := metadata.FromOutgoingContext(ctx) + if !bounded || time.Until(deadline) > 15*time.Second || len(options) != 0 || kind.Namespace() != block.NamespaceName || len(md.Get("node")) != 1 || md.Get("node")[0] != "10.0.0.8" || len(md.Get("nodes")) != 0 { + f.t.Fatal("block observation escaped its bounded fixed-node scope") + } + var result resource.List + if f.calls%2 == 1 { + if kind.Type() != block.VolumeStatusType { + f.t.Fatal("unexpected status type") + } + eph := block.NewVolumeStatus(block.NamespaceName, "EPHEMERAL") + *eph.TypedSpec() = block.VolumeStatusSpec{Phase: block.VolumePhaseReady, Type: block.VolumeTypePartition, UUID: "private-ephemeral-canary", Location: "/dev/sda6", ParentLocation: "/dev/sda", MountLocation: "/dev/dm-0", Size: 150 << 30, EncryptionProvider: block.EncryptionProviderLUKS2, EncryptionLockedToState: true, ConfiguredEncryptionKeys: []string{"nodeID"}} + raw := block.NewVolumeStatus(block.NamespaceName, "r-longhorn") + raw.TypedSpec().Phase = block.VolumePhaseWaiting + if f.mode == "changed" && f.calls > 2 { + raw.TypedSpec().Phase = block.VolumePhaseReady + } + result.Items = []resource.Resource{eph, raw} + if f.mode == "duplicate" { + result.Items = append(result.Items, eph) + } + if f.mode == "missing ephemeral" { + result.Items = result.Items[1:] + } + if f.mode == "tearing down" { + eph.Metadata().SetPhase(resource.PhaseTearingDown) + } + } else { + if kind.Type() != block.DiscoveredVolumeType { + f.t.Fatal("unexpected discovery type") + } + v := block.NewDiscoveredVolume(block.NamespaceName, "private-device-canary") + *v.TypedSpec() = block.DiscoveredVolumeSpec{DevPath: "/dev/sda6", ParentDevPath: "/dev/sda", PartitionLabel: "EPHEMERAL", PartitionUUID: "private-partition-canary", Size: 150 << 30} + result.Items = []resource.Resource{v} + } + if f.mode == "partial" && f.calls == 2 { + return result, errors.New("private-provider-canary") + } + return result, nil +} + +func TestNativeBlockLayoutReportsObservedStateWithoutPrivateIdentities(t *testing.T) { + for _, mode := range []string{"stable", "partial", "changed", "duplicate", "missing ephemeral", "tearing down", "canceled"} { + t.Run(mode, func(t *testing.T) { + f := &blockLayoutFixture{t: t, mode: mode} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if mode == "canceled" { + cancel() + } + report, err := (Native{API: fileAPI{}, Target: "10.0.0.8", Blocks: f}).ListBlockLayout(ctx) + if (err == nil) != (mode == "stable") { + t.Fatal("incomplete or changing block inventory qualified:", err) + } + if mode == "canceled" && f.calls != 0 { + t.Fatal("canceled observer reached native state") + } + if err != nil { + if report.Digest != "" || strings.Contains(err.Error(), "canary") { + t.Fatal("failed observation leaked or retained success") + } + return + } + encoded, _ := json.Marshal(report) + if strings.Contains(string(encoded), "canary") || strings.Contains(string(encoded), "/dev/") || f.calls != 4 || report.EphemeralPhase != "ready" || !report.EphemeralLUKS2 || !report.EphemeralNodeIDOnly || report.EphemeralSizeBytes != 150<<30 || !report.RawPresent || report.RawPhase != "waiting" || report.RawSizeBytes != 0 || report.RawDiscoveredPartitions != 0 || len(report.Digest) != 64 { + t.Fatal("observed block state lost scope, leaked identity or invented allocation:", report) + } + }) + } +} diff --git a/internal/longhorninventory/block_wire.go b/internal/longhorninventory/block_wire.go new file mode 100644 index 0000000000..be5ba7fa9d --- /dev/null +++ b/internal/longhorninventory/block_wire.go @@ -0,0 +1,82 @@ +package longhorninventory + +import ( + "context" + "io" + "net/netip" + "time" + + "github.com/cosi-project/runtime/api/v1alpha1" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/resource/protobuf" + "github.com/cosi-project/runtime/pkg/state" + "github.com/siderolabs/talos/pkg/machinery/resources/block" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" +) + +type BlockCollections interface { + List(context.Context, resource.Kind, ...state.ListOption) (resource.List, error) +} + +type BlockListAPI interface { + List(context.Context, *v1alpha1.ListRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[v1alpha1.ListResponse], error) +} + +type BoundedBlocks struct { + API BlockListAPI + Target string +} + +func (r BoundedBlocks) List(ctx context.Context, kind resource.Kind, options ...state.ListOption) (resource.List, error) { + invalid := nativeFailure{code: "block-list"} + if ctx == nil || !active(ctx) || r.API == nil || kind == nil || len(options) != 0 || kind.Namespace() != block.NamespaceName { + return resource.List{}, invalid + } + deadline, _ := ctx.Deadline() + address, err := netip.ParseAddr(r.Target) + md, _ := metadata.FromOutgoingContext(ctx) + if err != nil || address.String() != r.Target || !netip.MustParsePrefix("10.0.0.0/16").Contains(address) || time.Until(deadline) > 15*time.Second || len(md.Get("nodes")) != 0 || len(md.Get("node")) != 1 || md.Get("node")[0] != r.Target { + return resource.List{}, invalid + } + limit := 128 + switch kind.Type() { + case block.VolumeStatusType: + case block.DiscoveredVolumeType: + limit = 512 + default: + return resource.List{}, invalid + } + stream, err := r.API.List(ctx, &v1alpha1.ListRequest{Namespace: block.NamespaceName, Type: kind.Type(), Options: &v1alpha1.ListOptions{}}, grpc.MaxCallRecvMsgSize(maxMessageBytes)) + if err != nil || stream == nil { + return resource.List{}, invalid + } + result := resource.List{} + total := 0 + seen := map[string]bool{} + for { + response, recvErr := stream.Recv() + if recvErr == io.EOF && active(ctx) { + return result, nil + } + if recvErr != nil || response == nil || !active(ctx) { + return resource.List{}, invalid + } + size := proto.Size(response) + total += size + if size > maxMessageBytes || total > MaxMetadataBytes || len(result.Items) >= limit || response.Resource == nil || len(response.ProtoReflect().GetUnknown()) != 0 { + return resource.List{}, invalid + } + raw, decodeErr := protobuf.Unmarshal(response.Resource) + if decodeErr != nil { + return resource.List{}, invalid + } + item, decodeErr := protobuf.UnmarshalResource(raw) + if decodeErr != nil || item == nil || item.Metadata().Namespace() != block.NamespaceName || item.Metadata().Type() != kind.Type() || item.Metadata().Phase() != resource.PhaseRunning || item.Metadata().ID() == "" || len(item.Metadata().ID()) > 256 || seen[item.Metadata().ID()] { + return resource.List{}, invalid + } + seen[item.Metadata().ID()] = true + result.Items = append(result.Items, item) + } +} diff --git a/internal/longhorninventory/block_wire_test.go b/internal/longhorninventory/block_wire_test.go new file mode 100644 index 0000000000..a4906ade5d --- /dev/null +++ b/internal/longhorninventory/block_wire_test.go @@ -0,0 +1,98 @@ +package longhorninventory + +import ( + "context" + "errors" + "fmt" + "io" + "testing" + "time" + + "github.com/cosi-project/runtime/api/v1alpha1" + "github.com/cosi-project/runtime/pkg/resource/protobuf" + talos "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/resources/block" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +type blockListAPIFixture struct { + v1alpha1.StateClient + stream grpc.ServerStreamingClient[v1alpha1.ListResponse] +} + +func (f blockListAPIFixture) List(context.Context, *v1alpha1.ListRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[v1alpha1.ListResponse], error) { + return f.stream, nil +} + +type blockWireStream struct { + grpc.ClientStream + response *v1alpha1.ListResponse + end error + calls int + repeat bool +} + +func (s *blockWireStream) Recv() (*v1alpha1.ListResponse, error) { + s.calls++ + if s.calls == 1 || s.repeat && s.calls <= 130 { + if s.repeat { + response := proto.Clone(s.response).(*v1alpha1.ListResponse) + response.Resource.Metadata.Id = fmt.Sprintf("entry-%03d", s.calls) + return response, nil + } + return s.response, nil + } + return nil, s.end +} + +func TestBoundedBlockWireRequiresCleanCompletionBeforeReturningAnyResource(t *testing.T) { + for _, scenario := range []string{"complete", "joined EOF", "receive error", "entry bound", "message bound"} { + t.Run(scenario, func(t *testing.T) { + v := block.NewVolumeStatus(block.NamespaceName, "EPHEMERAL") + encoded, err := protobuf.FromResource(v) + if err != nil { + t.Fatal(err) + } + wire, err := encoded.Marshal() + if err != nil { + t.Fatal(err) + } + response := &v1alpha1.ListResponse{Resource: wire} + stream := &blockWireStream{response: response, end: io.EOF} + switch scenario { + case "joined EOF": + stream.end = errors.Join(io.EOF, errors.New("private-provider-canary")) + case "receive error": + stream.end = errors.New("private-provider-canary") + case "entry bound": + stream.repeat = true + case "message bound": + response.ProtoReflect().SetUnknown(make([]byte, maxMessageBytes+1)) + } + ctx, cancel := context.WithTimeout(talos.WithNode(context.Background(), "10.0.0.8"), 15*time.Second) + defer cancel() + got, err := (BoundedBlocks{API: blockListAPIFixture{stream: stream}, Target: "10.0.0.8"}).List(ctx, block.NewVolumeStatus(block.NamespaceName, "").Metadata()) + if (err == nil) != (scenario == "complete") { + t.Fatal("stream completion or bounds were misclassified:", err) + } + if err != nil && len(got.Items) != 0 { + t.Fatal("failed stream returned a resource prefix") + } + if err == nil { + if len(got.Items) != 1 { + t.Fatal("complete native stream lost its resource") + } + if _, ok := got.Items[0].(*block.VolumeStatus); !ok { + t.Fatal("native type was not decoded") + } + } + if stream.calls > 129 { + t.Fatal("entry bound was not enforced while streaming") + } + if scenario == "entry bound" && stream.calls != 129 { + t.Fatal("entry-limit path was not exercised") + } + }) + } +} diff --git a/internal/longhorninventory/native.go b/internal/longhorninventory/native.go index b3361a04c3..c52ca2fa3d 100644 --- a/internal/longhorninventory/native.go +++ b/internal/longhorninventory/native.go @@ -29,6 +29,7 @@ type Files interface { type Native struct { API Files Target string + Blocks BlockCollections } const maxMessageBytes = 64 << 10 diff --git a/internal/longhorninventory/storage_root.go b/internal/longhorninventory/storage_root.go index c5a4d6e05c..39a23be9bc 100644 --- a/internal/longhorninventory/storage_root.go +++ b/internal/longhorninventory/storage_root.go @@ -80,7 +80,7 @@ func FailureCode(err error) string { return "unknown" } switch failure.code { - case "request", "transport", "not-found", "permission", "unimplemented", "stream", "bounds", "metadata", "entry-error", "entry-shape", "structure": + case "request", "transport", "not-found", "permission", "unimplemented", "stream", "bounds", "metadata", "entry-error", "entry-shape", "structure", "block-client", "block-list", "block-shape", "block-change": return failure.code } return "unknown" diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 73e8b70c34..03b7542560 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -1,5 +1,14 @@ # Prepare Longhorn V2 workers +`inventory-storage-layout` adds read-only native block metadata to the storage-root +observation for a fixed temporary worker. Confirm it with +`inventory-longhorn-storage-layout-`. Two complete, bounded native volume +status and physical-discovery collections must agree inside the unchanged node, +hardware and boot bracket. Its schema-2 observation-only receipt reports sizes, +phases, encryption properties and counts, while withholding device paths and +identities. It grants no reset clearance and never treats a failed read as absence. +The original root-only mode keeps its existing scope and schema. + `inspect-host-memory` is a read-only diagnostic for a fixed permanent worker, confirmed with `inspect-longhorn-v2-host-memory-`. It reads only the native huge-page allocation and selected `/proc/meminfo` counters, bracketed by unchanged diff --git a/scripts/migrate-longhorn-v2/inventory.go b/scripts/migrate-longhorn-v2/inventory.go index 113ed863c1..85e9885116 100644 --- a/scripts/migrate-longhorn-v2/inventory.go +++ b/scripts/migrate-longhorn-v2/inventory.go @@ -49,12 +49,13 @@ const ( stageOutput stageHostMemoryCounters stageHostMemoryAllocationChanged + stageBlockLayout ) func (stage inventoryStage) Error() string { names := [...]string{"unknown", "deadline", "projection-before", "client", "host-before", "boot-before", "listing-before", "ownership-before", "listing-after", "projection-after", "api-consistency", "host-after", "boot-after", "ownership-after", - "listing-consistency", "close", "source-before", "source-after", "output", "host-memory-counters", "host-memory-allocation-changed"} + "listing-consistency", "close", "source-before", "source-after", "output", "host-memory-counters", "host-memory-allocation-changed", "block-layout"} if int(stage) >= len(names) { stage = stageUnknown } diff --git a/scripts/migrate-longhorn-v2/inventory_runtime.go b/scripts/migrate-longhorn-v2/inventory_runtime.go index 720e69a940..95dcdfe2a9 100644 --- a/scripts/migrate-longhorn-v2/inventory_runtime.go +++ b/scripts/migrate-longhorn-v2/inventory_runtime.go @@ -17,6 +17,8 @@ import ( "strings" "time" + "github.com/cosi-project/runtime/api/v1alpha1" + "github.com/devantler-tech/platform/internal/longhorninventory" talos "github.com/siderolabs/talos/pkg/machinery/client" talosconfig "github.com/siderolabs/talos/pkg/machinery/client/config" @@ -158,7 +160,8 @@ func newInventoryReader(ctx context.Context, address string) (nativeReplicaReade if err != nil { return nil, nil, errInventory } - return longhorninventory.Native{API: connection.MachineClient, Target: address}, connection.Close, nil + blocks := longhorninventory.BoundedBlocks{API: v1alpha1.NewStateClient(connection.Conn()), Target: address} + return longhorninventory.Native{API: connection.MachineClient, Target: address, Blocks: blocks}, connection.Close, nil } func runReplicaInventory(ctx context.Context, target, source, runID string) error { diff --git a/scripts/migrate-longhorn-v2/inventory_temporary_test.go b/scripts/migrate-longhorn-v2/inventory_temporary_test.go index cec7dfc140..13be22dc02 100644 --- a/scripts/migrate-longhorn-v2/inventory_temporary_test.go +++ b/scripts/migrate-longhorn-v2/inventory_temporary_test.go @@ -251,6 +251,12 @@ func TestTemporaryInventoryNativeWorkflowGuardPreservesResetBoundary(t *testing. operation, node, ref, attempt, confirm string allowed bool }{ + {"inventory-storage-layout", "temporary-v1-1", "refs/heads/main", "1", "inventory-longhorn-storage-layout-temporary-v1-1", true}, + {"inventory-storage-layout", "temporary-v2-1", "refs/heads/main", "1", "inventory-longhorn-storage-layout-temporary-v2-1", true}, + {"inventory-storage-layout", "prod-worker-1", "refs/heads/main", "1", "inventory-longhorn-storage-layout-prod-worker-1", false}, + {"inventory-storage-layout", "temporary-v1-1", "refs/heads/main", "1", "inventory-longhorn-storage-root-temporary-v1-1", false}, + {"inventory-storage-layout", "temporary-v1-1", "refs/heads/main", "2", "inventory-longhorn-storage-layout-temporary-v1-1", false}, + {"inventory-storage-layout", "temporary-v1-1", "refs/heads/feature", "1", "inventory-longhorn-storage-layout-temporary-v1-1", false}, {"inspect-host-memory", "prod-worker-1", "refs/heads/main", "1", "inspect-longhorn-v2-host-memory-prod-worker-1", true}, {"inspect-host-memory", "temporary-v1-1", "refs/heads/main", "1", "inspect-longhorn-v2-host-memory-temporary-v1-1", false}, {"inspect-host-memory", "prod-worker-1", "refs/heads/feature", "1", "inspect-longhorn-v2-host-memory-prod-worker-1", false}, diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index fbd4d442b7..1156fda6d9 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -22,6 +22,8 @@ func main() { authorization = authorizeHostMemory(os.Getenv, target) case "inventory-storage-root": authorization = authorizeStorageRoot(os.Getenv, target) + case "inventory-storage-layout": + authorization = authorizeStorageLayout(os.Getenv, target) default: authorization = authorizeRefresh(os.Getenv) } @@ -31,6 +33,13 @@ func main() { } ctx, cancel := context.WithTimeout(context.Background(), 110*time.Minute) defer cancel() + if operation == "inventory-storage-layout" { + if err := runStorageLayoutInventory(ctx, target, os.Getenv("GITHUB_SHA"), os.Getenv("GITHUB_RUN_ID")); err != nil { + fmt.Fprintln(os.Stderr, inventoryFailureMessage(err)) + os.Exit(1) + } + return + } if operation == "inspect-host-memory" { if err := runHostMemory(ctx, target, os.Getenv("GITHUB_SHA"), os.Getenv("GITHUB_RUN_ID")); err != nil { fmt.Fprintln(os.Stderr, inventoryFailureMessage(err)) diff --git a/scripts/migrate-longhorn-v2/storage_root_inventory.go b/scripts/migrate-longhorn-v2/storage_root_inventory.go index e34b7908c9..2dddd83f78 100644 --- a/scripts/migrate-longhorn-v2/storage_root_inventory.go +++ b/scripts/migrate-longhorn-v2/storage_root_inventory.go @@ -13,15 +13,27 @@ import ( type nativeStorageRootReader interface { ListStorageRoot(context.Context) (longhorninventory.Listing, error) } +type nativeBlockLayoutReader interface { + ListBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) +} type storageRootReceipt struct { - Schema int `json:"schema"` - ObservationOnly bool `json:"observationOnly"` - ResetClearance bool `json:"resetClearance"` - APIProjectionDigest string `json:"apiProjectionDigest"` - Report longhorninventory.StorageRootReport `json:"report"` + Schema int `json:"schema"` + ObservationOnly bool `json:"observationOnly"` + ResetClearance bool `json:"resetClearance"` + APIProjectionDigest string `json:"apiProjectionDigest"` + Report longhorninventory.StorageRootReport `json:"report"` + BlockLayout *longhorninventory.BlockLayoutReport `json:"blockLayout,omitempty"` +} + +func observeStorageLayout(ctx context.Context, c client, target string, factory inventoryReaderFactory) (storageRootReceipt, error) { + return observeStorageRootScope(ctx, c, target, factory, true) +} + +func observeStorageRoot(ctx context.Context, c client, target string, factory inventoryReaderFactory) (storageRootReceipt, error) { + return observeStorageRootScope(ctx, c, target, factory, false) } -func observeStorageRoot(ctx context.Context, c client, target string, factory inventoryReaderFactory) (receipt storageRootReceipt, err error) { +func observeStorageRootScope(ctx context.Context, c client, target string, factory inventoryReaderFactory, includeBlocks bool) (receipt storageRootReceipt, err error) { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() expires, _ := ctx.Deadline() @@ -66,6 +78,21 @@ func observeStorageRoot(ctx context.Context, c client, target string, factory in if err != nil { return storageRootReceipt{}, stageOwnershipBefore } + var blockLayout *longhorninventory.BlockLayoutReport + if includeBlocks { + blockReader, ok := reader.(nativeBlockLayoutReader) + if !ok { + return storageRootReceipt{}, stageClient + } + layout, layoutErr := blockReader.ListBlockLayout(ctx) + if layoutErr != nil { + return storageRootReceipt{}, errors.Join(stageBlockLayout, layoutErr) + } + if len(layout.Digest) != 64 || layout.VolumeStatuses <= 0 || layout.DiscoveredVolumes <= 0 { + return storageRootReceipt{}, stageBlockLayout + } + blockLayout = &layout + } second, err := rootReader.ListStorageRoot(ctx) if err != nil { return storageRootReceipt{}, errors.Join(stageListingAfter, err) @@ -94,7 +121,11 @@ func observeStorageRoot(ctx context.Context, c client, target string, factory in if !active() { return storageRootReceipt{}, stageDeadline } - return storageRootReceipt{Schema: 1, ObservationOnly: true, ResetClearance: false, APIProjectionDigest: before.Digest, Report: report}, nil + schema := 1 + if includeBlocks { + schema = 2 + } + return storageRootReceipt{Schema: schema, ObservationOnly: true, ResetClearance: false, APIProjectionDigest: before.Digest, Report: report, BlockLayout: blockLayout}, nil } func authorizeStorageRoot(getenv func(string) string, target string) error { @@ -105,6 +136,24 @@ func authorizeStorageRoot(getenv func(string) string, target string) error { } func runStorageRootInventory(ctx context.Context, target, source, runID string) error { + return runStorageInventoryScope(ctx, target, source, runID, false) +} + +func authorizeStorageLayout(getenv func(string) string, target string) error { + if !inventoryTemporarySelection.MatchString(target) || getenv("LONGHORN_V2_NODE") != target || getenv("LONGHORN_V2_OPERATION") != "inventory-storage-layout" { + return errInventory + } + return authorizeWorkflow(getenv, "inventory-longhorn-storage-layout-"+target) +} + +func runStorageLayoutInventory(ctx context.Context, target, source, runID string) error { + if !inventoryTemporarySelection.MatchString(target) { + return stageProjectionBefore + } + return runStorageInventoryScope(ctx, target, source, runID, true) +} + +func runStorageInventoryScope(ctx context.Context, target, source, runID string, includeBlocks bool) error { c := client{run: inventoryReadCommand, context: "admin@prod"} if inventoryTemporarySelection.MatchString(target) { config, err := os.ReadFile("ksail.prod.yaml") @@ -119,7 +168,7 @@ func runStorageRootInventory(ctx context.Context, target, source, runID string) if err := sourceCurrent(ctx, c.run, source); err != nil { return stageSourceBefore } - receipt, err := observeStorageRoot(ctx, c, target, newInventoryReader) + receipt, err := observeStorageRootScope(ctx, c, target, newInventoryReader, includeBlocks) if err != nil { return err } diff --git a/scripts/migrate-longhorn-v2/storage_root_inventory_test.go b/scripts/migrate-longhorn-v2/storage_root_inventory_test.go index 2388b85ec4..eaba358391 100644 --- a/scripts/migrate-longhorn-v2/storage_root_inventory_test.go +++ b/scripts/migrate-longhorn-v2/storage_root_inventory_test.go @@ -15,6 +15,16 @@ type storageRootReaderFixture struct { temporaryNativeReader calls, boots int change, partial, reboot bool + blockCalls int + blockError bool +} + +func (r *storageRootReaderFixture) ListBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) { + r.blockCalls++ + if r.blockError { + return longhorninventory.BlockLayoutReport{}, errors.New("private-block-canary") + } + return longhorninventory.BlockLayoutReport{VolumeStatuses: 2, DiscoveredVolumes: 1, EphemeralPhase: "ready", RawPhase: "waiting", Digest: strings.Repeat("a", 64)}, nil } func (r *storageRootReaderFixture) ListStorageRoot(context.Context) (longhorninventory.Listing, error) { @@ -34,7 +44,7 @@ func (r *storageRootReaderFixture) BootID(context.Context) (string, error) { } func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t *testing.T) { - for _, scenario := range []string{"complete", "partial", "root changed", "peer changed", "boot changed", "close failed"} { + for _, scenario := range []string{"complete", "partial", "root changed", "peer changed", "boot changed", "close failed", "layout complete", "layout read failed"} { t.Run(scenario, func(t *testing.T) { nodes := temporaryInventoryRoster() pools, err := configuredMigrationPools(poolConfigFixture()) @@ -72,9 +82,13 @@ func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t } return json.Marshal(object{"apiVersion": api, "kind": kind, "metadata": object{"resourceVersion": "1"}, "items": items}) }} - reader := &storageRootReaderFixture{temporaryNativeReader: temporaryNativeReader{boot: text(nodes[0], "status", "nodeInfo", "bootID")}, change: scenario == "root changed", partial: scenario == "partial", reboot: scenario == "boot changed"} + reader := &storageRootReaderFixture{temporaryNativeReader: temporaryNativeReader{boot: text(nodes[0], "status", "nodeInfo", "bootID")}, change: scenario == "root changed", partial: scenario == "partial", reboot: scenario == "boot changed", blockError: scenario == "layout read failed"} closed := false - receipt, err := observeStorageRoot(context.Background(), c, "temporary-v1-1", func(_ context.Context, address string) (nativeReplicaReader, func() error, error) { + observe := observeStorageRoot + if strings.HasPrefix(scenario, "layout ") { + observe = observeStorageLayout + } + receipt, err := observe(context.Background(), c, "temporary-v1-1", func(_ context.Context, address string) (nativeReplicaReader, func() error, error) { projection, _, projectionErr := c.inventoryProjection(context.Background(), "temporary-v1-1") if projectionErr != nil || address != projection.Address { t.Fatal("parent observer selected another endpoint") @@ -87,7 +101,7 @@ func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t return nil }, nil }) - if !closed || (err == nil) != (scenario == "complete") { + if !closed || (err == nil) != (scenario == "complete" || scenario == "layout complete") { t.Fatal("incomplete or changed parent metadata was accepted:", err) } encoded, _ := json.Marshal(receipt) @@ -100,7 +114,14 @@ func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t } return } - if receipt.Schema != 1 || !receipt.ObservationOnly || receipt.ResetClearance || receipt.Report.ReplicaEntryPresent || !receipt.Report.EngineBinariesPresent || receipt.Report.OtherEntries != 1 { + if scenario == "layout complete" { + if reader.blockCalls != 1 || receipt.Schema != 2 || receipt.BlockLayout == nil || receipt.BlockLayout.RawPhase != "waiting" { + t.Fatal("layout mode omitted native block facts") + } + } else if reader.blockCalls != 0 || receipt.Schema != 1 || receipt.BlockLayout != nil { + t.Fatal("legacy root mode expanded its read scope") + } + if !receipt.ObservationOnly || receipt.ResetClearance || receipt.Report.ReplicaEntryPresent || !receipt.Report.EngineBinariesPresent || receipt.Report.OtherEntries != 1 { t.Fatal("parent metadata lost its scope or claimed reset clearance") } }) @@ -122,3 +143,19 @@ func TestParentInventoryRequiresItsOwnReadOnlyConfirmation(t *testing.T) { t.Fatal("replica confirmation admitted a different read scope") } } + +func TestBlockLayoutObservationCannotUseOtherConfirmationsOrTargetPermanentWorkers(t *testing.T) { + env := dispatchFixture() + env["LONGHORN_V2_OPERATION"], env["LONGHORN_V2_NODE"], env["LONGHORN_V2_CONFIRM"] = "inventory-storage-layout", "temporary-v1-1", "inventory-longhorn-storage-layout-temporary-v1-1" + getenv := func(key string) string { return env[key] } + if authorizeStorageLayout(getenv, "temporary-v1-1") != nil { + t.Fatal("fixed layout observation refused") + } + if authorizeStorageRoot(getenv, "temporary-v1-1") == nil || authorizeInventory(getenv, "temporary-v1-1") == nil || authorizeRefresh(getenv) == nil || authorize(getenv, "temporary-v1-1") == nil { + t.Fatal("layout confirmation granted another operation") + } + env["LONGHORN_V2_NODE"], env["LONGHORN_V2_CONFIRM"] = "prod-worker-1", "inventory-longhorn-storage-layout-prod-worker-1" + if authorizeStorageLayout(getenv, "prod-worker-1") == nil { + t.Fatal("layout observation escaped its temporary pool") + } +} From 376561abf00e285356024d656419f9f240dd1df0 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 02:59:35 +0100 Subject: [PATCH 08/35] fix(storage): check capacity before temporary worker resets Part of #5787. Reuse the existing logical allocation and measured free-space budget before a preparation journal or cordon and in both complete post-drain snapshots. Four failing flow cases reproduced the missing gate before the change; the three affected race packages and lint now pass. This remains private pending native storage qualification and source curation. --- .../migrate-longhorn-v2/temporary_prepare.go | 3 ++ .../temporary_prepare_flow_test.go | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/scripts/migrate-longhorn-v2/temporary_prepare.go b/scripts/migrate-longhorn-v2/temporary_prepare.go index 56912d17d3..e887541437 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare.go @@ -85,6 +85,9 @@ func (p *temporaryPreparation) prepare(ctx context.Context) error { } func (p temporaryPreparation) unusedCapacity(s inventory, pinned bool) error { + if err := verifyRebuildBudget(p.target, s); err != nil { + return err + } nodes := s.nodes for _, n := range nodes { if text(n, "metadata", "name") == p.target && value(n, "spec", "unschedulable") == true { diff --git a/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go index cf6f8e91d3..62245c1ce6 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go @@ -233,6 +233,51 @@ func TestUncertainTemporaryResetIsNeverReplayed(t *testing.T) { } } +func TestUnusedTemporaryCapacityHoldPrecedesJournalAndCordon(t *testing.T) { + for _, kind := range []string{"logical", "actual"} { + t.Run(kind, func(t *testing.T) { + f, p := unusedPreparationAPI(t) + if kind == "logical" { + // Below the individual disk size, but above its allocation budget. + f.base.s.volumes[0]["spec"].(object)["size"] = fmt.Sprint(87 << 30) + } else { + f.base.s.volumes[0]["status"].(object)["actualSize"] = float64(63 << 30) + } + if p.prepare(context.Background()) == nil || f.base.cm != nil || f.base.resets != 0 || value(f.base.node, "spec", "unschedulable") == true { + t.Fatal("insufficient capacity was accepted or preparation mutated the worker before holding") + } + }) + } +} + +func TestUnusedTemporaryCapacityIsRecheckedInBothPostDrainObservations(t *testing.T) { + for _, growthAt := range []int{1, 2} { + t.Run(fmt.Sprint(growthAt), func(t *testing.T) { + f, p := unusedPreparationAPI(t) + run := p.c.run + drained, observations := false, 0 + p.c.run = func(ctx context.Context, input []byte, program string, args ...string) ([]byte, error) { + if program == "kubectl" && len(args) > 3 { + if drained && len(args) == 5 && args[2] == "get" && args[3] == "nodes" && args[4] == "--output=json" { + observations++ + if observations == growthAt { + f.base.s.volumes[0]["status"].(object)["actualSize"] = float64(63 << 30) + } + } + } + out, err := run(ctx, input, program, args...) + if err == nil && program == "kubectl" && len(args) > 2 && args[2] == "drain" { + drained = true + } + return out, err + } + if p.prepare(context.Background()) == nil || !drained || observations != growthAt || f.base.resets != 0 || p.j.Phase == "reset-requested" || p.j.Phase == "prepared" { + t.Fatal("post-drain storage growth reached reset intent or was not observed") + } + }) + } +} + func TestTemporaryPreparationRebindsIdentityAndStorageBeforeResumingWrites(t *testing.T) { for _, scenario := range []string{"storage enabled", "node replaced", "unexpected reboot"} { t.Run(scenario, func(t *testing.T) { From 2ee296234a70e6f6ff0bb2df0cc8b526b4ef52b4 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 04:08:39 +0100 Subject: [PATCH 09/35] fix(storage): qualify unused workers from stable parent inventory Bind parent absence to the original encrypted partition and complete native block metadata. Keep replica, backing-image and unknown entries closed. Part of #5787. --- scripts/migrate-longhorn-v2/README.md | 11 ++- .../migrate-longhorn-v2/temporary_prepare.go | 26 ++++-- .../temporary_prepare_flow_test.go | 79 ++++++++++++++++++- .../migrate-longhorn-v2/temporary_storage.go | 34 +++++++- .../temporary_storage_test.go | 34 ++++++++ 5 files changed, 170 insertions(+), 14 deletions(-) diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index a2d5cdcf95..52a8e3d3e0 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -35,9 +35,12 @@ reviewed two-member V1 pool and one-member V2 pool. The fresh V2 pool has no res This operation wipes **only the selected temporary worker's EPHEMERAL partition** and reboots it. Its separate journal binds the original node UID, provider and hardware identities, boot, -encrypted partition UUID, and native empty-storage receipt. It requires unregistered, fenced -Longhorn storage, complete native replica listings and block-resource inventories, and no -allocated raw partition. Unaccounted replica-root entries, incomplete reads or changed identities hold the run. +encrypted partition UUID, and native storage commitments. It requires unregistered, fenced +Longhorn storage with no placed replicas, complete native parent-directory and block-resource +inventories, and no allocated raw partition. Only the recognized engine, metadata, log and +socket cache directories may remain; a replica entry, backing images, any file, unknown entry, +incomplete read or changed identity holds the run. The original encrypted partition is checked +before each parent listing. A missing replica-path error is never interpreted as empty storage. All three temporary workers receive owned scale-down protection before the selected worker is cordoned. Drain uses eviction and respects disruption budgets. Two fresh storage observations after drain precede the recorded reset intent. @@ -133,7 +136,7 @@ Kubernetes worker before host configuration can be trusted or a host command can ## Prepare a permanent worker `prepare-worker` rebuilds one permanent worker's local disk so it can hold V2 replicas. It wipes -that worker's `EPHEMERAL` partition, so it is the one destructive step in this tool. Nothing on the +that worker's `EPHEMERAL` partition and requires the separate permanent-worker safeguards. Nothing on the cluster enables it yet: it holds with no write until the temporary V1 pool accepts replicas, the worker layout is loaded by Talos, the raw-disk defaults are on the Node, huge pages are ready and Longhorn's V2 engine is on. Those land in later changes (platform#5655). diff --git a/scripts/migrate-longhorn-v2/temporary_prepare.go b/scripts/migrate-longhorn-v2/temporary_prepare.go index e887541437..d822c7d5e9 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "maps" + + "github.com/devantler-tech/platform/internal/longhorninventory" ) type temporaryPreparation struct { @@ -198,7 +200,7 @@ func (p temporaryPreparation) observeUnused(ctx context.Context, owned bool) (st if err != nil { return "", err } - if _, err = verifyNode(n, p.j, true); err != nil || before.BootID != p.j.OriginalBootID || before.StorageUID != p.j.LonghornUID { + if _, err = verifyNode(n, p.j, true); err != nil || before.Registered || len(before.Replicas) != 0 || before.BootID != p.j.OriginalBootID || before.StorageUID != p.j.LonghornUID { return "", errInventory } factory := p.reader @@ -207,20 +209,34 @@ func (p temporaryPreparation) observeUnused(ctx context.Context, owned bool) (st return openInventoryReader(ctx, address, p.j.EphemeralUUID) } } - receipt, err := observeReplicaInventory(ctx, c, p.temporarySelection, factory) + receipt, err := observeStorageLayout(ctx, c, p.temporarySelection, factory) if err != nil { - return "", err + return "", errors.New(inventoryFailureMessage(err)) } - if receipt.APIProjectionDigest != before.Digest || receipt.Registered || !receipt.Report.EmptyAndAccounted { + if receipt.APIProjectionDigest != before.Digest || !unusedParentAbsent(receipt.Report) || receipt.BlockLayout == nil { return "", errInventory } if err = sourceCurrent(ctx, p.c.run, p.source); err != nil { return "", err } - digest := sha256.Sum256([]byte(receipt.APIProjectionDigest + ":" + receipt.Report.Digest)) + digest := sha256.Sum256([]byte(receipt.APIProjectionDigest + ":" + receipt.Report.Digest + ":" + receipt.BlockLayout.Digest)) return hex.EncodeToString(digest[:]), nil } +// Only recognized non-volume cache directories may remain on this fixed, +// unregistered temporary worker. The native wrapper separately binds the +// original encrypted partition and rejects any allocated raw or user volume. +func unusedParentAbsent(report longhorninventory.StorageRootReport) bool { + known := 0 + for _, present := range []bool{report.EngineBinariesPresent, report.MetadataPresent, report.LogsPresent, report.SocketDirectoryPresent} { + if present { + known++ + } + } + return len(report.Digest) == 64 && report.Files == 0 && report.OtherEntries == 0 && + !report.ReplicaEntryPresent && !report.ReplicaDirectoryPresent && !report.BackingImagesPresent && report.Directories == known +} + func (p temporaryPreparation) protectUnusedCapacity(ctx context.Context) error { s, err := p.c.snapshot(ctx) if err != nil { diff --git a/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go index 62245c1ce6..aeb8f34e62 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go @@ -5,10 +5,13 @@ import ( "encoding/json" "errors" "fmt" + "os" "reflect" "strings" "testing" "time" + + "github.com/devantler-tech/platform/internal/longhorninventory" ) type unusedPreparationFixture struct { @@ -87,7 +90,11 @@ func unusedPreparationAPI(t *testing.T) (*unusedPreparationFixture, temporaryPre if address != "10.0.99.7" { t.Fatal("temporary inspection selected another endpoint") } - return temporaryNativeReader{boot: text(base.node, "status", "nodeInfo", "bootID"), unmatched: f.badNativeAfter > 0 && f.nativeCalls >= f.badNativeAfter}, func() error { return nil }, nil + mode := "cache directories only" + if f.badNativeAfter > 0 && f.nativeCalls >= f.badNativeAfter { + mode = "foreign directory" + } + return &unusedParentReaderFixture{temporaryNativeReader: temporaryNativeReader{boot: text(base.node, "status", "nodeInfo", "bootID")}, mode: mode}, func() error { return nil }, nil } return f, p } @@ -190,6 +197,76 @@ func TestUnusedTemporaryPreparationPreservesV1AndLeavesRawStorageFenced(t *testi } } +type unusedParentReaderFixture struct { + temporaryNativeReader + rootCalls int + mode string +} + +func (*unusedParentReaderFixture) ListReplicas(context.Context) (longhorninventory.Listing, error) { + return longhorninventory.Listing{}, errors.New("missing replica path is not an emptiness proof") +} + +func (r *unusedParentReaderFixture) ListStorageRoot(context.Context) (longhorninventory.Listing, error) { + r.rootCalls++ + entries := []longhorninventory.Entry{{Name: ".", Mode: uint32(os.ModeDir), IsDir: true}} + for _, name := range []string{"engine-binaries", "metadata", "logs", "unix-domain-socket"} { + entries = append(entries, longhorninventory.Entry{Name: name, Mode: uint32(os.ModeDir), IsDir: true}) + } + switch r.mode { + case "replicas", "backing-images", "foreign directory": + entries = append(entries, longhorninventory.Entry{Name: r.mode, Mode: uint32(os.ModeDir), IsDir: true}) + case "file": + entries = append(entries, longhorninventory.Entry{Name: "unexpected", Mode: 0o600}) + case "changed": + if r.rootCalls > 1 { + entries = append(entries, longhorninventory.Entry{Name: "changed", Mode: uint32(os.ModeDir), IsDir: true}) + } + case "error": + return longhorninventory.Listing{Complete: true, Entries: entries}, errors.New("private-parent-error") + } + return longhorninventory.Listing{Complete: r.mode != "partial", Entries: entries}, nil +} + +func (*unusedParentReaderFixture) ListBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) { + return longhorninventory.BlockLayoutReport{VolumeStatuses: 2, DiscoveredVolumes: 1, EphemeralPhase: "ready", RawPresent: true, RawPhase: "waiting", Digest: strings.Repeat("a", 64)}, nil +} + +func TestUnusedPreparationQualifiesStableParentAbsenceWithoutAReplicaPath(t *testing.T) { + for _, mode := range []string{"cache directories only", "replicas", "backing-images", "foreign directory", "file", "partial", "changed", "error"} { + t.Run(mode, func(t *testing.T) { + f, p := unusedPreparationAPI(t) + var readers []*unusedParentReaderFixture + p.reader = func(context.Context, string) (nativeReplicaReader, func() error, error) { + f.nativeCalls++ + r := &unusedParentReaderFixture{temporaryNativeReader: temporaryNativeReader{boot: text(f.base.node, "status", "nodeInfo", "bootID")}, mode: mode} + readers = append(readers, r) + return r, func() error { return nil }, nil + } + err := p.prepare(context.Background()) + if mode == "cache directories only" { + if err != nil || f.base.resets != 1 || len(readers) < 3 { + t.Fatalf("positive stable parent absence did not reach the guarded reset: err=%v resets=%d observations=%d", err, f.base.resets, len(readers)) + } + observations := 0 + for _, r := range readers { + if r.rootCalls != 0 && r.rootCalls != 2 { + t.Fatal("reset skipped an independent parent observation") + } + if r.rootCalls == 2 { + observations++ + } + } + if observations < 3 { + t.Fatal("reset lacked initial and two post-drain parent observations") + } + } else if err == nil || f.base.resets != 0 || strings.Contains(err.Error(), "private-parent-error") { + t.Fatal("unqualified parent inventory allowed reset or exposed provider details") + } + }) + } +} + func TestUnusedTemporaryPreparationHoldsBeforeResetOnLostEvidence(t *testing.T) { for _, scenario := range []string{"missing baseline", "native data", "native data after drain", "foreign pin", "unready peer", "source changed", "layout changed"} { t.Run(scenario, func(t *testing.T) { diff --git a/scripts/migrate-longhorn-v2/temporary_storage.go b/scripts/migrate-longhorn-v2/temporary_storage.go index d968f9ed06..f10cb19d92 100644 --- a/scripts/migrate-longhorn-v2/temporary_storage.go +++ b/scripts/migrate-longhorn-v2/temporary_storage.go @@ -22,21 +22,47 @@ type unusedTemporaryReader struct { // boot bracket on EACH file observation. A partial resource list never clears // a reset, even when its returned prefix happens to contain the original disk. func (r unusedTemporaryReader) ListReplicas(ctx context.Context) (longhorninventory.Listing, error) { + if err := r.verifyPartition(ctx); err != nil { + return longhorninventory.Listing{}, err + } + return r.nativeReplicaReader.ListReplicas(ctx) +} + +func (r unusedTemporaryReader) ListStorageRoot(ctx context.Context) (longhorninventory.Listing, error) { + if err := r.verifyPartition(ctx); err != nil { + return longhorninventory.Listing{}, err + } + reader, ok := r.nativeReplicaReader.(nativeStorageRootReader) + if !ok { + return longhorninventory.Listing{}, errInventory + } + return reader.ListStorageRoot(ctx) +} + +func (r unusedTemporaryReader) ListBlockLayout(ctx context.Context) (longhorninventory.BlockLayoutReport, error) { + reader, ok := r.nativeReplicaReader.(nativeBlockLayoutReader) + if !ok { + return longhorninventory.BlockLayoutReport{}, errInventory + } + return reader.ListBlockLayout(ctx) +} + +func (r unusedTemporaryReader) verifyPartition(ctx context.Context) error { address, addressErr := netip.ParseAddr(r.address) if ctx.Err() != nil || r.blocks == nil || r.nativeReplicaReader == nil || r.ephemeralUUID == "" || addressErr != nil || address.String() != r.address || !netip.MustParsePrefix("10.0.0.0/16").Contains(address) { - return longhorninventory.Listing{}, errInventory + return errInventory } ctx, cancel := context.WithTimeout(talos.WithNode(ctx, r.address), 20*time.Second) defer cancel() statuses, err := r.blocks.List(ctx, block.NewVolumeStatus(block.NamespaceName, "").Metadata()) if err != nil || ctx.Err() != nil { - return longhorninventory.Listing{}, errInventory + return errInventory } discovered, err := r.blocks.List(ctx, block.NewDiscoveredVolume(block.NamespaceName, "").Metadata()) if err != nil || ctx.Err() != nil || verifyUnusedTemporaryPartition(statuses, discovered, r.ephemeralUUID) != nil { - return longhorninventory.Listing{}, errInventory + return errInventory } - return r.nativeReplicaReader.ListReplicas(ctx) + return nil } // Check the complete native block-resource inventory, never a CLI error string. diff --git a/scripts/migrate-longhorn-v2/temporary_storage_test.go b/scripts/migrate-longhorn-v2/temporary_storage_test.go index 9526df5d9b..0a09c80ac6 100644 --- a/scripts/migrate-longhorn-v2/temporary_storage_test.go +++ b/scripts/migrate-longhorn-v2/temporary_storage_test.go @@ -126,6 +126,40 @@ func (r *temporaryFileReaderFixture) ListReplicas(context.Context) (longhorninve return longhorninventory.Listing{Complete: true}, nil } +func (r *temporaryFileReaderFixture) ListStorageRoot(context.Context) (longhorninventory.Listing, error) { + r.calls++ + return longhorninventory.Listing{Complete: true}, nil +} + +func TestTemporaryParentReadRequiresTheOriginalPhysicalPartition(t *testing.T) { + for _, scenario := range []string{"complete", "partial status", "partial discovery", "raw allocated", "original replaced"} { + t.Run(scenario, func(t *testing.T) { + s, d, uuid := temporaryPartitionFixture() + blocks := &temporaryBlockReaderFixture{t: t, statuses: s, discovered: d} + files := &temporaryFileReaderFixture{} + switch scenario { + case "partial status": + blocks.failAt = 1 + case "partial discovery": + blocks.failAt = 2 + case "raw allocated": + s.Items[1].(*block.VolumeStatus).TypedSpec().Phase = block.VolumePhaseReady + case "original replaced": + uuid = "another-partition" + } + reader := unusedTemporaryReader{nativeReplicaReader: files, blocks: blocks, address: "10.0.99.4", ephemeralUUID: uuid} + listing, err := reader.ListStorageRoot(context.Background()) + if scenario == "complete" { + if err != nil || !listing.Complete || files.calls != 1 || blocks.calls != 2 { + t.Fatal("complete physical proof did not reach the parent read:", err) + } + } else if err == nil || listing.Complete || files.calls != 0 { + t.Fatal("unproven physical partition reached the parent read") + } + }) + } +} + func TestTemporaryNativeProofStopsBeforeFilesOnPartialOrChangedStorage(t *testing.T) { for _, scenario := range []string{"complete", "partial status", "partial discovery", "raw allocated between observations", "canceled", "missing original binding"} { t.Run(scenario, func(t *testing.T) { From 03f41a4b302df2f8aaa92588d5b6941ec34ce0c1 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 04:50:17 +0100 Subject: [PATCH 10/35] fix(storage): recover incomplete huge-page allocations Add a single-worker protected reboot with durable intent, original storage and database identity checks, peer recovery exclusion, strict eviction and positive allocation readback. Part of #5791. --- .../workflows/prepare-longhorn-v2-node.yaml | 5 +- scripts/migrate-longhorn-v2/README.md | 20 + scripts/migrate-longhorn-v2/host_reboot.go | 423 ++++++++++++++++++ .../migrate-longhorn-v2/host_reboot_test.go | 335 ++++++++++++++ .../inventory_temporary_test.go | 6 + scripts/migrate-longhorn-v2/main.go | 14 + 6 files changed, 802 insertions(+), 1 deletion(-) create mode 100644 scripts/migrate-longhorn-v2/host_reboot.go create mode 100644 scripts/migrate-longhorn-v2/host_reboot_test.go diff --git a/.github/workflows/prepare-longhorn-v2-node.yaml b/.github/workflows/prepare-longhorn-v2-node.yaml index e567fe1077..104cadc687 100644 --- a/.github/workflows/prepare-longhorn-v2-node.yaml +++ b/.github/workflows/prepare-longhorn-v2-node.yaml @@ -17,7 +17,7 @@ on: description: The guarded migration step to run required: true type: choice - options: [prepare-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas, inventory-storage-root, inventory-storage-layout, inspect-host-memory] + options: [prepare-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas, inventory-storage-root, inventory-storage-layout, inspect-host-memory, reboot-host-for-hugepages] node: description: Permanent worker to prepare, or a fixed read-only inventory selection required: false @@ -86,6 +86,9 @@ jobs: inspect-host-memory) [[ "${LONGHORN_V2_NODE}" =~ ^prod-worker-[123]$ && "${LONGHORN_V2_CONFIRM}" == "inspect-longhorn-v2-host-memory-${LONGHORN_V2_NODE}" ]] ;; + reboot-host-for-hugepages) + [[ "${LONGHORN_V2_NODE}" =~ ^prod-worker-[123]$ && "${LONGHORN_V2_CONFIRM}" == "reboot-longhorn-v2-host-${LONGHORN_V2_NODE}" ]] + ;; inventory-storage-root) [[ "${LONGHORN_V2_NODE}" =~ ^(prod-worker-[123]|temporary-v1-[12]|temporary-v2-1)$ && "${LONGHORN_V2_CONFIRM}" == "inventory-longhorn-storage-root-${LONGHORN_V2_NODE}" ]] ;; diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 03b7542560..94563dd408 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -18,6 +18,26 @@ host text or private identities. An allocation shortfall is reported as observed it grants no reset, reboot, restart or V2-activation clearance. Incomplete reads, changing allocation, identity changes and close failures produce no success receipt. +For a confirmed allocation shortfall, `reboot-host-for-hugepages` selects exactly one +permanent worker with `reboot-longhorn-v2-host-`. It requires the loaded +reservation, unused native huge pages, disabled V2, the complete fixed migration roster, all V1 +volumes fully redundant, and all three database clusters healthy. It records the +original node, hardware, boot, encrypted EPHEMERAL partition, storage disks and +database identities before an owned cordon and an eviction-based drain. Two fresh +checks of complete redundancy, databases and original storage precede durable +reboot intent. The single reboot request has no reset or disk-wipe selector. +An unfinished or unreadable peer journal, another worker's cordon, or lost proof +of a peer's recorded recovery prevents starting the next worker. + +Recovery never repeats an uncertain reboot. It requires a changed boot with the +same node, hardware, OS, encrypted partition, registered disks and database system +identities, 1,024 native 2 MiB pages, matching 2 GiB Kubernetes capacity, clear node +pressure and full V1 redundancy. Only its own cordon is released and read back. +Dispatch each worker separately and verify platform recovery before the next one. +An interrupted run keeps its journal and cordon; a new first-attempt main dispatch +may adopt it only after the previous run has ended. This operation enables no V2 +storage and removes no iSCSI support. + The `Prepare Longhorn V2 Migration` workflow runs guarded steps towards platform#5582. It does not migrate a claim, enable V2, disable V1 or remove iSCSI. diff --git a/scripts/migrate-longhorn-v2/host_reboot.go b/scripts/migrate-longhorn-v2/host_reboot.go new file mode 100644 index 0000000000..6646b8a88c --- /dev/null +++ b/scripts/migrate-longhorn-v2/host_reboot.go @@ -0,0 +1,423 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "reflect" + "regexp" + "strings" + "time" +) + +type hostMemoryReboot struct { + c client + target, source, owner string + pools map[string]migrationPool + inspect func(context.Context) (hostMemoryReport, error) +} + +func authorizeHostReboot(getenv func(string) string, target string) error { + if !inventoryTarget.MatchString(target) || getenv("LONGHORN_V2_NODE") != target || getenv("LONGHORN_V2_OPERATION") != "reboot-host-for-hugepages" { + return errHostMemory + } + return authorizeWorkflow(getenv, "reboot-longhorn-v2-host-"+target) +} + +type hostRebootRecord struct { + Version int `json:"version"` + Target string `json:"target"` + Source string `json:"source"` + OwnerRun string `json:"ownerRun"` + Phase string `json:"phase"` + NodeUID string `json:"nodeUID"` + ProviderID string `json:"providerID"` + MachineUUID string `json:"machineUUID"` + LonghornUID string `json:"longhornUID"` + EphemeralUUID string `json:"ephemeralUUID"` + OriginalDiskUUID string `json:"originalDiskUUID"` + OriginalBootID string `json:"originalBootID"` + RecoveredBootID string `json:"recoveredBootID"` + OSImage string `json:"osImage"` + Architecture string `json:"architecture"` + OriginalDisks object `json:"originalDisks"` + Databases map[string]hostDatabaseIdentity `json:"databases"` +} + +type hostDatabaseIdentity struct{ UID, SystemID string } + +var hostSystemID = regexp.MustCompile(`^[1-9][0-9]{0,19}$`) + +func (r hostRebootRecord) identity() journal { + return journal{Schema: 1, Target: r.Target, Source: r.Source, OwnerRun: r.OwnerRun, Phase: "draining", NodeUID: r.NodeUID, ProviderID: r.ProviderID, + MachineUUID: r.MachineUUID, LonghornUID: r.LonghornUID, EphemeralUUID: r.EphemeralUUID, OriginalDisks: r.OriginalDisks} +} + +func (r hostRebootRecord) validate(target string) error { + if r.Version != 1 || validateJournal(r.identity(), target) != nil || r.OriginalDiskUUID == "" || !inventoryBoot.MatchString(r.OriginalBootID) || r.OSImage == "" || r.Architecture != "amd64" || len(r.Databases) != 3 { + return errHostMemory + } + for _, key := range []string{"observability/coroot-db", "umami/umami-db", "wedding-app/wedding-db"} { + if r.Databases[key].UID == "" || !hostSystemID.MatchString(r.Databases[key].SystemID) { + return errHostMemory + } + } + switch r.Phase { + case "draining", "reboot-requested": + if r.RecoveredBootID != "" { + return errHostMemory + } + case "recovered": + if !inventoryBoot.MatchString(r.RecoveredBootID) || r.RecoveredBootID == r.OriginalBootID { + return errHostMemory + } + default: + return errHostMemory + } + return nil +} + +func hostRegisteredDiskUUID(storage object) string { + disks, ok := value(storage, "spec", "disks").(object) + statuses, statusOK := value(storage, "status", "diskStatus").(object) + if !ok || !statusOK || len(disks) != 1 || len(statuses) != 1 { + return "" + } + for name, raw := range disks { + disk, diskOK := raw.(object) + status, statusOK := statuses[name].(object) + if !diskOK || !statusOK || text(disk, "path") != "/var/lib/longhorn" || text(disk, "diskType") != "filesystem" || + text(status, "diskPath") != "/var/lib/longhorn" || text(status, "diskType") != "filesystem" { + return "" + } + return text(status, "diskUUID") + } + return "" +} + +func (h hostMemoryReboot) name() string { return "longhorn-v2-host-memory-" + h.target } + +func (h hostMemoryReboot) peersClear(ctx context.Context, nodes []object) error { + byName, err := indexed(nodes) + if err != nil { + return errHostMemory + } + for _, target := range []string{"prod-worker-1", "prod-worker-2", "prod-worker-3"} { + if target == h.target { + continue + } + n := byName[target] + if n == nil || value(n, "spec", "unschedulable") == true || text(n, "metadata", "annotations", fenceAnnotation) != "" { + return errors.New("another worker has an outstanding maintenance cordon") + } + peer := h + peer.target = target + cm, r, err := peer.read(ctx) + if err != nil || cm != nil && r.Phase != "recovered" { + return errors.New("another worker has unfinished or unreadable host memory maintenance") + } + if cm != nil { + if _, err = verifyNode(n, r.identity(), true); err != nil || text(n, "status", "nodeInfo", "bootID") != r.RecoveredBootID || !hugepagesReady(n) { + return errors.New("another worker's recorded recovery is no longer proven") + } + } + } + return nil +} + +func (h hostMemoryReboot) read(ctx context.Context) (object, hostRebootRecord, error) { + cm, err := h.c.get(ctx, "configmap", "longhorn-system", h.name(), true) + if err != nil || cm == nil { + return cm, hostRebootRecord{}, err + } + if text(cm, "metadata", "name") != h.name() || text(cm, "metadata", "namespace") != "longhorn-system" { + return nil, hostRebootRecord{}, errHostMemory + } + encoded := text(cm, "data", "record") + if len(encoded) == 0 || len(encoded) > 64<<10 { + return nil, hostRebootRecord{}, errHostMemory + } + d := json.NewDecoder(bytes.NewBufferString(encoded)) + d.DisallowUnknownFields() + var r hostRebootRecord + var extra any + if d.Decode(&r) != nil || d.Decode(&extra) != io.EOF || r.validate(h.target) != nil { + return nil, hostRebootRecord{}, errHostMemory + } + return cm, r, nil +} + +func (h hostMemoryReboot) write(ctx context.Context, cm object, r hostRebootRecord) error { + if cm != nil && (text(cm, "metadata", "name") != h.name() || text(cm, "metadata", "namespace") != "longhorn-system") { + return errHostMemory + } + if r.validate(h.target) != nil || r.Source != h.source || r.OwnerRun != h.owner { + return errHostMemory + } + if err := sourceCurrent(ctx, h.c.run, h.source); err != nil { + return err + } + b, err := json.Marshal(r) + if err != nil { + return errHostMemory + } + if cm != nil { + return h.c.patch(ctx, "configmap", "longhorn-system", cm, object{"op": "replace", "path": "/data/record", "value": string(b)}) + } + body, err := json.Marshal(object{"apiVersion": "v1", "kind": "ConfigMap", "metadata": object{"name": h.name(), "namespace": "longhorn-system"}, "data": object{"record": string(b)}}) + if err != nil { + return errHostMemory + } + _, err = h.c.kubectl(ctx, body, "create", "--filename=/dev/stdin", "--output=name") + return err +} + +func (h hostMemoryReboot) owned(ctx context.Context, expected hostRebootRecord) (object, error) { + cm, actual, err := h.read(ctx) + if err != nil || cm == nil || !reflect.DeepEqual(expected, actual) || actual.OwnerRun != h.owner || actual.Source != h.source { + return nil, errHostMemory + } + return cm, sourceCurrent(ctx, h.c.run, h.source) +} + +func (h hostMemoryReboot) databases(ctx context.Context, expected map[string]hostDatabaseIdentity) (map[string]hostDatabaseIdentity, error) { + items, err := h.c.list(ctx, "clusters.postgresql.cnpg.io", "*") + if err != nil || len(items) != 3 { + return nil, errHostMemory + } + result := map[string]hostDatabaseIdentity{} + for _, db := range items { + key := text(db, "metadata", "namespace") + "/" + text(db, "metadata", "name") + if key != "observability/coroot-db" && key != "umami/umami-db" && key != "wedding-app/wedding-db" || result[key].UID != "" || + text(db, "metadata", "uid") == "" || text(db, "metadata", "resourceVersion") == "" || value(db, "metadata", "deletionTimestamp") != nil || + value(db, "spec", "instances") != float64(3) || value(db, "status", "readyInstances") != float64(3) || + text(db, "status", "phase") != "Cluster in healthy state" || !hostSystemID.MatchString(text(db, "status", "systemID")) { + return nil, errHostMemory + } + result[key] = hostDatabaseIdentity{UID: text(db, "metadata", "uid"), SystemID: text(db, "status", "systemID")} + } + if expected != nil && !reflect.DeepEqual(result, expected) { + return nil, errHostMemory + } + return result, nil +} + +func (h hostMemoryReboot) healthy(ctx context.Context, expected map[string]hostDatabaseIdentity) (inventory, map[string]hostDatabaseIdentity, error) { + setting, err := h.c.get(ctx, "settings.longhorn.io", "longhorn-system", "v2-data-engine", false) + if err != nil || text(setting, "metadata", "name") != "v2-data-engine" || text(setting, "metadata", "namespace") != "longhorn-system" || text(setting, "value") != "false" { + return inventory{}, nil, errors.New("host memory maintenance requires V2 to remain disabled") + } + s, err := h.c.snapshot(ctx) + if err != nil { + return inventory{}, nil, err + } + if verifyCapacityRoster(s.nodes, h.pools, false) != nil || verifyRedundancy(s) != nil { + return inventory{}, nil, errHostMemory + } + for _, v := range s.volumes { + if text(v, "spec", "dataEngine") != "v1" { + return inventory{}, nil, errHostMemory + } + } + databases, err := h.databases(ctx, expected) + return s, databases, err +} + +func (h hostMemoryReboot) targetState(ctx context.Context, r hostRebootRecord) (object, string, error) { + n, err := h.c.get(ctx, "node", "", h.target, false) + if err != nil { + return nil, "", err + } + address, err := verifyNode(n, r.identity(), true) + status, _ := value(n, "status").(map[string]any) + fence := text(n, "metadata", "annotations", fenceAnnotation) + if err != nil || !ready(status) || !workerPressureClear(n) || fence != "" && fence != h.name() || value(n, "spec", "unschedulable") == true && fence != h.name() || + text(n, "status", "nodeInfo", "osImage") != r.OSImage || text(n, "status", "nodeInfo", "architecture") != r.Architecture || h.c.verifyHost(ctx, address, n) != nil { + return nil, "", errHostMemory + } + storage, err := h.c.get(ctx, "nodes.longhorn.io", "longhorn-system", h.target, false) + if err != nil || text(storage, "metadata", "name") != h.target || text(storage, "metadata", "namespace") != "longhorn-system" || text(storage, "metadata", "uid") != r.LonghornUID || hostRegisteredDiskUUID(storage) != r.OriginalDiskUUID || !reflect.DeepEqual(value(storage, "spec", "disks"), r.OriginalDisks) || + value(storage, "spec", "allowScheduling") != true || value(storage, "spec", "evictionRequested") != false { + return nil, "", errHostMemory + } + volume, err := h.c.talosVolume(ctx, address, "EPHEMERAL") + if err != nil || text(volume, "spec", "uuid") != r.EphemeralUUID || text(volume, "spec", "phase") != "ready" || text(volume, "spec", "encryptionProvider") != "luks2" { + return nil, "", errHostMemory + } + setting, err := h.c.talosResource(ctx, address, "kernelparamspec", "sysctl.vm.nr_hugepages") + if err != nil || text(setting, "metadata", "namespace") != "runtime" || text(setting, "spec", "value") != "1024" { + return nil, "", errHostMemory + } + return n, address, nil +} + +func (h hostMemoryReboot) memory(ctx context.Context) (hostMemoryReport, error) { + if h.inspect != nil { + return h.inspect(ctx) + } + receipt, err := observeHostMemory(ctx, h.c, h.target, newInventoryReader) + return receipt.Report, err +} + +func (h hostMemoryReboot) initialize(ctx context.Context) (hostRebootRecord, error) { + s, databases, err := h.healthy(ctx, nil) + if err != nil { + return hostRebootRecord{}, err + } + nodes, err := indexed(s.nodes) + if err != nil { + return hostRebootRecord{}, err + } + storage, err := indexed(s.storageNodes) + if err != nil { + return hostRebootRecord{}, err + } + n, lh := nodes[h.target], storage[h.target] + r := hostRebootRecord{Version: 1, Target: h.target, Source: h.source, OwnerRun: h.owner, Phase: "draining", NodeUID: text(n, "metadata", "uid"), ProviderID: text(n, "spec", "providerID"), + MachineUUID: text(n, "status", "nodeInfo", "systemUUID"), OriginalBootID: text(n, "status", "nodeInfo", "bootID"), LonghornUID: text(lh, "metadata", "uid"), + OSImage: text(n, "status", "nodeInfo", "osImage"), Architecture: text(n, "status", "nodeInfo", "architecture"), Databases: databases, OriginalDiskUUID: hostRegisteredDiskUUID(lh)} + r.OriginalDisks, _ = value(lh, "spec", "disks").(object) + address, err := verifyNode(n, r.identity(), true) + if err != nil || value(n, "spec", "unschedulable") == true || text(n, "metadata", "annotations", fenceAnnotation) != "" { + return hostRebootRecord{}, errHostMemory + } + eph, err := h.c.talosVolume(ctx, address, "EPHEMERAL") + if err != nil { + return hostRebootRecord{}, err + } + r.EphemeralUUID = text(eph, "spec", "uuid") + if r.validate(h.target) != nil { + return hostRebootRecord{}, errHostMemory + } + if _, _, err = h.targetState(ctx, r); err != nil { + return hostRebootRecord{}, err + } + memory, err := h.memory(ctx) + if err != nil || memory.PersistentPages >= 1024 || memory.TotalPages != memory.PersistentPages || memory.FreePages != memory.TotalPages || memory.PageSizeKiB != 2048 || memory.ReservedPages != 0 || memory.SurplusPages != 0 { + return hostRebootRecord{}, errors.New("native allocation no longer proves an unused huge-page shortfall; inspect host memory before a reboot") + } + return r, h.write(ctx, nil, r) +} + +func (h hostMemoryReboot) run(ctx context.Context) error { + if !inventoryTarget.MatchString(h.target) { + return errHostMemory + } + if err := sourceCurrent(ctx, h.c.run, h.source); err != nil { + return err + } + cm, r, err := h.read(ctx) + if err != nil { + return err + } + if cm == nil { + r, err = h.initialize(ctx) + if err != nil { + return err + } + } else if r.OwnerRun != h.owner || r.Source != h.source { + if previousRunCompleted(ctx, h.c, r.OwnerRun, r.Source) != nil { + return errHostMemory + } + r.OwnerRun, r.Source = h.owner, h.source + if err = h.write(ctx, cm, r); err != nil { + return err + } + } + if r.Phase == "draining" { + s, _, err := h.healthy(ctx, r.Databases) + if err != nil { + return err + } + if err = h.peersClear(ctx, s.nodes); err != nil { + return err + } + n, address, err := h.targetState(ctx, r) + if err != nil || text(n, "status", "nodeInfo", "bootID") != r.OriginalBootID { + return errHostMemory + } + if _, err = h.owned(ctx, r); err != nil { + return err + } + changes := append(annotationPatch(n, h.name()), object{"op": "add", "path": "/spec/unschedulable", "value": true}) + if err = h.c.patch(ctx, "node", "", n, changes...); err != nil { + return err + } + if _, err = h.owned(ctx, r); err != nil { + return err + } + if _, err = h.c.kubectl(ctx, nil, "drain", h.target, "--ignore-daemonsets", "--delete-emptydir-data", "--timeout=40m"); err != nil { + return errors.New("host memory maintenance drain was blocked; no reboot was requested") + } + // Keep the existing full redundancy floor: no request while a copy or + // database is still recovering from the eviction. + for range 2 { + s, _, err := h.healthy(ctx, r.Databases) + if err != nil { + return err + } + if err = h.peersClear(ctx, s.nodes); err != nil { + return err + } + n, address, err = h.targetState(ctx, r) + if err != nil || text(n, "status", "nodeInfo", "bootID") != r.OriginalBootID || value(n, "spec", "unschedulable") != true || text(n, "metadata", "annotations", fenceAnnotation) != h.name() { + return errHostMemory + } + } + cm, err = h.owned(ctx, r) + if err != nil { + return err + } + r.Phase = "reboot-requested" + if err = h.write(ctx, cm, r); err != nil { + return err + } + // The durable intent precedes the only request. An uncertain response + // is never retried: recovery must prove a new boot and intact storage. + _, _ = h.c.talosCommand(ctx, 15*time.Minute, address, "reboot", "--wait", "--timeout=15m") + } + recovery, cancel := context.WithTimeout(ctx, 20*time.Minute) + defer cancel() + var recovered object + if err = waitReady(recovery, func(ctx context.Context) error { + n, _, err := h.targetState(ctx, r) + boot := text(n, "status", "nodeInfo", "bootID") + if err != nil || !inventoryBoot.MatchString(boot) || boot == r.OriginalBootID || r.RecoveredBootID != "" && boot != r.RecoveredBootID || !hugepagesReady(n) { + return errHostMemory + } + memory, err := h.memory(ctx) + if err != nil || memory.PersistentPages != 1024 || memory.TotalPages != 1024 || memory.PageSizeKiB != 2048 { + return errHostMemory + } + if _, _, err = h.healthy(ctx, r.Databases); err != nil { + return err + } + recovered = n + return nil + }); err != nil { + return err + } + if _, err = h.owned(ctx, r); err != nil { + return err + } + if text(recovered, "metadata", "annotations", fenceAnnotation) == h.name() { + if err = h.c.patch(ctx, "node", "", recovered, object{"op": "add", "path": "/spec/unschedulable", "value": false}, object{"op": "remove", "path": "/metadata/annotations/" + strings.ReplaceAll(fenceAnnotation, "/", "~1")}); err != nil { + return err + } + } + released, _, err := h.targetState(ctx, r) + if err != nil || value(released, "spec", "unschedulable") == true || text(released, "metadata", "annotations", fenceAnnotation) != "" || text(released, "status", "nodeInfo", "bootID") != text(recovered, "status", "nodeInfo", "bootID") { + return errHostMemory + } + if r.Phase == "recovered" { + return nil + } + cm, err = h.owned(ctx, r) + if err != nil { + return err + } + r.Phase, r.RecoveredBootID = "recovered", text(recovered, "status", "nodeInfo", "bootID") + return h.write(ctx, cm, r) +} diff --git a/scripts/migrate-longhorn-v2/host_reboot_test.go b/scripts/migrate-longhorn-v2/host_reboot_test.go new file mode 100644 index 0000000000..f252895237 --- /dev/null +++ b/scripts/migrate-longhorn-v2/host_reboot_test.go @@ -0,0 +1,335 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "testing" + "time" +) + +type hostRebootFixture struct { + badSettingScope bool + peerJournal object + base *preparationAPI + databases []object + reboots, drains int + rebooted, lost, noRecovery bool + desired string + drainError bool + afterDrain, afterReboot func() + previousLive bool +} + +func hostRebootAPI(t *testing.T) (*hostRebootFixture, hostMemoryReboot) { + t.Helper() + b, old := newPreparationAPI(t, "draining") + b.s = benchmarkInventoryFixture() + b.s.nodes = append(b.s.nodes, capacityFixture()[3:]...) + for i, n := range b.s.nodes { + n["metadata"].(object)["annotations"] = object{} + n["metadata"].(object)["uid"] = fmt.Sprintf("node-%d", i) + if value(n, "spec") == nil { + n["spec"] = object{} + } + n["spec"].(object)["unschedulable"] = false + n["spec"].(object)["providerID"] = fmt.Sprintf("hcloud://%d", 500+i) + n["status"].(object)["nodeInfo"] = object{"systemUUID": fmt.Sprintf("hardware-%d", i), "bootID": fmt.Sprintf("12345678-1234-4234-8234-%012x", i+1), "osImage": "Talos fixture", "architecture": "amd64"} + n["status"].(object)["addresses"] = []any{object{"type": "InternalIP", "address": fmt.Sprintf("10.0.99.%d", i+4)}} + } + b.node, b.storage, b.cm, b.v2Off = b.s.nodes[0], b.s.storageNodes[0], nil, true + for _, storage := range b.s.storageNodes { + storage["metadata"].(object)["namespace"] = "longhorn-system" + storage["spec"].(object)["disks"] = object{"data": object{"path": "/var/lib/longhorn", "diskType": "filesystem", "allowScheduling": true, "evictionRequested": false}} + status := value(storage, "status", "diskStatus", "data").(object) + status["diskPath"], status["diskType"] = "/var/lib/longhorn", "filesystem" + } + for _, replica := range b.s.replicas { + replica["spec"].(object)["diskPath"] = "/var/lib/longhorn" + } + b.node["status"].(object)["capacity"] = object{"hugepages-2Mi": "0"} + b.node["status"].(object)["allocatable"] = object{"hugepages-2Mi": "0", "memory": "14980Mi"} + f := &hostRebootFixture{base: b, desired: "1024"} + for i, selected := range []struct{ namespace, name string }{{"observability", "coroot-db"}, {"umami", "umami-db"}, {"wedding-app", "wedding-db"}} { + db := resource(selected.name) + db["metadata"].(object)["namespace"] = selected.namespace + db["spec"] = object{"instances": float64(3)} + db["status"] = object{"readyInstances": float64(3), "systemID": fmt.Sprintf("1234567%d", i), "phase": "Cluster in healthy state"} + f.databases = append(f.databases, db) + } + h := hostMemoryReboot{c: client{context: "admin@prod", run: f.run}, target: "prod-worker-1", source: old.source, owner: old.owner, pools: old.pools} + if err := verifyCapacityRoster(b.s.nodes, h.pools, false); err != nil { + t.Fatal("invalid initial capacity fixture:", err) + } + if err := verifyRedundancy(b.s); err != nil { + t.Fatal("invalid initial redundancy fixture:", err) + } + h.inspect = func(context.Context) (hostMemoryReport, error) { + pages := uint64(1) + if f.rebooted { + pages = 1024 + } + return hostMemoryReport{PersistentPages: pages, TotalPages: pages, FreePages: pages, PageSizeKiB: 2048, TotalKiB: 16 << 20, AvailableKiB: 7 << 20}, nil + } + return f, h +} + +func (f *hostRebootFixture) run(ctx context.Context, input []byte, program string, args ...string) ([]byte, error) { + f.base.t.Helper() + if program == "talosctl" && len(args) >= 2 && args[1] == "reboot" { + want := []string{"--nodes=10.0.99.4", "reboot", "--wait", "--timeout=15m"} + var record object + if !reflect.DeepEqual(args, want) || json.Unmarshal([]byte(text(f.base.cm, "data", "record")), &record) != nil || text(record, "phase") != "reboot-requested" || f.drains != 1 || value(f.base.node, "spec", "unschedulable") != true { + f.base.t.Fatal("reboot escaped its drained, journaled, exact-node scope") + } + f.reboots++ + if !f.noRecovery { + f.rebooted = true + f.base.node["status"].(object)["nodeInfo"].(object)["bootID"] = "12345678-1234-4234-8234-000000000099" + f.base.node["status"].(object)["capacity"] = object{"hugepages-2Mi": "2Gi"} + f.base.node["status"].(object)["allocatable"] = object{"hugepages-2Mi": "2Gi", "memory": "12932Mi"} + } + if f.afterReboot != nil { + f.afterReboot() + } + if f.lost { + return nil, errors.New("response lost after request") + } + return nil, nil + } + if program == "talosctl" && len(args) == 5 && args[1] == "get" && args[2] == "kernelparamspec" { + if args[3] != "sysctl.vm.nr_hugepages" { + f.base.t.Fatal("host configuration read escaped the single public parameter") + } + return json.Marshal(object{"metadata": object{"namespace": "runtime", "id": args[3]}, "spec": object{"value": f.desired}}) + } + if program == "gh" { + var record object + if json.Unmarshal([]byte(text(f.base.cm, "data", "record")), &record) != nil { + f.base.t.Fatal("previous run was requested without a readable journal") + } + status := "completed" + if f.previousLive { + status = "in_progress" + } + return json.Marshal(object{"status": status, "head_sha": text(record, "source"), "head_branch": "main", "event": "workflow_dispatch", "run_attempt": 1, "path": ".github/workflows/prepare-longhorn-v2-node.yaml"}) + } + if program == "kubectl" && len(args) > 3 { + if len(args) > 4 && args[2] == "get" && args[3] == "settings.longhorn.io" && args[4] == "v2-data-engine" { + data, err := f.base.run(ctx, input, program, args...) + if err != nil { + return nil, err + } + var setting object + if json.Unmarshal(data, &setting) != nil { + f.base.t.Fatal("invalid setting fixture") + } + setting["metadata"].(object)["namespace"] = "longhorn-system" + if f.badSettingScope { + setting["metadata"].(object)["namespace"] = "unrelated" + } + return json.Marshal(setting) + } + if len(args) > 4 && args[2] == "get" && args[3] == "configmap" && args[4] != "longhorn-v2-host-memory-prod-worker-1" { + if args[4] == "longhorn-v2-host-memory-prod-worker-2" && f.peerJournal != nil { + return json.Marshal(f.peerJournal) + } + return nil, nil + } + if args[2] == "get" && args[3] == "clusters.postgresql.cnpg.io" { + return json.Marshal(object{"items": f.databases}) + } + if args[2] == "drain" { + f.drains++ + if f.afterDrain != nil { + f.afterDrain() + } + if f.drainError { + return nil, errors.New("eviction blocked by disruption budget") + } + } + } + return f.base.run(ctx, input, program, args...) +} + +func TestHostRebootRequiresPositiveRecoveryAndPreservesEveryDisk(t *testing.T) { + for _, lost := range []bool{false, true} { + t.Run(fmt.Sprint("response lost ", lost), func(t *testing.T) { + f, h := hostRebootAPI(t) + f.lost = lost + before, _ := json.Marshal(f.base.s.storageNodes) + if err := h.run(context.Background()); err != nil { + t.Fatal("guarded reboot did not recover:", err) + } + after, _ := json.Marshal(f.base.s.storageNodes) + if f.reboots != 1 || f.base.resets != 0 || !reflect.DeepEqual(before, after) || value(f.base.node, "spec", "unschedulable") == true || text(f.base.node, "metadata", "annotations", fenceAnnotation) != "" { + t.Fatal("reboot changed disks, replayed a reset or failed to release only its cordon") + } + if err := h.run(context.Background()); err != nil || f.reboots != 1 { + t.Fatal("completed recovery replayed the reboot:", err) + } + }) + } +} + +func TestHostRebootHoldsBeforeRequestOnLostSafetyEvidence(t *testing.T) { + for _, mode := range []string{"source", "desired pages", "foreign cordon", "database", "replica", "drain", "identity after drain", "source after drain", "V2 flag", "setting scope", "storage scope after drain"} { + t.Run(mode, func(t *testing.T) { + f, h := hostRebootAPI(t) + switch mode { + case "source": + f.base.staleSource = true + case "desired pages": + f.desired = "1" + case "foreign cordon": + f.base.node["spec"].(object)["unschedulable"] = true + f.base.node["metadata"].(object)["annotations"].(object)[fenceAnnotation] = "foreign" + case "database": + f.databases[0]["status"].(object)["readyInstances"] = float64(2) + case "replica": + f.base.s.replicas[0]["spec"].(object)["failedAt"] = time.Now().UTC().Format(time.RFC3339) + case "drain": + f.drainError = true + case "identity after drain": + f.afterDrain = func() { f.base.node["metadata"].(object)["uid"] = "replacement" } + case "source after drain": + f.afterDrain = func() { f.base.staleSource = true } + case "V2 flag": + f.base.v2Off = false + case "setting scope": + f.badSettingScope = true + case "storage scope after drain": + f.afterDrain = func() { f.base.storage["metadata"].(object)["namespace"] = "unrelated" } + } + if h.run(context.Background()) == nil || f.reboots != 0 || f.base.resets != 0 { + t.Fatal("lost safety evidence authorized a host request") + } + }) + } +} + +func TestHostRebootRejectsReplacedRegisteredDiskEvenWhenReplicasAreHealthy(t *testing.T) { + f, h := hostRebootAPI(t) + f.afterReboot = func() { + value(f.base.storage, "status", "diskStatus", "data").(object)["diskUUID"] = "replacement-disk" + for _, replica := range f.base.s.replicas { + if text(replica, "spec", "nodeID") == h.target { + replica["spec"].(object)["diskID"] = "replacement-disk" + } + } + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if h.run(ctx) == nil || f.reboots != 1 || value(f.base.node, "spec", "unschedulable") != true { + t.Fatal("replacement registered storage qualified as the original recovered disk") + } +} + +func TestUncertainHostRebootCannotBeRepeated(t *testing.T) { + f, h := hostRebootAPI(t) + f.lost, f.noRecovery = true, true + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if h.run(ctx) == nil || f.reboots != 1 { + t.Fatal("uncertain reboot was accepted or lost its durable intent") + } + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + defer cancel() + if h.run(ctx) == nil || f.reboots != 1 { + t.Fatal("recovery repeated an uncertain reboot") + } +} + +func TestHostRebootCannotStartWhileAnotherWorkerHasAnUnfinishedJournal(t *testing.T) { + f, h := hostRebootAPI(t) + if _, err := h.initialize(context.Background()); err != nil { + t.Fatal(err) + } + _, r, err := h.read(context.Background()) + if err != nil { + t.Fatal(err) + } + peer, storage := f.base.s.nodes[1], f.base.s.storageNodes[1] + r.Target, r.Phase = "prod-worker-2", "reboot-requested" + r.NodeUID, r.ProviderID, r.MachineUUID = text(peer, "metadata", "uid"), text(peer, "spec", "providerID"), text(peer, "status", "nodeInfo", "systemUUID") + r.OriginalBootID, r.LonghornUID, r.OriginalDiskUUID = text(peer, "status", "nodeInfo", "bootID"), text(storage, "metadata", "uid"), hostRegisteredDiskUUID(storage) + if r.validate("prod-worker-2") != nil { + t.Fatal("peer fixture did not represent a valid pending request") + } + encoded, _ := json.Marshal(r) + f.peerJournal = resource("longhorn-v2-host-memory-prod-worker-2") + f.peerJournal["metadata"].(object)["namespace"] = "longhorn-system" + f.peerJournal["data"] = object{"record": string(encoded)} + if h.run(context.Background()) == nil || f.reboots != 0 || f.drains != 0 || value(f.base.node, "spec", "unschedulable") == true { + t.Fatal("another worker was disrupted before the earlier reboot was proven complete") + } +} + +func TestHostRebootRejectsAJournalOutsideItsFixedNameOrNamespace(t *testing.T) { + for _, field := range []string{"name", "namespace"} { + t.Run(field, func(t *testing.T) { + f, h := hostRebootAPI(t) + if _, err := h.initialize(context.Background()); err != nil { + t.Fatal(err) + } + f.base.cm["metadata"].(object)[field] = "unrelated" + if h.run(context.Background()) == nil || f.reboots != 0 || f.drains != 0 || value(f.base.node, "spec", "unschedulable") == true { + t.Fatal("an API response outside the fixed journal scope authorized maintenance") + } + }) + } +} + +func TestHostRebootRecoveryAdoptsOnlyATerminalOwnerWithoutAnotherRequest(t *testing.T) { + f, h := hostRebootAPI(t) + f.noRecovery = true + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if h.run(ctx) == nil || f.reboots != 1 { + t.Fatal("initial uncertain request was not preserved") + } + h.owner, f.previousLive = "5678", true + if h.run(context.Background()) == nil || f.reboots != 1 { + t.Fatal("recovery stole a live run's journal") + } + f.previousLive, f.rebooted = false, true + f.base.node["status"].(object)["nodeInfo"].(object)["bootID"] = "12345678-1234-4234-8234-000000000099" + f.base.node["status"].(object)["capacity"] = object{"hugepages-2Mi": "2Gi"} + f.base.node["status"].(object)["allocatable"] = object{"hugepages-2Mi": "2Gi", "memory": "12932Mi"} + if err := h.run(context.Background()); err != nil || f.reboots != 1 || f.drains != 1 || value(f.base.node, "spec", "unschedulable") == true { + t.Fatal("terminal-owner recovery replayed the request or failed positive recovery:", err) + } +} + +func TestHostRebootAuthorizationIsDistinctAndFixed(t *testing.T) { + for _, mode := range []string{"confirmed", "read confirmation", "reset confirmation", "temporary", "fourth worker", "feature", "rerun", "other operation"} { + t.Run(mode, func(t *testing.T) { + env := dispatchFixture() + env["LONGHORN_V2_OPERATION"], env["LONGHORN_V2_NODE"], env["LONGHORN_V2_CONFIRM"] = "reboot-host-for-hugepages", "prod-worker-1", "reboot-longhorn-v2-host-prod-worker-1" + switch mode { + case "read confirmation": + env["LONGHORN_V2_CONFIRM"] = "inspect-longhorn-v2-host-memory-prod-worker-1" + case "reset confirmation": + env["LONGHORN_V2_CONFIRM"] = "prepare-longhorn-v2-prod-worker-1" + case "temporary": + env["LONGHORN_V2_NODE"] = "temporary-v1-1" + env["LONGHORN_V2_CONFIRM"] = "reboot-longhorn-v2-host-temporary-v1-1" + case "fourth worker": + env["LONGHORN_V2_NODE"] = "prod-worker-4" + env["LONGHORN_V2_CONFIRM"] = "reboot-longhorn-v2-host-prod-worker-4" + case "feature": + env["GITHUB_REF"] = "refs/heads/feature" + case "rerun": + env["GITHUB_RUN_ATTEMPT"] = "2" + case "other operation": + env["LONGHORN_V2_OPERATION"] = "inspect-host-memory" + } + getenv := func(key string) string { return env[key] } + if (authorizeHostReboot(getenv, env["LONGHORN_V2_NODE"]) == nil) != (mode == "confirmed") { + t.Fatal("reboot authorization crossed its operation, confirmation, selection or attempt boundary") + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/inventory_temporary_test.go b/scripts/migrate-longhorn-v2/inventory_temporary_test.go index 13be22dc02..b7e6bf4316 100644 --- a/scripts/migrate-longhorn-v2/inventory_temporary_test.go +++ b/scripts/migrate-longhorn-v2/inventory_temporary_test.go @@ -251,6 +251,12 @@ func TestTemporaryInventoryNativeWorkflowGuardPreservesResetBoundary(t *testing. operation, node, ref, attempt, confirm string allowed bool }{ + {"reboot-host-for-hugepages", "prod-worker-1", "refs/heads/main", "1", "reboot-longhorn-v2-host-prod-worker-1", true}, + {"reboot-host-for-hugepages", "temporary-v1-1", "refs/heads/main", "1", "reboot-longhorn-v2-host-temporary-v1-1", false}, + {"reboot-host-for-hugepages", "prod-worker-4", "refs/heads/main", "1", "reboot-longhorn-v2-host-prod-worker-4", false}, + {"reboot-host-for-hugepages", "prod-worker-1", "refs/heads/feature", "1", "reboot-longhorn-v2-host-prod-worker-1", false}, + {"reboot-host-for-hugepages", "prod-worker-1", "refs/heads/main", "2", "reboot-longhorn-v2-host-prod-worker-1", false}, + {"reboot-host-for-hugepages", "prod-worker-1", "refs/heads/main", "1", "prepare-longhorn-v2-prod-worker-1", false}, {"inventory-storage-layout", "temporary-v1-1", "refs/heads/main", "1", "inventory-longhorn-storage-layout-temporary-v1-1", true}, {"inventory-storage-layout", "temporary-v2-1", "refs/heads/main", "1", "inventory-longhorn-storage-layout-temporary-v2-1", true}, {"inventory-storage-layout", "prod-worker-1", "refs/heads/main", "1", "inventory-longhorn-storage-layout-prod-worker-1", false}, diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index 1156fda6d9..54ed016e2f 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -20,6 +20,8 @@ func main() { authorization = authorizeInventory(os.Getenv, target) case "inspect-host-memory": authorization = authorizeHostMemory(os.Getenv, target) + case "reboot-host-for-hugepages": + authorization = authorizeHostReboot(os.Getenv, target) case "inventory-storage-root": authorization = authorizeStorageRoot(os.Getenv, target) case "inventory-storage-layout": @@ -79,6 +81,18 @@ func main() { fmt.Fprintln(os.Stderr, err) os.Exit(1) } + if operation == "reboot-host-for-hugepages" { + h := hostMemoryReboot{c: client{run: execute, context: "admin@prod"}, target: target, source: os.Getenv("GITHUB_SHA"), owner: os.Getenv("GITHUB_RUN_ID"), pools: pools} + if err = h.run(ctx); err != nil { + fmt.Fprintln(os.Stderr, "Host memory maintenance held:", err) + fmt.Fprintln(os.Stderr, "Preserve its journal and owned cordon; uncertain reboot requests are never repeated.") + os.Exit(1) + } + if _, err = fmt.Fprintln(os.Stdout, "Host memory maintenance recovered with original storage identities, native allocation, advertised capacity and database identity proved."); err != nil { + os.Exit(1) + } + return + } if operation == "prepare-worker" { p := preparation{c: client{run: execute, context: "admin@prod"}, target: target, name: "longhorn-v2-prepare-" + target, source: os.Getenv("GITHUB_SHA"), owner: os.Getenv("GITHUB_RUN_ID"), pools: pools} From daf8ff3f56dca63b8f230569d7495a2a58bca3ae Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 05:31:37 +0100 Subject: [PATCH 11/35] fix(storage): honor the host recovery deadline Exercise the actual recovery probe deadline, retain shorter parent deadlines, and check fixture encoding before qualifying a pending peer. Part of #5791. --- scripts/migrate-longhorn-v2/host_reboot.go | 2 +- .../migrate-longhorn-v2/host_reboot_test.go | 39 ++++++++++++++++++- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/scripts/migrate-longhorn-v2/host_reboot.go b/scripts/migrate-longhorn-v2/host_reboot.go index 6646b8a88c..0e8d88bb1f 100644 --- a/scripts/migrate-longhorn-v2/host_reboot.go +++ b/scripts/migrate-longhorn-v2/host_reboot.go @@ -381,7 +381,7 @@ func (h hostMemoryReboot) run(ctx context.Context) error { recovery, cancel := context.WithTimeout(ctx, 20*time.Minute) defer cancel() var recovered object - if err = waitReady(recovery, func(ctx context.Context) error { + if err = waitReadyWithin(recovery, 20*time.Minute, func(ctx context.Context) error { n, _, err := h.targetState(ctx, r) boot := text(n, "status", "nodeInfo", "bootID") if err != nil || !inventoryBoot.MatchString(boot) || boot == r.OriginalBootID || r.RecoveredBootID != "" && boot != r.RecoveredBootID || !hugepagesReady(n) { diff --git a/scripts/migrate-longhorn-v2/host_reboot_test.go b/scripts/migrate-longhorn-v2/host_reboot_test.go index f252895237..2c51bc6128 100644 --- a/scripts/migrate-longhorn-v2/host_reboot_test.go +++ b/scripts/migrate-longhorn-v2/host_reboot_test.go @@ -174,6 +174,40 @@ func TestHostRebootRequiresPositiveRecoveryAndPreservesEveryDisk(t *testing.T) { } } +func TestHostRebootRecoveryProbeGetsTheDeclaredWindowAndParentDeadline(t *testing.T) { + for _, parentLimit := range []time.Duration{0, 2 * time.Minute} { + t.Run(parentLimit.String(), func(t *testing.T) { + f, h := hostRebootAPI(t) + ctx := context.Background() + if parentLimit > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, parentLimit) + defer cancel() + } + want := 20 * time.Minute + if parentLimit > 0 { + want = parentLimit + } + observed := false + inspect := h.inspect + h.inspect = func(ctx context.Context) (hostMemoryReport, error) { + if f.rebooted { + deadline, bounded := ctx.Deadline() + remaining := time.Until(deadline) + if !bounded || remaining > want || remaining < want-time.Minute { + t.Fatalf("recovery probe got an incorrect deadline: remaining=%v want=%v bounded=%v", remaining, want, bounded) + } + observed = true + } + return inspect(ctx) + } + if err := h.run(ctx); err != nil || !observed { + t.Fatal("recovery never completed its bounded native observation:", err) + } + }) + } +} + func TestHostRebootHoldsBeforeRequestOnLostSafetyEvidence(t *testing.T) { for _, mode := range []string{"source", "desired pages", "foreign cordon", "database", "replica", "drain", "identity after drain", "source after drain", "V2 flag", "setting scope", "storage scope after drain"} { t.Run(mode, func(t *testing.T) { @@ -258,7 +292,10 @@ func TestHostRebootCannotStartWhileAnotherWorkerHasAnUnfinishedJournal(t *testin if r.validate("prod-worker-2") != nil { t.Fatal("peer fixture did not represent a valid pending request") } - encoded, _ := json.Marshal(r) + encoded, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } f.peerJournal = resource("longhorn-v2-host-memory-prod-worker-2") f.peerJournal["metadata"].(object)["namespace"] = "longhorn-system" f.peerJournal["data"] = object{"record": string(encoded)} From 8def99504f212a41139eb182fd2a00c5b093f564 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 05:45:54 +0100 Subject: [PATCH 12/35] docs(storage): explain the host reboot safety boundaries Document journal ownership, original storage bindings, peer exclusion, native observation and recovery semantics. Part of #5791. --- scripts/migrate-longhorn-v2/host_reboot.go | 28 ++++++++++++++++++++++ scripts/migrate-longhorn-v2/main.go | 2 ++ 2 files changed, 30 insertions(+) diff --git a/scripts/migrate-longhorn-v2/host_reboot.go b/scripts/migrate-longhorn-v2/host_reboot.go index 0e8d88bb1f..9eef28cb39 100644 --- a/scripts/migrate-longhorn-v2/host_reboot.go +++ b/scripts/migrate-longhorn-v2/host_reboot.go @@ -19,6 +19,8 @@ type hostMemoryReboot struct { inspect func(context.Context) (hostMemoryReport, error) } +// authorizeHostReboot requires a separate confirmation for the fixed permanent +// worker selection, then applies the shared protected-workflow admission gate. func authorizeHostReboot(getenv func(string) string, target string) error { if !inventoryTarget.MatchString(target) || getenv("LONGHORN_V2_NODE") != target || getenv("LONGHORN_V2_OPERATION") != "reboot-host-for-hugepages" { return errHostMemory @@ -50,11 +52,15 @@ type hostDatabaseIdentity struct{ UID, SystemID string } var hostSystemID = regexp.MustCompile(`^[1-9][0-9]{0,19}$`) +// identity supplies the existing worker validator with immutable bindings; the +// reboot record retains its own operation phases and storage-preservation proof. func (r hostRebootRecord) identity() journal { return journal{Schema: 1, Target: r.Target, Source: r.Source, OwnerRun: r.OwnerRun, Phase: "draining", NodeUID: r.NodeUID, ProviderID: r.ProviderID, MachineUUID: r.MachineUUID, LonghornUID: r.LonghornUID, EphemeralUUID: r.EphemeralUUID, OriginalDisks: r.OriginalDisks} } +// validate rejects incomplete or out-of-profile records before their phase can authorize +// an action, including incomplete original storage and database identities. func (r hostRebootRecord) validate(target string) error { if r.Version != 1 || validateJournal(r.identity(), target) != nil || r.OriginalDiskUUID == "" || !inventoryBoot.MatchString(r.OriginalBootID) || r.OSImage == "" || r.Architecture != "amd64" || len(r.Databases) != 3 { return errHostMemory @@ -79,6 +85,8 @@ func (r hostRebootRecord) validate(target string) error { return nil } +// hostRegisteredDiskUUID accepts the single registered V1 filesystem disk. +// Recording its identity separately detects replacement behind an unchanged path. func hostRegisteredDiskUUID(storage object) string { disks, ok := value(storage, "spec", "disks").(object) statuses, statusOK := value(storage, "status", "diskStatus").(object) @@ -99,6 +107,8 @@ func hostRegisteredDiskUUID(storage object) string { func (h hostMemoryReboot) name() string { return "longhorn-v2-host-memory-" + h.target } +// peersClear prevents a new disruption while another worker has an unfinished +// request or has lost its recovery proof. Recovery of this worker stays possible. func (h hostMemoryReboot) peersClear(ctx context.Context, nodes []object) error { byName, err := indexed(nodes) if err != nil { @@ -127,6 +137,8 @@ func (h hostMemoryReboot) peersClear(ctx context.Context, nodes []object) error return nil } +// read accepts only the fixed journal object and a singular, validated record. +// A malformed or mis-scoped response cannot supply a later patch destination. func (h hostMemoryReboot) read(ctx context.Context) (object, hostRebootRecord, error) { cm, err := h.c.get(ctx, "configmap", "longhorn-system", h.name(), true) if err != nil || cm == nil { @@ -149,6 +161,8 @@ func (h hostMemoryReboot) read(ctx context.Context) (object, hostRebootRecord, e return cm, r, nil } +// write persists intent only for the current reviewed source and run. Existing +// records retain the API object's UID and resource-version preconditions. func (h hostMemoryReboot) write(ctx context.Context, cm object, r hostRebootRecord) error { if cm != nil && (text(cm, "metadata", "name") != h.name() || text(cm, "metadata", "namespace") != "longhorn-system") { return errHostMemory @@ -174,6 +188,8 @@ func (h hostMemoryReboot) write(ctx context.Context, cm object, r hostRebootReco return err } +// owned rebinds the complete journal and current main immediately before a +// worker mutation, so an intervening takeover or state change holds the action. func (h hostMemoryReboot) owned(ctx context.Context, expected hostRebootRecord) (object, error) { cm, actual, err := h.read(ctx) if err != nil || cm == nil || !reflect.DeepEqual(expected, actual) || actual.OwnerRun != h.owner || actual.Source != h.source { @@ -182,6 +198,8 @@ func (h hostMemoryReboot) owned(ctx context.Context, expected hostRebootRecord) return cm, sourceCurrent(ctx, h.c.run, h.source) } +// databases requires every declared database cluster to be fully healthy and +// preserves its cluster and system identities across the maintenance operation. func (h hostMemoryReboot) databases(ctx context.Context, expected map[string]hostDatabaseIdentity) (map[string]hostDatabaseIdentity, error) { items, err := h.c.list(ctx, "clusters.postgresql.cnpg.io", "*") if err != nil || len(items) != 3 { @@ -204,6 +222,8 @@ func (h hostMemoryReboot) databases(ctx context.Context, expected map[string]hos return result, nil } +// healthy gates this pre-V2 operation on the full migration roster, complete V1 +// redundancy and database health. Every call collects a fresh API inventory. func (h hostMemoryReboot) healthy(ctx context.Context, expected map[string]hostDatabaseIdentity) (inventory, map[string]hostDatabaseIdentity, error) { setting, err := h.c.get(ctx, "settings.longhorn.io", "longhorn-system", "v2-data-engine", false) if err != nil || text(setting, "metadata", "name") != "v2-data-engine" || text(setting, "metadata", "namespace") != "longhorn-system" || text(setting, "value") != "false" { @@ -225,6 +245,8 @@ func (h hostMemoryReboot) healthy(ctx context.Context, expected map[string]hostD return s, databases, err } +// targetState binds the live endpoint to the original hardware, registered disk +// and encrypted filesystem. Only this operation's cordon may accompany recovery. func (h hostMemoryReboot) targetState(ctx context.Context, r hostRebootRecord) (object, string, error) { n, err := h.c.get(ctx, "node", "", h.target, false) if err != nil { @@ -253,6 +275,8 @@ func (h hostMemoryReboot) targetState(ctx context.Context, r hostRebootRecord) ( return n, address, nil } +// memory uses the native observer's unchanged API, hardware and boot bracket. +// The injected inspection boundary lets flow tests exercise recovery deadlines. func (h hostMemoryReboot) memory(ctx context.Context) (hostMemoryReport, error) { if h.inspect != nil { return h.inspect(ctx) @@ -261,6 +285,8 @@ func (h hostMemoryReboot) memory(ctx context.Context) (hostMemoryReport, error) return receipt.Report, err } +// initialize records an unused allocation shortfall and original identities +// before any cordon or eviction. A completed or busy allocation cannot start it. func (h hostMemoryReboot) initialize(ctx context.Context) (hostRebootRecord, error) { s, databases, err := h.healthy(ctx, nil) if err != nil { @@ -301,6 +327,8 @@ func (h hostMemoryReboot) initialize(ctx context.Context) (hostRebootRecord, err return r, h.write(ctx, nil, r) } +// run coordinates one journaled reboot and proves recovery before releasing its +// cordon. A saved reboot-requested phase resumes observation without redelivery. func (h hostMemoryReboot) run(ctx context.Context) error { if !inventoryTarget.MatchString(h.target) { return errHostMemory diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index 54ed016e2f..58dae5dd79 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -7,6 +7,8 @@ import ( "time" ) +// main admits explicit migration operations through their distinct confirmations +// and protected workflow identities before constructing any production client. func main() { operation := os.Getenv("LONGHORN_V2_OPERATION") target := os.Getenv("LONGHORN_V2_NODE") From 21640913ace9a0fd17ea337fd9f95d155f3aafb8 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 05:48:57 +0100 Subject: [PATCH 13/35] fix(storage): qualify failed unallocated raw requests Require a never-located prior phase, empty allocation identities and complete physical absence alongside the original encrypted partition proof. Part of #5787. --- scripts/migrate-longhorn-v2/README.md | 6 ++ .../migrate-longhorn-v2/temporary_storage.go | 3 +- .../temporary_storage_test.go | 60 +++++++++++++++++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 52a8e3d3e0..787b489eef 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -41,6 +41,12 @@ inventories, and no allocated raw partition. Only the recognized engine, metadat socket cache directories may remain; a replica entry, backing images, any file, unknown entry, incomplete read or changed identity holds the run. The original encrypted partition is checked before each parent listing. A missing replica-path error is never interpreted as empty storage. +An allocation in `failed` state qualifies only when its previous phase was `waiting` or +`missing`, its type is a partition, every allocation size, location and identity field is +empty, and complete physical discovery contains no raw or other user partition. A failure +after locating, provisioning or opening a device holds the run. Provider error messages do +not establish absence. These checks accompany both parent reads and the unchanged API, +hardware, boot and original-partition bindings; observation-only receipts grant no reset clearance. All three temporary workers receive owned scale-down protection before the selected worker is cordoned. Drain uses eviction and respects disruption budgets. Two fresh storage observations after drain precede the recorded reset intent. diff --git a/scripts/migrate-longhorn-v2/temporary_storage.go b/scripts/migrate-longhorn-v2/temporary_storage.go index f10cb19d92..719cb4e44d 100644 --- a/scripts/migrate-longhorn-v2/temporary_storage.go +++ b/scripts/migrate-longhorn-v2/temporary_storage.go @@ -84,7 +84,8 @@ func verifyUnusedTemporaryPartition(statuses, discovered cosiresource.List, ephe original = spec } if strings.HasPrefix(id, "r-") || strings.HasPrefix(id, "u-") { - if id != "r-longhorn" || spec.Phase != block.VolumePhaseWaiting || spec.Location != "" || spec.MountLocation != "" || spec.ParentLocation != "" || spec.UUID != "" || spec.PartitionUUID != "" || spec.PartitionIndex != 0 || spec.Size != 0 { + unallocated := spec.Phase == block.VolumePhaseWaiting || spec.Phase == block.VolumePhaseFailed && (spec.PreFailPhase == block.VolumePhaseWaiting || spec.PreFailPhase == block.VolumePhaseMissing) + if id != "r-longhorn" || !unallocated || spec.Type != block.VolumeTypePartition || spec.Location != "" || spec.MountLocation != "" || spec.ParentLocation != "" || spec.UUID != "" || spec.PartitionUUID != "" || spec.PartitionIndex != 0 || spec.Size != 0 { return errInventory } } diff --git a/scripts/migrate-longhorn-v2/temporary_storage_test.go b/scripts/migrate-longhorn-v2/temporary_storage_test.go index 0a09c80ac6..3226ef8b32 100644 --- a/scripts/migrate-longhorn-v2/temporary_storage_test.go +++ b/scripts/migrate-longhorn-v2/temporary_storage_test.go @@ -21,11 +21,71 @@ func temporaryPartitionFixture() (cosiresource.List, cosiresource.List, string) EncryptionProvider: block.EncryptionProviderLUKS2, EncryptionLockedToState: true, ConfiguredEncryptionKeys: []string{"nodeID"}} raw := block.NewVolumeStatus(block.NamespaceName, "r-longhorn") raw.TypedSpec().Phase = block.VolumePhaseWaiting + raw.TypedSpec().Type = block.VolumeTypePartition physical := block.NewDiscoveredVolume(block.NamespaceName, "sda6") *physical.TypedSpec() = block.DiscoveredVolumeSpec{DevPath: "/dev/sda6", ParentDevPath: "/dev/sda", PartitionLabel: "EPHEMERAL", PartitionUUID: "partition-original", Size: 150 << 30} return cosiresource.List{Items: []cosiresource.Resource{eph, raw}}, cosiresource.List{Items: []cosiresource.Resource{physical}}, eph.TypedSpec().UUID } +func TestFailedTemporaryAllocationRequiresNeverLocatedStateAndNoPhysicalPartition(t *testing.T) { + for _, prior := range []block.VolumePhase{block.VolumePhaseWaiting, block.VolumePhaseMissing} { + s, d, uuid := temporaryPartitionFixture() + raw := s.Items[1].(*block.VolumeStatus).TypedSpec() + raw.Phase, raw.PreFailPhase = block.VolumePhaseFailed, prior + if err := verifyUnusedTemporaryPartition(s, d, uuid); err != nil { + t.Fatalf("complete never-allocated failure was rejected: prior=%v err=%v", prior, err) + } + } + for name, change := range map[string]func(*cosiresource.List, *cosiresource.List){ + "previously located": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().PreFailPhase = block.VolumePhaseLocated + }, + "previously provisioned": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().PreFailPhase = block.VolumePhaseProvisioned + }, + "previously prepared": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().PreFailPhase = block.VolumePhasePrepared + }, + "previously ready": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().PreFailPhase = block.VolumePhaseReady + }, + "unknown prior phase": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().PreFailPhase = block.VolumePhase(99) + }, + "wrong raw type": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().Type = block.VolumeTypeDisk + }, + "allocated size": func(s, _ *cosiresource.List) { s.Items[1].(*block.VolumeStatus).TypedSpec().Size = 96 << 30 }, + "device location": func(s, _ *cosiresource.List) { s.Items[1].(*block.VolumeStatus).TypedSpec().Location = "/dev/sda7" }, + "parent device": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().ParentLocation = "/dev/sda" + }, + "mount location": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().MountLocation = "/dev/dm-1" + }, + "filesystem identity": func(s, _ *cosiresource.List) { s.Items[1].(*block.VolumeStatus).TypedSpec().UUID = "existing-data" }, + "partition identity": func(s, _ *cosiresource.List) { + s.Items[1].(*block.VolumeStatus).TypedSpec().PartitionUUID = "existing-partition" + }, + "partition index": func(s, _ *cosiresource.List) { s.Items[1].(*block.VolumeStatus).TypedSpec().PartitionIndex = 7 }, + "discovered raw partition": func(_, d *cosiresource.List) { + v := block.NewDiscoveredVolume(block.NamespaceName, "raw-device") + *v.TypedSpec() = block.DiscoveredVolumeSpec{DevPath: "/dev/sda7", ParentDevPath: "/dev/sda", PartitionLabel: "r-longhorn", Size: 96 << 30} + d.Items = append(d.Items, v) + }, + } { + t.Run(name, func(t *testing.T) { + s, d, uuid := temporaryPartitionFixture() + raw := s.Items[1].(*block.VolumeStatus).TypedSpec() + raw.Phase, raw.PreFailPhase = block.VolumePhaseFailed, block.VolumePhaseWaiting + change(&s, &d) + if verifyUnusedTemporaryPartition(s, d, uuid) == nil { + t.Fatal("failed allocation hid a previous device, unknown state or discovered data") + } + }) + } +} + func TestNativeTemporaryPartitionRequiresPositiveOriginalAndNoRawStorage(t *testing.T) { statuses, discovered, uuid := temporaryPartitionFixture() if err := verifyUnusedTemporaryPartition(statuses, discovered, uuid); err != nil { From 6121b342465d466331df08dc623ebf526ba04001 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 06:02:51 +0100 Subject: [PATCH 14/35] docs(storage): explain unused worker preparation invariants --- .../migrate-longhorn-v2/inventory_observe.go | 3 +++ .../migrate-longhorn-v2/inventory_runtime.go | 3 +++ .../inventory_temporary.go | 3 +++ scripts/migrate-longhorn-v2/journal.go | 10 +++++++++ .../temporary_authorization.go | 2 ++ .../migrate-longhorn-v2/temporary_prepare.go | 22 +++++++++++++++++++ .../migrate-longhorn-v2/temporary_recovery.go | 2 ++ .../migrate-longhorn-v2/temporary_storage.go | 10 ++++++++- 8 files changed, 54 insertions(+), 1 deletion(-) diff --git a/scripts/migrate-longhorn-v2/inventory_observe.go b/scripts/migrate-longhorn-v2/inventory_observe.go index e611323b3f..eddfcaa298 100644 --- a/scripts/migrate-longhorn-v2/inventory_observe.go +++ b/scripts/migrate-longhorn-v2/inventory_observe.go @@ -32,6 +32,9 @@ type inventoryReceipt struct { Report longhorninventory.Report `json:"report"` } +// inventoryProjection binds a complete Kubernetes storage observation to the +// selected physical node and reviewed pool roster. A preparation journal permits +// only its own identity-bound cordon in the observation view. func (c client) inventoryProjection(ctx context.Context, target string) (inventoryProjection, object, error) { var nodes, storage, replicas []object for _, query := range []struct { diff --git a/scripts/migrate-longhorn-v2/inventory_runtime.go b/scripts/migrate-longhorn-v2/inventory_runtime.go index 67e7a596e1..91eaecb778 100644 --- a/scripts/migrate-longhorn-v2/inventory_runtime.go +++ b/scripts/migrate-longhorn-v2/inventory_runtime.go @@ -126,6 +126,9 @@ func newInventoryReader(ctx context.Context, address string) (nativeReplicaReade return openInventoryReader(ctx, address, "") } +// openInventoryReader opens a node-bound native client using the protected +// workflow's memory-backed credentials and bounded block API. The unused-worker +// profile additionally verifies its original partition before directory reads. func openInventoryReader(ctx context.Context, address, unusedEphemeralUUID string) (nativeReplicaReader, func() error, error) { directory := os.Getenv("LONGHORN_V2_CREDENTIAL_DIR") if !regexp.MustCompile(`^/dev/shm/longhorn-v2\.[A-Za-z0-9]+$`).MatchString(directory) || os.Getenv("TALOSCONFIG") != filepath.Join(directory, "talosconfig") { diff --git a/scripts/migrate-longhorn-v2/inventory_temporary.go b/scripts/migrate-longhorn-v2/inventory_temporary.go index fa7300b451..bbbadb967e 100644 --- a/scripts/migrate-longhorn-v2/inventory_temporary.go +++ b/scripts/migrate-longhorn-v2/inventory_temporary.go @@ -14,6 +14,9 @@ func resolveTemporaryInventoryTarget(selection string, nodes []object, pools map return resolveTemporarySelection(selection, nodes, pools, nil) } +// resolveTemporarySelection binds an ordinal to the complete reviewed pool roster. +// Recovery may relax readiness only for the journaled original physical member; +// it never substitutes another node or grants permission for another reset. func resolveTemporarySelection(selection string, nodes []object, pools map[string]migrationPool, postReset *journal) (string, string, error) { var role string var ordinal int diff --git a/scripts/migrate-longhorn-v2/journal.go b/scripts/migrate-longhorn-v2/journal.go index 61ee1edac1..9fe197e712 100644 --- a/scripts/migrate-longhorn-v2/journal.go +++ b/scripts/migrate-longhorn-v2/journal.go @@ -27,6 +27,9 @@ type journal struct { RawVolumeAbsent bool `json:"rawVolumeAbsent,omitempty"` } +// validateJournal requires complete original identities and a known procedure +// phase. Its disjoint schemas keep permanent V1-disk records separate from +// temporary-worker records carrying positive empty-storage evidence. func validateJournal(j journal, target string) error { if j.Target != target || !regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString(j.Source) || !regexp.MustCompile(`^[1-9][0-9]*$`).MatchString(j.OwnerRun) || @@ -71,6 +74,8 @@ type preparation struct { temporarySelection string } +// validateRecord binds a structurally valid journal to the explicitly selected +// preparation profile and, for temporary workers, its fixed journal name. func (p preparation) validateRecord(j journal) error { if err := validateJournal(j, p.target); err != nil { return err @@ -81,6 +86,8 @@ func (p preparation) validateRecord(j journal) error { return nil } +// readRecord decodes exactly one strict journal and validates its original +// identities against the selected preparation profile before it can be used. func (p preparation) readRecord(ctx context.Context) (object, journal, error) { cm, err := p.c.get(ctx, "configmap", "longhorn-system", p.name, true) if err != nil || cm == nil { @@ -106,6 +113,9 @@ func reviewedWorkflowPath(path string) bool { return path == workflow || path == workflow+"@main" || path == workflow+"@refs/heads/main" } +// claimRecord creates a validated journal on current main or adopts a former +// owner only after its reviewed first-attempt workflow is proven terminal. +// Adoption preserves the recorded phase and original storage identities. func (p *preparation) claimRecord(ctx context.Context, initial journal) error { cm, j, err := p.readRecord(ctx) if err != nil { diff --git a/scripts/migrate-longhorn-v2/temporary_authorization.go b/scripts/migrate-longhorn-v2/temporary_authorization.go index deb529f221..0252d82b76 100644 --- a/scripts/migrate-longhorn-v2/temporary_authorization.go +++ b/scripts/migrate-longhorn-v2/temporary_authorization.go @@ -2,6 +2,8 @@ package main import "errors" +// authorizeTemporaryPreparation restricts the destructive profile to its two +// reviewed aliases and requires the selection-specific protected-workflow token. func authorizeTemporaryPreparation(getenv func(string) string, selection string) error { if (selection != "temporary-v1-1" && selection != "temporary-v1-2") || getenv("LONGHORN_V2_NODE") != selection || getenv("LONGHORN_V2_OPERATION") != "prepare-unused-temporary-worker" { return errors.New("the separate unused-temporary-worker preparation selection is required") diff --git a/scripts/migrate-longhorn-v2/temporary_prepare.go b/scripts/migrate-longhorn-v2/temporary_prepare.go index d822c7d5e9..1fef3e4418 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare.go @@ -16,6 +16,9 @@ type temporaryPreparation struct { reader inventoryReaderFactory } +// prepare advances the separately journaled unused-worker profile, recording reset +// intent before its single request and accepting recovery only after native proof. +// A resumed reset request observes recovery without issuing another reset. func (p *temporaryPreparation) prepare(ctx context.Context) error { if err := p.beginUnused(ctx); err != nil { return err @@ -86,6 +89,9 @@ func (p *temporaryPreparation) prepare(ctx context.Context) error { return nil } +// unusedCapacity checks the fixed pool roster, reserved migration budget and +// hugepage capacity, tolerating only this journal's own cordon during preparation. +// Once pinned, every temporary pool member must retain the migration-owned pin. func (p temporaryPreparation) unusedCapacity(s inventory, pinned bool) error { if err := verifyRebuildBudget(p.target, s); err != nil { return err @@ -121,6 +127,9 @@ func (p temporaryPreparation) unusedCapacity(s inventory, pinned bool) error { return nil } +// beginUnused binds the fixed selection to its original physical and storage +// identities. New journals require a cleaned baseline and positive empty-storage +// evidence; existing journals must pass profile-specific adoption checks. func (p *temporaryPreparation) beginUnused(ctx context.Context) error { cm, _, err := p.readRecord(ctx) if err != nil { @@ -187,6 +196,9 @@ func (p *temporaryPreparation) beginUnused(ctx context.Context) error { return p.claimRecord(ctx, p.j) } +// observeUnused brackets complete native storage observations with current-source +// and API identity checks, then commits the original encrypted partition and +// cache-only parent directory evidence into the journal's digest. func (p temporaryPreparation) observeUnused(ctx context.Context, owned bool) (string, error) { if err := sourceCurrent(ctx, p.c.run, p.source); err != nil { return "", err @@ -237,6 +249,8 @@ func unusedParentAbsent(report longhorninventory.StorageRootReport) bool { !report.ReplicaEntryPresent && !report.ReplicaDirectoryPresent && !report.BackingImagesPresent && report.Directories == known } +// protectUnusedCapacity claims scale-down protection for the complete temporary +// pool through identity-bound patches and verifies the resulting pins afresh. func (p temporaryPreparation) protectUnusedCapacity(ctx context.Context) error { s, err := p.c.snapshot(ctx) if err != nil { @@ -290,6 +304,9 @@ func (p temporaryPreparation) protectUnusedCapacity(ctx context.Context) error { return p.unusedCapacity(current, true) } +// resetUnused drains through normal eviction, repeats all evacuation and native +// absence checks, and records durable intent before one EPHEMERAL-only reset. +// A lost response leaves recovery to observation rather than request replay. func (p *temporaryPreparation) resetUnused(ctx context.Context) error { n, address, err := p.node(ctx, true) if err != nil { @@ -338,6 +355,9 @@ func (p *temporaryPreparation) resetUnused(ctx context.Context) error { return nil } +// unusedPostState requires the original node identity with a new boot, a replaced +// encrypted EPHEMERAL partition and the separately encrypted raw volume. It also +// verifies advertised hugepages and that Longhorn storage remains unregistered. func (p temporaryPreparation) unusedPostState(ctx context.Context) (node object, err error) { n, address, err := p.node(ctx, true) if err != nil { @@ -393,6 +413,8 @@ func (p temporaryPreparation) unusedPostState(ctx context.Context) (node object, return n, nil } +// unusedStorageFenced accepts only an empty Longhorn node whose scheduling is +// disabled, eviction is requested and storage enablement has not been applied. func unusedStorageFenced(lh object) bool { disks, ok := value(lh, "spec", "disks").(map[string]any) diskStatus, statusOK := value(lh, "status", "diskStatus").(map[string]any) diff --git a/scripts/migrate-longhorn-v2/temporary_recovery.go b/scripts/migrate-longhorn-v2/temporary_recovery.go index bf67dc88ab..e2a3dbda9e 100644 --- a/scripts/migrate-longhorn-v2/temporary_recovery.go +++ b/scripts/migrate-longhorn-v2/temporary_recovery.go @@ -1,5 +1,7 @@ package main +// resolveTemporaryResetRecovery selects only the original journaled pool member +// after durable reset intent, allowing observation while that member recovers. func resolveTemporaryResetRecovery(selection string, nodes []object, pools map[string]migrationPool, record journal) (string, error) { if record.Schema != 2 || record.Phase != "reset-requested" || record.TemporarySelection != selection || validateJournal(record, record.Target) != nil { return "", errInventory diff --git a/scripts/migrate-longhorn-v2/temporary_storage.go b/scripts/migrate-longhorn-v2/temporary_storage.go index 719cb4e44d..5caba86911 100644 --- a/scripts/migrate-longhorn-v2/temporary_storage.go +++ b/scripts/migrate-longhorn-v2/temporary_storage.go @@ -28,6 +28,8 @@ func (r unusedTemporaryReader) ListReplicas(ctx context.Context) (longhorninvent return r.nativeReplicaReader.ListReplicas(ctx) } +// ListStorageRoot verifies the original encrypted partition and absence of any +// allocated raw storage before reading the complete Longhorn parent directory. func (r unusedTemporaryReader) ListStorageRoot(ctx context.Context) (longhorninventory.Listing, error) { if err := r.verifyPartition(ctx); err != nil { return longhorninventory.Listing{}, err @@ -39,6 +41,8 @@ func (r unusedTemporaryReader) ListStorageRoot(ctx context.Context) (longhorninv return reader.ListStorageRoot(ctx) } +// ListBlockLayout preserves the native reader's complete typed layout report and +// refuses a reader that cannot provide that observation capability. func (r unusedTemporaryReader) ListBlockLayout(ctx context.Context) (longhorninventory.BlockLayoutReport, error) { reader, ok := r.nativeReplicaReader.(nativeBlockLayoutReader) if !ok { @@ -47,6 +51,8 @@ func (r unusedTemporaryReader) ListBlockLayout(ctx context.Context) (longhorninv return reader.ListBlockLayout(ctx) } +// verifyPartition reads both bounded native block collections on the fixed node +// under one deadline; partial results or a changed partition withhold clearance. func (r unusedTemporaryReader) verifyPartition(ctx context.Context) error { address, addressErr := netip.ParseAddr(r.address) if ctx.Err() != nil || r.blocks == nil || r.nativeReplicaReader == nil || r.ephemeralUUID == "" || addressErr != nil || address.String() != r.address || !netip.MustParsePrefix("10.0.0.0/16").Contains(address) { @@ -65,7 +71,9 @@ func (r unusedTemporaryReader) verifyPartition(ctx context.Context) error { return nil } -// Check the complete native block-resource inventory, never a CLI error string. +// verifyUnusedTemporaryPartition requires the original ready encrypted partition +// and complete physical absence of raw or user storage. A failed raw request is +// unallocated only if its previous phase and every allocation field prove that. func verifyUnusedTemporaryPartition(statuses, discovered cosiresource.List, ephemeralUUID string) error { if ephemeralUUID == "" || len(statuses.Items) == 0 || len(statuses.Items) > 128 || len(discovered.Items) == 0 || len(discovered.Items) > 512 { return errInventory From e7a4c1fc763f979f6aab06a567b11dca9f6876ed Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 06:08:14 +0100 Subject: [PATCH 15/35] fix(storage): bind Talos parameters and reject busy peers before journaling --- scripts/migrate-longhorn-v2/host_reboot.go | 7 ++- .../migrate-longhorn-v2/host_reboot_test.go | 58 +++++++++++-------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/scripts/migrate-longhorn-v2/host_reboot.go b/scripts/migrate-longhorn-v2/host_reboot.go index 9eef28cb39..f23a0540c9 100644 --- a/scripts/migrate-longhorn-v2/host_reboot.go +++ b/scripts/migrate-longhorn-v2/host_reboot.go @@ -268,7 +268,7 @@ func (h hostMemoryReboot) targetState(ctx context.Context, r hostRebootRecord) ( if err != nil || text(volume, "spec", "uuid") != r.EphemeralUUID || text(volume, "spec", "phase") != "ready" || text(volume, "spec", "encryptionProvider") != "luks2" { return nil, "", errHostMemory } - setting, err := h.c.talosResource(ctx, address, "kernelparamspec", "sysctl.vm.nr_hugepages") + setting, err := h.c.talosResource(ctx, address, "kernelparamspec", "proc.sys.vm.nr_hugepages") if err != nil || text(setting, "metadata", "namespace") != "runtime" || text(setting, "spec", "value") != "1024" { return nil, "", errHostMemory } @@ -324,6 +324,11 @@ func (h hostMemoryReboot) initialize(ctx context.Context) (hostRebootRecord, err if err != nil || memory.PersistentPages >= 1024 || memory.TotalPages != memory.PersistentPages || memory.FreePages != memory.TotalPages || memory.PageSizeKiB != 2048 || memory.ReservedPages != 0 || memory.SurplusPages != 0 { return hostRebootRecord{}, errors.New("native allocation no longer proves an unused huge-page shortfall; inspect host memory before a reboot") } + // A refused dispatch must not leave a second unfinished journal that would + // prevent the original worker from resuming its own maintenance. + if err = h.peersClear(ctx, s.nodes); err != nil { + return hostRebootRecord{}, err + } return r, h.write(ctx, nil, r) } diff --git a/scripts/migrate-longhorn-v2/host_reboot_test.go b/scripts/migrate-longhorn-v2/host_reboot_test.go index 2c51bc6128..b897cedbe5 100644 --- a/scripts/migrate-longhorn-v2/host_reboot_test.go +++ b/scripts/migrate-longhorn-v2/host_reboot_test.go @@ -100,7 +100,7 @@ func (f *hostRebootFixture) run(ctx context.Context, input []byte, program strin return nil, nil } if program == "talosctl" && len(args) == 5 && args[1] == "get" && args[2] == "kernelparamspec" { - if args[3] != "sysctl.vm.nr_hugepages" { + if args[3] != "proc.sys.vm.nr_hugepages" { f.base.t.Fatal("host configuration read escaped the single public parameter") } return json.Marshal(object{"metadata": object{"namespace": "runtime", "id": args[3]}, "spec": object{"value": f.desired}}) @@ -277,30 +277,38 @@ func TestUncertainHostRebootCannotBeRepeated(t *testing.T) { } func TestHostRebootCannotStartWhileAnotherWorkerHasAnUnfinishedJournal(t *testing.T) { - f, h := hostRebootAPI(t) - if _, err := h.initialize(context.Background()); err != nil { - t.Fatal(err) - } - _, r, err := h.read(context.Background()) - if err != nil { - t.Fatal(err) - } - peer, storage := f.base.s.nodes[1], f.base.s.storageNodes[1] - r.Target, r.Phase = "prod-worker-2", "reboot-requested" - r.NodeUID, r.ProviderID, r.MachineUUID = text(peer, "metadata", "uid"), text(peer, "spec", "providerID"), text(peer, "status", "nodeInfo", "systemUUID") - r.OriginalBootID, r.LonghornUID, r.OriginalDiskUUID = text(peer, "status", "nodeInfo", "bootID"), text(storage, "metadata", "uid"), hostRegisteredDiskUUID(storage) - if r.validate("prod-worker-2") != nil { - t.Fatal("peer fixture did not represent a valid pending request") - } - encoded, err := json.Marshal(r) - if err != nil { - t.Fatal(err) - } - f.peerJournal = resource("longhorn-v2-host-memory-prod-worker-2") - f.peerJournal["metadata"].(object)["namespace"] = "longhorn-system" - f.peerJournal["data"] = object{"record": string(encoded)} - if h.run(context.Background()) == nil || f.reboots != 0 || f.drains != 0 || value(f.base.node, "spec", "unschedulable") == true { - t.Fatal("another worker was disrupted before the earlier reboot was proven complete") + for _, phase := range []string{"draining", "reboot-requested"} { + t.Run(phase, func(t *testing.T) { + f, h := hostRebootAPI(t) + if _, err := h.initialize(context.Background()); err != nil { + t.Fatal(err) + } + _, r, err := h.read(context.Background()) + if err != nil { + t.Fatal(err) + } + peer, storage := f.base.s.nodes[1], f.base.s.storageNodes[1] + r.Target, r.Phase = "prod-worker-2", phase + r.NodeUID, r.ProviderID, r.MachineUUID = text(peer, "metadata", "uid"), text(peer, "spec", "providerID"), text(peer, "status", "nodeInfo", "systemUUID") + r.OriginalBootID, r.LonghornUID, r.OriginalDiskUUID = text(peer, "status", "nodeInfo", "bootID"), text(storage, "metadata", "uid"), hostRegisteredDiskUUID(storage) + if r.validate("prod-worker-2") != nil { + t.Fatal("peer fixture did not represent a valid pending request") + } + encoded, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + f.peerJournal = resource("longhorn-v2-host-memory-prod-worker-2") + f.peerJournal["metadata"].(object)["namespace"] = "longhorn-system" + f.peerJournal["data"] = object{"record": string(encoded)} + f.base.cm = nil + if h.run(context.Background()) == nil || f.reboots != 0 || f.drains != 0 || value(f.base.node, "spec", "unschedulable") == true { + t.Fatal("another worker was disrupted before the earlier reboot was proven complete") + } + if f.base.cm != nil { + t.Fatal("a refused peer dispatch left a new journal that blocks the original worker's recovery") + } + }) } } From 10b343a4751126ae5731f6c9eb988e5cdb91144b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 06:11:13 +0100 Subject: [PATCH 16/35] docs(storage): clarify migration admission and inventory binding --- scripts/migrate-longhorn-v2/inventory.go | 3 +++ scripts/migrate-longhorn-v2/main.go | 2 ++ 2 files changed, 5 insertions(+) diff --git a/scripts/migrate-longhorn-v2/inventory.go b/scripts/migrate-longhorn-v2/inventory.go index 50b34595ed..33742f7025 100644 --- a/scripts/migrate-longhorn-v2/inventory.go +++ b/scripts/migrate-longhorn-v2/inventory.go @@ -78,6 +78,9 @@ func inventoryFailureMessage(err error) string { return stageUnknown.Error() } +// projectInventory validates the observed node, disk registrations and replica +// bindings before committing their identities. Unregistered temporary nodes must +// retain the reviewed pool identity and an explicitly empty, fenced storage view. func projectInventory(target string, node, storage object, replicas []object) (inventoryProjection, error) { p := inventoryProjection{BootID: text(node, "status", "nodeInfo", "bootID"), StorageUID: text(storage, "metadata", "uid")} if (!inventoryTarget.MatchString(target) && !inventoryTemporaryNode.MatchString(target)) || !inventoryBoot.MatchString(p.BootID) || diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index e831d253e5..9d2963c008 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -8,6 +8,8 @@ import ( "time" ) +// main admits explicit migration operations through their distinct confirmations +// and protected workflow identities before constructing any production client. func main() { operation := os.Getenv("LONGHORN_V2_OPERATION") target := os.Getenv("LONGHORN_V2_NODE") From af20318216a62caba109434166cfe30ae2376208 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 07:23:21 +0100 Subject: [PATCH 17/35] fix(storage): wait for database replacement after host drain --- scripts/migrate-longhorn-v2/host_reboot.go | 41 +++++++++- .../migrate-longhorn-v2/host_reboot_test.go | 76 +++++++++++++++++++ 2 files changed, 115 insertions(+), 2 deletions(-) diff --git a/scripts/migrate-longhorn-v2/host_reboot.go b/scripts/migrate-longhorn-v2/host_reboot.go index f23a0540c9..2c6796498e 100644 --- a/scripts/migrate-longhorn-v2/host_reboot.go +++ b/scripts/migrate-longhorn-v2/host_reboot.go @@ -52,6 +52,8 @@ type hostDatabaseIdentity struct{ UID, SystemID string } var hostSystemID = regexp.MustCompile(`^[1-9][0-9]{0,19}$`) +var errHostDatabasesRecovering = errors.New("the original database quorum is still recovering after eviction") + // identity supplies the existing worker validator with immutable bindings; the // reboot record retains its own operation phases and storage-preservation proof. func (r hostRebootRecord) identity() journal { @@ -206,22 +208,54 @@ func (h hostMemoryReboot) databases(ctx context.Context, expected map[string]hos return nil, errHostMemory } result := map[string]hostDatabaseIdentity{} + recovering := false for _, db := range items { key := text(db, "metadata", "namespace") + "/" + text(db, "metadata", "name") if key != "observability/coroot-db" && key != "umami/umami-db" && key != "wedding-app/wedding-db" || result[key].UID != "" || text(db, "metadata", "uid") == "" || text(db, "metadata", "resourceVersion") == "" || value(db, "metadata", "deletionTimestamp") != nil || - value(db, "spec", "instances") != float64(3) || value(db, "status", "readyInstances") != float64(3) || - text(db, "status", "phase") != "Cluster in healthy state" || !hostSystemID.MatchString(text(db, "status", "systemID")) { + value(db, "spec", "instances") != float64(3) || !hostSystemID.MatchString(text(db, "status", "systemID")) { return nil, errHostMemory } result[key] = hostDatabaseIdentity{UID: text(db, "metadata", "uid"), SystemID: text(db, "status", "systemID")} + ready := value(db, "status", "readyInstances") + phase := text(db, "status", "phase") + if (ready != float64(2) && ready != float64(3)) || phase == "" { + return nil, errHostMemory + } + recovering = recovering || ready != float64(3) || phase != "Cluster in healthy state" } if expected != nil && !reflect.DeepEqual(result, expected) { return nil, errHostMemory } + if recovering { + return nil, errHostDatabasesRecovering + } return result, nil } +// waitDrainRecovery gives an unchanged database quorum a bounded startup window +// after eviction. Changed ownership, source, target identity, peer maintenance or +// other safety evidence fails immediately; the final two fresh checks still run. +func (h hostMemoryReboot) waitDrainRecovery(ctx context.Context, r hostRebootRecord) error { + return waitReadyWithin(ctx, 20*time.Minute, func(ctx context.Context) error { + if _, err := h.owned(ctx, r); err != nil { + return errors.Join(errReadinessTerminal, err) + } + n, _, err := h.targetState(ctx, r) + if err != nil || text(n, "status", "nodeInfo", "bootID") != r.OriginalBootID || value(n, "spec", "unschedulable") != true || text(n, "metadata", "annotations", fenceAnnotation) != h.name() { + return errors.Join(errReadinessTerminal, errHostMemory) + } + s, _, err := h.healthy(ctx, r.Databases) + if err != nil && !errors.Is(err, errHostDatabasesRecovering) { + return errors.Join(errReadinessTerminal, err) + } + if peerErr := h.peersClear(ctx, s.nodes); peerErr != nil { + return errors.Join(errReadinessTerminal, peerErr) + } + return err + }) +} + // healthy gates this pre-V2 operation on the full migration roster, complete V1 // redundancy and database health. Every call collects a fresh API inventory. func (h hostMemoryReboot) healthy(ctx context.Context, expected map[string]hostDatabaseIdentity) (inventory, map[string]hostDatabaseIdentity, error) { @@ -384,6 +418,9 @@ func (h hostMemoryReboot) run(ctx context.Context) error { if _, err = h.c.kubectl(ctx, nil, "drain", h.target, "--ignore-daemonsets", "--delete-emptydir-data", "--timeout=40m"); err != nil { return errors.New("host memory maintenance drain was blocked; no reboot was requested") } + if err = h.waitDrainRecovery(ctx, r); err != nil { + return err + } // Keep the existing full redundancy floor: no request while a copy or // database is still recovering from the eviction. for range 2 { diff --git a/scripts/migrate-longhorn-v2/host_reboot_test.go b/scripts/migrate-longhorn-v2/host_reboot_test.go index b897cedbe5..d35f3543f1 100644 --- a/scripts/migrate-longhorn-v2/host_reboot_test.go +++ b/scripts/migrate-longhorn-v2/host_reboot_test.go @@ -20,6 +20,7 @@ type hostRebootFixture struct { desired string drainError bool afterDrain, afterReboot func() + afterDatabaseRead func() previousLive bool } @@ -139,6 +140,9 @@ func (f *hostRebootFixture) run(ctx context.Context, input []byte, program strin return nil, nil } if args[2] == "get" && args[3] == "clusters.postgresql.cnpg.io" { + if f.drains > 0 && f.afterDatabaseRead != nil { + f.afterDatabaseRead() + } return json.Marshal(object{"items": f.databases}) } if args[2] == "drain" { @@ -174,6 +178,78 @@ func TestHostRebootRequiresPositiveRecoveryAndPreservesEveryDisk(t *testing.T) { } } +func TestHostRebootWaitsForTheOriginalDatabaseQuorumAfterDrain(t *testing.T) { + f, h := hostRebootAPI(t) + reads := 0 + f.afterDrain = func() { + f.databases[0]["status"].(object)["readyInstances"] = float64(2) + f.afterDatabaseRead = func() { + reads++ + if reads == 2 { + f.databases[0]["status"].(object)["readyInstances"] = float64(3) + } + } + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := h.run(ctx); err != nil { + t.Fatal("ordinary asynchronous database replacement aborted the guarded reboot:", err) + } + if reads < 2 || f.reboots != 1 || value(f.base.node, "spec", "unschedulable") == true { + t.Fatal("database recovery was not observed before reboot and scheduling recovery") + } +} + +func TestHostRebootPreservesItsCordonWhenDrainRecoveryTimesOut(t *testing.T) { + f, h := hostRebootAPI(t) + f.afterDrain = func() { f.databases[0]["status"].(object)["readyInstances"] = float64(2) } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err := h.run(ctx) + if !errors.Is(err, context.DeadlineExceeded) || f.drains != 1 || f.reboots != 0 || value(f.base.node, "spec", "unschedulable") != true { + t.Fatal("unproven post-drain recovery did not stop at its deadline with maintenance retained:", err) + } +} + +func TestHostRebootDrainRecoveryDoesNotRetryChangedIdentityOrLostQuorum(t *testing.T) { + for name, change := range map[string]func(object){ + "cluster identity": func(db object) { db["metadata"].(object)["uid"] = "replacement" }, + "system identity": func(db object) { db["status"].(object)["systemID"] = "987654321" }, + "lost quorum": func(db object) { db["status"].(object)["readyInstances"] = float64(1) }, + } { + t.Run(name, func(t *testing.T) { + f, h := hostRebootAPI(t) + reads := 0 + f.afterDrain = func() { + f.databases[0]["status"].(object)["readyInstances"] = float64(2) + change(f.databases[0]) + } + f.afterDatabaseRead = func() { reads++ } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + err := h.run(ctx) + if !errors.Is(err, errReadinessTerminal) || reads != 1 || f.reboots != 0 { + t.Fatal("an invalid recovery state was retried or allowed a reboot:", err) + } + }) + } +} + +func TestHostRebootDrainRecoveryRejectsAnUnreadablePeerDuringDatabaseStartup(t *testing.T) { + f, h := hostRebootAPI(t) + f.afterDrain = func() { + f.databases[0]["status"].(object)["readyInstances"] = float64(2) + f.peerJournal = resource("longhorn-v2-host-memory-prod-worker-2") + f.peerJournal["metadata"].(object)["namespace"] = "longhorn-system" + f.peerJournal["data"] = object{"record": "invalid"} + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := h.run(ctx); !errors.Is(err, errReadinessTerminal) || f.reboots != 0 { + t.Fatal("database startup hid an unreadable peer maintenance record:", err) + } +} + func TestHostRebootRecoveryProbeGetsTheDeclaredWindowAndParentDeadline(t *testing.T) { for _, parentLimit := range []time.Duration{0, 2 * time.Minute} { t.Run(parentLimit.String(), func(t *testing.T) { From 4437f86096e42d5342ffbcee81bace71f5cd4a53 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 07:27:03 +0100 Subject: [PATCH 18/35] docs(storage): describe bounded database recovery after drain --- scripts/migrate-longhorn-v2/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 87599b74f0..f34c4e6e44 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -23,8 +23,11 @@ permanent worker with `reboot-longhorn-v2-host-`. It requires the loaded reservation, unused native huge pages, disabled V2, the complete fixed migration roster, all V1 volumes fully redundant, and all three database clusters healthy. It records the original node, hardware, boot, encrypted EPHEMERAL partition, storage disks and -database identities before an owned cordon and an eviction-based drain. Two fresh -checks of complete redundancy, databases and original storage precede durable +database identities before an owned cordon and an eviction-based drain. After +drain, the unchanged database quorum has up to 20 minutes to finish replacement +startup. Changed identities, lost quorum or other safety evidence stop the wait; +a timeout preserves the cordon without requesting a reboot. Two fresh checks of +complete redundancy, databases and original storage still precede durable reboot intent. The single reboot request has no reset or disk-wipe selector. An unfinished or unreadable peer journal, another worker's cordon, or lost proof of a peer's recorded recovery prevents starting the next worker. From 4275b3e83f6db254a5d08dbbb7e7b4d6f0c7b3cb Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 08:08:36 +0100 Subject: [PATCH 19/35] fix(storage): preserve runtime through host reboot recovery --- scripts/migrate-longhorn-v2/README.md | 13 +- scripts/migrate-longhorn-v2/host_drain.go | 70 ++++++++ scripts/migrate-longhorn-v2/host_reboot.go | 51 +----- .../migrate-longhorn-v2/host_reboot_test.go | 156 +++++++++++++++++- scripts/migrate-longhorn-v2/host_recovery.go | 133 +++++++++++++++ scripts/migrate-longhorn-v2/preflight.go | 19 ++- .../migrate-longhorn-v2/temporary_prepare.go | 3 +- 7 files changed, 392 insertions(+), 53 deletions(-) create mode 100644 scripts/migrate-longhorn-v2/host_drain.go create mode 100644 scripts/migrate-longhorn-v2/host_recovery.go diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index f34c4e6e44..39c5b3a8a9 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -23,7 +23,11 @@ permanent worker with `reboot-longhorn-v2-host-`. It requires the loaded reservation, unused native huge pages, disabled V2, the complete fixed migration roster, all V1 volumes fully redundant, and all three database clusters healthy. It records the original node, hardware, boot, encrypted EPHEMERAL partition, storage disks and -database identities before an owned cordon and an eviction-based drain. After +database identities before an owned cordon and an eviction-based application drain. +The live `block-if-contains-last-replica` policy and target V1 instance-manager +ownership are verified on each health read. Only that ready storage Pod is kept +running during drain; a workload impersonating its selector label fails admission. +Application evictions keep the normal disruption-budget checks. After drain, the unchanged database quorum has up to 20 minutes to finish replacement startup. Changed identities, lost quorum or other safety evidence stop the wait; a timeout preserves the cordon without requesting a reboot. Two fresh checks of @@ -35,7 +39,12 @@ of a peer's recorded recovery prevents starting the next worker. Recovery never repeats an uncertain reboot. It requires a changed boot with the same node, hardware, OS, encrypted partition, registered disks and database system identities, 1,024 native 2 MiB pages, matching 2 GiB Kubernetes capacity, clear node -pressure and full V1 redundancy. Only its own cordon is released and read back. +pressure and two intact copies outside the target before restoring scheduling. +The verified new boot is journaled as `returning`, and only the owned cordon is +released so Longhorn can recreate its instance manager. The maintenance marker +and unfinished journal remain until full V1 redundancy and database health return; +a timeout at this stage leaves the node schedulable under that retained marker. +Only then is the marker removed and recovery recorded. Both release steps are read back. Dispatch each worker separately and verify platform recovery before the next one. An interrupted run keeps its journal and cordon; a new first-attempt main dispatch may adopt it only after the previous run has ended. This operation enables no V2 diff --git a/scripts/migrate-longhorn-v2/host_drain.go b/scripts/migrate-longhorn-v2/host_drain.go new file mode 100644 index 0000000000..2fd34ad112 --- /dev/null +++ b/scripts/migrate-longhorn-v2/host_drain.go @@ -0,0 +1,70 @@ +package main + +import "context" + +const hostApplicationDrainSelector = "--pod-selector=longhorn.io/component!=instance-manager" + +// drainRuntime proves that the selector excludes only the target's ready V1 +// instance-manager Pod. Longhorn 1.13 does not recreate this Pod while cordoned; +// retaining it until the journaled reboot preserves the three-copy precondition. +// Every application Pod remains subject to the normal eviction and PDB checks. +func (h hostMemoryReboot) drainRuntime(ctx context.Context, s inventory) error { + if err := h.drainPolicy(ctx); err != nil { + return err + } + return h.excludedRuntime(ctx, s) +} + +// drainPolicy binds every maintenance stage to the reviewed production policy. +func (h hostMemoryReboot) drainPolicy(ctx context.Context) error { + policy, err := h.c.get(ctx, "settings.longhorn.io", "longhorn-system", "node-drain-policy", false) + if err != nil || text(policy, "metadata", "name") != "node-drain-policy" || text(policy, "metadata", "namespace") != "longhorn-system" || text(policy, "value") != "block-if-contains-last-replica" { + return errHostMemory + } + return nil +} + +// excludedRuntime refuses an unrelated, replaced or unready Pod carrying the +// selector label, so application workloads cannot escape eviction by that label. +func (h hostMemoryReboot) excludedRuntime(ctx context.Context, s inventory) error { + managers, err := h.c.list(ctx, "instancemanagers.longhorn.io", "longhorn-system") + if err != nil { + return errHostMemory + } + byName, err := indexed(managers) + if err != nil { + return errHostMemory + } + var target object + for _, manager := range byName { + if text(manager, "spec", "nodeID") != h.target { + continue + } + if target != nil || text(manager, "spec", "dataEngine") != "v1" || text(manager, "status", "currentState") != "running" || text(manager, "metadata", "namespace") != "longhorn-system" || text(manager, "metadata", "uid") == "" || text(manager, "metadata", "resourceVersion") == "" || value(manager, "metadata", "deletionTimestamp") != nil { + return errHostMemory + } + target = manager + } + if target == nil { + return errHostMemory + } + excluded := 0 + for _, pod := range s.pods { + if text(pod, "spec", "nodeName") != h.target || text(pod, "metadata", "labels", "longhorn.io/component") != "instance-manager" { + continue + } + excluded++ + owners, ok := value(pod, "metadata", "ownerReferences").([]any) + if !ok || len(owners) != 1 || text(pod, "metadata", "namespace") != "longhorn-system" || text(pod, "metadata", "name") != text(target, "metadata", "name") || text(pod, "metadata", "uid") == "" || text(pod, "metadata", "resourceVersion") == "" || value(pod, "metadata", "deletionTimestamp") != nil || text(pod, "status", "phase") != "Running" || !ready(valueObject(pod, "status")) { + return errHostMemory + } + owner, ok := owners[0].(object) + if !ok || text(owner, "apiVersion") != "longhorn.io/v1beta2" || text(owner, "kind") != "InstanceManager" || text(owner, "name") != text(target, "metadata", "name") || text(owner, "uid") != text(target, "metadata", "uid") || value(owner, "controller") != true { + return errHostMemory + } + } + if excluded != 1 { + return errHostMemory + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/host_reboot.go b/scripts/migrate-longhorn-v2/host_reboot.go index 2c6796498e..a0afcc6751 100644 --- a/scripts/migrate-longhorn-v2/host_reboot.go +++ b/scripts/migrate-longhorn-v2/host_reboot.go @@ -8,7 +8,6 @@ import ( "io" "reflect" "regexp" - "strings" "time" ) @@ -77,7 +76,7 @@ func (r hostRebootRecord) validate(target string) error { if r.RecoveredBootID != "" { return errHostMemory } - case "recovered": + case "returning", "recovered": if !inventoryBoot.MatchString(r.RecoveredBootID) || r.RecoveredBootID == r.OriginalBootID { return errHostMemory } @@ -275,6 +274,9 @@ func (h hostMemoryReboot) healthy(ctx context.Context, expected map[string]hostD return inventory{}, nil, errHostMemory } } + if err = h.drainRuntime(ctx, s); err != nil { + return inventory{}, nil, err + } databases, err := h.databases(ctx, expected) return s, databases, err } @@ -415,7 +417,7 @@ func (h hostMemoryReboot) run(ctx context.Context) error { if _, err = h.owned(ctx, r); err != nil { return err } - if _, err = h.c.kubectl(ctx, nil, "drain", h.target, "--ignore-daemonsets", "--delete-emptydir-data", "--timeout=40m"); err != nil { + if _, err = h.c.kubectl(ctx, nil, "drain", h.target, "--ignore-daemonsets", "--delete-emptydir-data", "--timeout=40m", hostApplicationDrainSelector); err != nil { return errors.New("host memory maintenance drain was blocked; no reboot was requested") } if err = h.waitDrainRecovery(ctx, r); err != nil { @@ -448,46 +450,5 @@ func (h hostMemoryReboot) run(ctx context.Context) error { // is never retried: recovery must prove a new boot and intact storage. _, _ = h.c.talosCommand(ctx, 15*time.Minute, address, "reboot", "--wait", "--timeout=15m") } - recovery, cancel := context.WithTimeout(ctx, 20*time.Minute) - defer cancel() - var recovered object - if err = waitReadyWithin(recovery, 20*time.Minute, func(ctx context.Context) error { - n, _, err := h.targetState(ctx, r) - boot := text(n, "status", "nodeInfo", "bootID") - if err != nil || !inventoryBoot.MatchString(boot) || boot == r.OriginalBootID || r.RecoveredBootID != "" && boot != r.RecoveredBootID || !hugepagesReady(n) { - return errHostMemory - } - memory, err := h.memory(ctx) - if err != nil || memory.PersistentPages != 1024 || memory.TotalPages != 1024 || memory.PageSizeKiB != 2048 { - return errHostMemory - } - if _, _, err = h.healthy(ctx, r.Databases); err != nil { - return err - } - recovered = n - return nil - }); err != nil { - return err - } - if _, err = h.owned(ctx, r); err != nil { - return err - } - if text(recovered, "metadata", "annotations", fenceAnnotation) == h.name() { - if err = h.c.patch(ctx, "node", "", recovered, object{"op": "add", "path": "/spec/unschedulable", "value": false}, object{"op": "remove", "path": "/metadata/annotations/" + strings.ReplaceAll(fenceAnnotation, "/", "~1")}); err != nil { - return err - } - } - released, _, err := h.targetState(ctx, r) - if err != nil || value(released, "spec", "unschedulable") == true || text(released, "metadata", "annotations", fenceAnnotation) != "" || text(released, "status", "nodeInfo", "bootID") != text(recovered, "status", "nodeInfo", "bootID") { - return errHostMemory - } - if r.Phase == "recovered" { - return nil - } - cm, err = h.owned(ctx, r) - if err != nil { - return err - } - r.Phase, r.RecoveredBootID = "recovered", text(recovered, "status", "nodeInfo", "bootID") - return h.write(ctx, cm, r) + return h.recover(ctx, r) } diff --git a/scripts/migrate-longhorn-v2/host_reboot_test.go b/scripts/migrate-longhorn-v2/host_reboot_test.go index d35f3543f1..2b5e28ce82 100644 --- a/scripts/migrate-longhorn-v2/host_reboot_test.go +++ b/scripts/migrate-longhorn-v2/host_reboot_test.go @@ -21,7 +21,11 @@ type hostRebootFixture struct { drainError bool afterDrain, afterReboot func() afterDatabaseRead func() + afterScheduling func() previousLive bool + managers []object + drainPolicy string + evictUnfilteredRuntime bool } func hostRebootAPI(t *testing.T) (*hostRebootFixture, hostMemoryReboot) { @@ -52,7 +56,19 @@ func hostRebootAPI(t *testing.T) (*hostRebootFixture, hostMemoryReboot) { } b.node["status"].(object)["capacity"] = object{"hugepages-2Mi": "0"} b.node["status"].(object)["allocatable"] = object{"hugepages-2Mi": "0", "memory": "14980Mi"} - f := &hostRebootFixture{base: b, desired: "1024"} + f := &hostRebootFixture{base: b, desired: "1024", drainPolicy: "block-if-contains-last-replica"} + im := resource("instance-manager-host-fixture") + im["metadata"].(object)["namespace"] = "longhorn-system" + im["spec"] = object{"nodeID": "prod-worker-1", "dataEngine": "v1"} + im["status"] = object{"currentState": "running"} + f.managers = []object{im} + pod := resource(text(im, "metadata", "name")) + pod["metadata"].(object)["namespace"] = "longhorn-system" + pod["metadata"].(object)["labels"] = object{"longhorn.io/component": "instance-manager"} + pod["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "longhorn.io/v1beta2", "kind": "InstanceManager", "name": text(im, "metadata", "name"), "uid": text(im, "metadata", "uid"), "controller": true}} + pod["spec"] = object{"nodeName": "prod-worker-1"} + pod["status"] = object{"phase": "Running", "conditions": readyConditions()} + b.s.pods = append(b.s.pods, pod) for i, selected := range []struct{ namespace, name string }{{"observability", "coroot-db"}, {"umami", "umami-db"}, {"wedding-app", "wedding-db"}} { db := resource(selected.name) db["metadata"].(object)["namespace"] = selected.namespace @@ -118,6 +134,15 @@ func (f *hostRebootFixture) run(ctx context.Context, input []byte, program strin return json.Marshal(object{"status": status, "head_sha": text(record, "source"), "head_branch": "main", "event": "workflow_dispatch", "run_attempt": 1, "path": ".github/workflows/prepare-longhorn-v2-node.yaml"}) } if program == "kubectl" && len(args) > 3 { + if len(args) > 4 && args[2] == "get" && args[3] == "settings.longhorn.io" && args[4] == "node-drain-policy" { + setting := resource("node-drain-policy") + setting["metadata"].(object)["namespace"] = "longhorn-system" + setting["value"] = f.drainPolicy + return json.Marshal(setting) + } + if args[2] == "get" && args[3] == "instancemanagers.longhorn.io" { + return json.Marshal(object{"items": f.managers}) + } if len(args) > 4 && args[2] == "get" && args[3] == "settings.longhorn.io" && args[4] == "v2-data-engine" { data, err := f.base.run(ctx, input, program, args...) if err != nil { @@ -147,6 +172,17 @@ func (f *hostRebootFixture) run(ctx context.Context, input []byte, program strin } if args[2] == "drain" { f.drains++ + filtered := false + for _, arg := range args { + filtered = filtered || arg == "--pod-selector=longhorn.io/component!=instance-manager" + } + if f.evictUnfilteredRuntime && !filtered { + for _, replica := range f.base.s.replicas { + if text(replica, "spec", "nodeID") == "prod-worker-1" { + replica["status"].(object)["currentState"] = "stopped" + } + } + } if f.afterDrain != nil { f.afterDrain() } @@ -155,7 +191,118 @@ func (f *hostRebootFixture) run(ctx context.Context, input []byte, program strin } } } - return f.base.run(ctx, input, program, args...) + wasCordoned := value(f.base.node, "spec", "unschedulable") == true + output, err := f.base.run(ctx, input, program, args...) + if err == nil && program == "kubectl" && len(args) > 3 && args[2] == "patch" && args[3] == "node" && f.rebooted && wasCordoned && value(f.base.node, "spec", "unschedulable") == false && f.afterScheduling != nil { + f.afterScheduling() + } + return output, err +} + +func TestHostRebootRestoresSchedulingBeforeWaitingForTheStorageController(t *testing.T) { + f, h := hostRebootAPI(t) + originalStates := map[string]string{} + for _, replica := range f.base.s.replicas { + originalStates[text(replica, "metadata", "name")] = text(replica, "status", "currentState") + } + f.afterReboot = func() { + f.managers[0]["status"].(object)["currentState"] = "starting" + for _, replica := range f.base.s.replicas { + if text(replica, "spec", "nodeID") == h.target { + replica["status"].(object)["currentState"] = "stopped" + } + } + } + f.afterScheduling = func() { + var record hostRebootRecord + if json.Unmarshal([]byte(text(f.base.cm, "data", "record")), &record) != nil || record.Phase != "returning" || record.RecoveredBootID == "" || text(f.base.node, "metadata", "annotations", fenceAnnotation) != h.name() { + t.Fatal("scheduling resumed without durable recovered-host proof and retained maintenance ownership") + } + f.managers[0]["status"].(object)["currentState"] = "running" + for _, replica := range f.base.s.replicas { + if text(replica, "spec", "nodeID") == h.target { + replica["status"].(object)["currentState"] = originalStates[text(replica, "metadata", "name")] + } + } + } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if err := h.run(ctx); err != nil || f.reboots != 1 { + t.Fatal("storage runtime could not restart after positively verified host recovery:", err) + } +} + +func TestHostRebootDoesNotResumeSchedulingWithoutTwoIntactPeerCopies(t *testing.T) { + f, h := hostRebootAPI(t) + f.afterReboot = func() { + for _, replica := range f.base.s.replicas { + if text(replica, "spec", "nodeID") == "prod-worker-2" { + replica["status"].(object)["currentState"] = "stopped" + } + } + } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if h.run(ctx) == nil || f.reboots != 1 || value(f.base.node, "spec", "unschedulable") != true { + t.Fatal("scheduling resumed without two intact peer copies") + } +} + +func TestReturningHostKeepsItsMarkerUntilFullRecoveryWithoutRepeatingTheReboot(t *testing.T) { + f, h := hostRebootAPI(t) + f.afterReboot = func() { f.managers[0]["status"].(object)["currentState"] = "starting" } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + err := h.run(ctx) + var record hostRebootRecord + if !errors.Is(err, context.DeadlineExceeded) || json.Unmarshal([]byte(text(f.base.cm, "data", "record")), &record) != nil || record.Phase != "returning" || f.reboots != 1 || value(f.base.node, "spec", "unschedulable") != false || text(f.base.node, "metadata", "annotations", fenceAnnotation) != h.name() { + t.Fatal("unfinished storage recovery lost its durable ownership or was accepted:", err) + } + f.managers[0]["status"].(object)["currentState"] = "running" + if err = h.run(t.Context()); err != nil || f.reboots != 1 || text(f.base.node, "metadata", "annotations", fenceAnnotation) != "" { + t.Fatal("resuming storage recovery repeated the reboot or failed to finish:", err) + } +} + +func TestHostRebootDrainsApplicationsWithoutEvictingProvenStorageRuntime(t *testing.T) { + f, h := hostRebootAPI(t) + f.evictUnfilteredRuntime = true + if err := h.run(t.Context()); err != nil || f.reboots != 1 { + t.Fatal("ordinary instance-manager eviction prevented the controlled reboot:", err) + } +} + +func TestHostRebootRefusesUnknownDrainPolicyOrExcludedWorkloads(t *testing.T) { + for name, change := range map[string]func(*hostRebootFixture){ + "unknown policy": func(f *hostRebootFixture) { f.drainPolicy = "always-allow" }, + "missing policy": func(f *hostRebootFixture) { f.drainPolicy = "" }, + "unproven controller": func(f *hostRebootFixture) { f.managers[0]["metadata"].(object)["uid"] = "replacement" }, + "missing runtime": func(f *hostRebootFixture) { f.managers = nil }, + "unready runtime": func(f *hostRebootFixture) { f.managers[0]["status"].(object)["currentState"] = "starting" }, + "unrelated excluded workload": func(f *hostRebootFixture) { + pod := resource("unrelated-workload") + pod["metadata"].(object)["namespace"] = "application" + pod["metadata"].(object)["labels"] = object{"longhorn.io/component": "instance-manager"} + pod["spec"] = object{"nodeName": "prod-worker-1"} + f.base.s.pods = append(f.base.s.pods, pod) + }, + } { + t.Run(name, func(t *testing.T) { + f, h := hostRebootAPI(t) + change(f) + if h.run(t.Context()) == nil || f.drains != 0 || f.reboots != 0 { + t.Fatal("unproven storage runtime or policy reached maintenance") + } + }) + } +} + +func TestHostRebootRebindsTheExcludedRuntimeAfterDrain(t *testing.T) { + f, h := hostRebootAPI(t) + f.afterDrain = func() { f.managers[0]["metadata"].(object)["uid"] = "replacement" } + if h.run(t.Context()) == nil || f.drains != 1 || f.reboots != 0 { + t.Fatal("changed excluded runtime reached the reboot request") + } } func TestHostRebootRequiresPositiveRecoveryAndPreservesEveryDisk(t *testing.T) { @@ -353,7 +500,7 @@ func TestUncertainHostRebootCannotBeRepeated(t *testing.T) { } func TestHostRebootCannotStartWhileAnotherWorkerHasAnUnfinishedJournal(t *testing.T) { - for _, phase := range []string{"draining", "reboot-requested"} { + for _, phase := range []string{"draining", "reboot-requested", "returning"} { t.Run(phase, func(t *testing.T) { f, h := hostRebootAPI(t) if _, err := h.initialize(context.Background()); err != nil { @@ -367,6 +514,9 @@ func TestHostRebootCannotStartWhileAnotherWorkerHasAnUnfinishedJournal(t *testin r.Target, r.Phase = "prod-worker-2", phase r.NodeUID, r.ProviderID, r.MachineUUID = text(peer, "metadata", "uid"), text(peer, "spec", "providerID"), text(peer, "status", "nodeInfo", "systemUUID") r.OriginalBootID, r.LonghornUID, r.OriginalDiskUUID = text(peer, "status", "nodeInfo", "bootID"), text(storage, "metadata", "uid"), hostRegisteredDiskUUID(storage) + if phase == "returning" { + r.RecoveredBootID = "12345678-1234-4234-8234-000000000099" + } if r.validate("prod-worker-2") != nil { t.Fatal("peer fixture did not represent a valid pending request") } diff --git a/scripts/migrate-longhorn-v2/host_recovery.go b/scripts/migrate-longhorn-v2/host_recovery.go new file mode 100644 index 0000000000..5387a7a925 --- /dev/null +++ b/scripts/migrate-longhorn-v2/host_recovery.go @@ -0,0 +1,133 @@ +package main + +import ( + "context" + "errors" + "strings" + "time" +) + +// recoveredHost requires the original hardware, encrypted disk, OS and loaded +// reservation on a new boot, with native and Kubernetes huge-page capacity. +func (h hostMemoryReboot) recoveredHost(ctx context.Context, r hostRebootRecord) (object, error) { + n, _, err := h.targetState(ctx, r) + boot := text(n, "status", "nodeInfo", "bootID") + if err != nil || !inventoryBoot.MatchString(boot) || boot == r.OriginalBootID || r.RecoveredBootID != "" && boot != r.RecoveredBootID || !hugepagesReady(n) { + return nil, errHostMemory + } + memory, err := h.memory(ctx) + if err != nil || memory.PersistentPages != 1024 || memory.TotalPages != 1024 || memory.PageSizeKiB != 2048 { + return nil, errHostMemory + } + return n, nil +} + +// recoveryPeers proves the two independent live copies and unchanged database +// quorums before restoring scheduling. This is not the three-copy acceptance +// check: the returning journal continues to block the next worker. +func (h hostMemoryReboot) recoveryPeers(ctx context.Context, r hostRebootRecord) error { + setting, err := h.c.get(ctx, "settings.longhorn.io", "longhorn-system", "v2-data-engine", false) + if err != nil || text(setting, "metadata", "name") != "v2-data-engine" || text(setting, "metadata", "namespace") != "longhorn-system" || text(setting, "value") != "false" || h.drainPolicy(ctx) != nil { + return errHostMemory + } + s, err := h.c.snapshot(ctx) + if err != nil || verifyCapacityRoster(s.nodes, h.pools, false) != nil || verifyHealthyCopies(s, 2, h.target) != nil || h.peersClear(ctx, s.nodes) != nil { + return errHostMemory + } + for _, volume := range s.volumes { + if text(volume, "spec", "dataEngine") != "v1" { + return errHostMemory + } + } + _, err = h.databases(ctx, r.Databases) + if err != nil && !errors.Is(err, errHostDatabasesRecovering) { + return err + } + return nil +} + +// recover journals the verified new boot before returning the owned node to +// scheduling. Longhorn can then restart its instance manager. The marker and +// unfinished journal remain until full redundancy and database health return; +// neither a timeout nor a resumed invocation repeats the reboot. +func (h hostMemoryReboot) recover(ctx context.Context, r hostRebootRecord) error { + recovery, cancel := context.WithTimeout(ctx, 20*time.Minute) + defer cancel() + if r.Phase == "reboot-requested" { + var host object + if err := waitReadyWithin(recovery, 20*time.Minute, func(ctx context.Context) error { + if _, err := h.owned(ctx, r); err != nil { + return errors.Join(errReadinessTerminal, err) + } + var err error + host, err = h.recoveredHost(ctx, r) + if err != nil { + return err + } + return h.recoveryPeers(ctx, r) + }); err != nil { + return err + } + cm, err := h.owned(recovery, r) + if err != nil { + return err + } + r.Phase, r.RecoveredBootID = "returning", text(host, "status", "nodeInfo", "bootID") + if err = h.write(recovery, cm, r); err != nil { + return err + } + } + host, err := h.recoveredHost(recovery, r) + if err != nil || h.recoveryPeers(recovery, r) != nil { + return errHostMemory + } + if value(host, "spec", "unschedulable") == true { + if r.Phase != "returning" || text(host, "metadata", "annotations", fenceAnnotation) != h.name() { + return errHostMemory + } + if _, err = h.owned(recovery, r); err != nil { + return err + } + if err = h.c.patch(recovery, "node", "", host, object{"op": "add", "path": "/spec/unschedulable", "value": false}); err != nil { + return err + } + } + if err = waitReadyWithin(recovery, 20*time.Minute, func(ctx context.Context) error { + if _, err := h.owned(ctx, r); err != nil { + return errors.Join(errReadinessTerminal, err) + } + var err error + host, err = h.recoveredHost(ctx, r) + if err != nil || value(host, "spec", "unschedulable") == true { + return errHostMemory + } + s, _, err := h.healthy(ctx, r.Databases) + if err != nil { + return err + } + return h.peersClear(ctx, s.nodes) + }); err != nil { + return err + } + if text(host, "metadata", "annotations", fenceAnnotation) == h.name() { + if _, err = h.owned(recovery, r); err != nil { + return err + } + if err = h.c.patch(recovery, "node", "", host, object{"op": "remove", "path": "/metadata/annotations/" + strings.ReplaceAll(fenceAnnotation, "/", "~1")}); err != nil { + return err + } + } + released, err := h.recoveredHost(recovery, r) + if err != nil || value(released, "spec", "unschedulable") == true || text(released, "metadata", "annotations", fenceAnnotation) != "" { + return errHostMemory + } + if r.Phase == "recovered" { + return nil + } + cm, err := h.owned(recovery, r) + if err != nil { + return err + } + r.Phase = "recovered" + return h.write(recovery, cm, r) +} diff --git a/scripts/migrate-longhorn-v2/preflight.go b/scripts/migrate-longhorn-v2/preflight.go index 656d246150..2c8ecc49a8 100644 --- a/scripts/migrate-longhorn-v2/preflight.go +++ b/scripts/migrate-longhorn-v2/preflight.go @@ -116,6 +116,18 @@ func verifyEvacuated(target string, s inventory) error { } func verifyRedundancy(s inventory) error { + return verifyHealthyCopies(s, 3, "") +} + +// verifyHealthyCopies preserves the normal three-copy floor. Its only other +// profile proves two copies outside the single rebooted worker before returning +// that verified host to scheduling; it never establishes maintenance completion. +func verifyHealthyCopies(s inventory, required int, excludedNode string) error { + fullRedundancy := required == 3 && excludedNode == "" + returningHost := required == 2 && inventoryTarget.MatchString(excludedNode) + if !fullRedundancy && !returningHost { + return errors.New("unsupported storage redundancy profile") + } if len(s.volumes) == 0 { return errors.New("complete volume inventory is required") } @@ -170,6 +182,9 @@ func verifyRedundancy(s inventory) error { continue } node := text(r, "spec", "nodeID") + if node == excludedNode { + continue + } nodeStatus, _ := value(nodes[node], "status").(map[string]any) if !ready(nodeStatus) || !diskReady(storage[node], r) { continue @@ -181,8 +196,8 @@ func verifyRedundancy(s inventory) error { } copies[node] = true } - if len(copies) < 3 { - return errors.New("three healthy copies on distinct other nodes are not proven for every volume") + if len(copies) < required { + return errors.New("the required healthy copies on distinct eligible nodes are not proven for every volume") } } return nil diff --git a/scripts/migrate-longhorn-v2/temporary_prepare.go b/scripts/migrate-longhorn-v2/temporary_prepare.go index 1fef3e4418..0a68994ce7 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "maps" + "strings" "github.com/devantler-tech/platform/internal/longhorninventory" ) @@ -58,7 +59,7 @@ func (p *temporaryPreparation) prepare(ctx context.Context) error { } fence := text(n, "metadata", "annotations", fenceAnnotation) if fence == p.name && value(n, "spec", "unschedulable") == true { - if err = p.mutate(ctx, "node", "", n, object{"op": "add", "path": "/spec/unschedulable", "value": false}, object{"op": "remove", "path": "/metadata/annotations/storage.platform.devantler.tech~1longhorn-v2-preparation"}); err != nil { + if err = p.mutate(ctx, "node", "", n, object{"op": "add", "path": "/spec/unschedulable", "value": false}, object{"op": "remove", "path": "/metadata/annotations/" + strings.ReplaceAll(fenceAnnotation, "/", "~1")}); err != nil { return err } } else if fence != "" || value(n, "spec", "unschedulable") == true { From e99be98b4eb5c8f4b1d68da3c25aa1a0b3b26946 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 08:42:23 +0100 Subject: [PATCH 20/35] fix(storage): admit verified recovery states in platform gates --- scripts/migrate-longhorn-v2/README.md | 2 + .../migrate-longhorn-v2/host_reboot_test.go | 33 +++++++ scripts/migrate-longhorn-v2/preflight.go | 3 +- .../tests/test-talos-schematic-readback.sh | 88 ++++++++++++++++++- scripts/verify-talos-schematic-readback.sh | 66 +++++++++++--- 5 files changed, 180 insertions(+), 12 deletions(-) diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 39c5b3a8a9..0922e47a37 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -40,6 +40,8 @@ Recovery never repeats an uncertain reboot. It requires a changed boot with the same node, hardware, OS, encrypted partition, registered disks and database system identities, 1,024 native 2 MiB pages, matching 2 GiB Kubernetes capacity, clear node pressure and two intact copies outside the target before restoring scheduling. +That intermediate check permits a degraded attached volume only with two live +read-write peer replicas; final acceptance still requires healthy volumes and all three copies. The verified new boot is journaled as `returning`, and only the owned cordon is released so Longhorn can recreate its instance manager. The maintenance marker and unfinished journal remain until full V1 redundancy and database health return; diff --git a/scripts/migrate-longhorn-v2/host_reboot_test.go b/scripts/migrate-longhorn-v2/host_reboot_test.go index 2b5e28ce82..41dbcc11ae 100644 --- a/scripts/migrate-longhorn-v2/host_reboot_test.go +++ b/scripts/migrate-longhorn-v2/host_reboot_test.go @@ -207,6 +207,11 @@ func TestHostRebootRestoresSchedulingBeforeWaitingForTheStorageController(t *tes } f.afterReboot = func() { f.managers[0]["status"].(object)["currentState"] = "starting" + for _, volume := range f.base.s.volumes { + if text(volume, "status", "state") == "attached" { + volume["status"].(object)["robustness"] = "degraded" + } + } for _, replica := range f.base.s.replicas { if text(replica, "spec", "nodeID") == h.target { replica["status"].(object)["currentState"] = "stopped" @@ -219,6 +224,11 @@ func TestHostRebootRestoresSchedulingBeforeWaitingForTheStorageController(t *tes t.Fatal("scheduling resumed without durable recovered-host proof and retained maintenance ownership") } f.managers[0]["status"].(object)["currentState"] = "running" + for _, volume := range f.base.s.volumes { + if text(volume, "status", "state") == "attached" { + volume["status"].(object)["robustness"] = "healthy" + } + } for _, replica := range f.base.s.replicas { if text(replica, "spec", "nodeID") == h.target { replica["status"].(object)["currentState"] = originalStates[text(replica, "metadata", "name")] @@ -248,6 +258,29 @@ func TestHostRebootDoesNotResumeSchedulingWithoutTwoIntactPeerCopies(t *testing. } } +func TestReturningHostStillRequiresTwoLivePeersAndFinalHealthyVolumes(t *testing.T) { + f, h := hostRebootAPI(t) + for _, volume := range f.base.s.volumes { + if text(volume, "status", "state") == "attached" { + volume["status"].(object)["robustness"] = "degraded" + } + } + if err := verifyHealthyCopies(f.base.s, 2, h.target); err != nil { + t.Fatal("two proven peer copies were rejected during return to scheduling:", err) + } + if verifyRedundancy(f.base.s) == nil { + t.Fatal("degraded volumes satisfied the final full-redundancy acceptance gate") + } + for _, replica := range f.base.s.replicas { + if text(replica, "spec", "nodeID") == "prod-worker-2" { + replica["status"].(object)["currentState"] = "stopped" + } + } + if verifyHealthyCopies(f.base.s, 2, h.target) == nil { + t.Fatal("the degraded label substituted for an intact live peer replica") + } +} + func TestReturningHostKeepsItsMarkerUntilFullRecoveryWithoutRepeatingTheReboot(t *testing.T) { f, h := hostRebootAPI(t) f.afterReboot = func() { f.managers[0]["status"].(object)["currentState"] = "starting" } diff --git a/scripts/migrate-longhorn-v2/preflight.go b/scripts/migrate-longhorn-v2/preflight.go index 2c8ecc49a8..36b88ad2c6 100644 --- a/scripts/migrate-longhorn-v2/preflight.go +++ b/scripts/migrate-longhorn-v2/preflight.go @@ -150,7 +150,8 @@ func verifyHealthyCopies(s inventory, required int, excludedNode string) error { return errors.New("volume engine or required redundancy is outside the migration contract") } state, health := text(v, "status", "state"), text(v, "status", "robustness") - suitable := state == "attached" && health == "healthy" || state == "detached" && (health == "healthy" || health == "unknown") + attached := state == "attached" && (health == "healthy" || returningHost && health == "degraded") + suitable := attached || state == "detached" && (health == "healthy" || health == "unknown") if !suitable { return errors.New("volume health is not suitable for node maintenance") } diff --git a/scripts/tests/test-talos-schematic-readback.sh b/scripts/tests/test-talos-schematic-readback.sh index 53facaae61..06a42e4952 100755 --- a/scripts/tests/test-talos-schematic-readback.sh +++ b/scripts/tests/test-talos-schematic-readback.sh @@ -39,7 +39,7 @@ mock_kubectl="$tmp_dir/kubectl" cat >"$mock_kubectl" <<'MOCK' #!/usr/bin/env bash set -euo pipefail -if [[ "$*" == "--context admin@prod --request-timeout=15s get configmap longhorn-v2-prepare-prod-worker-1 --namespace=longhorn-system -o json" ]]; then +if [[ "$*" == "--context admin@prod --request-timeout=15s get configmap ${MOCK_JOURNAL_NAME:-longhorn-v2-prepare-prod-worker-1} --namespace=longhorn-system -o json" ]]; then [[ -n "${MOCK_JOURNAL_FILE:-}" ]] || exit 1 cat "$MOCK_JOURNAL_FILE" exit 0 @@ -189,4 +189,90 @@ jq '(.items[] | select(.metadata.name == "prod-control-plane-2")) |= (.spec.unsc assert_rejected 'a cordoned non-worker carrying a preparation annotation' unset MOCK_JOURNAL_FILE +# New maintenance profiles retain their own cordons if native recovery is held. +# Their exact annotations, journal scopes and physical identities must match. +for profile in host temporary; do + worker=prod-worker-1 + journal_name=longhorn-v2-host-memory-prod-worker-1 + if [[ "$profile" == temporary ]]; then + worker=autoscale-longhorn-v1-0123456789abcdef + journal_name=longhorn-v2-prepare-temporary-v1-1 + fi + base_nodes=$(jq --arg worker "$worker" --arg journal "$journal_name" ' + (.items[] | select(.metadata.name == $worker)) |= ( + .spec.unschedulable = true | .metadata.uid = "held-worker-uid" | + .spec.providerID = "hcloud://123" | + .status.nodeInfo.systemUUID = "held-hardware-uuid" | + .status.nodeInfo.bootID = "12345678-1234-4234-8234-000000000001" | + .metadata.annotations["storage.platform.devantler.tech/longhorn-v2-preparation"] = $journal) + ' "$tmp_dir/nodes.json") + base_record=$(jq -n --arg worker "$worker" --arg journal "$journal_name" --arg profile "$profile" '{ + metadata: {name: $journal, namespace: "longhorn-system", uid: "journal-uid", resourceVersion: "1"}, + data: {record: ({target: $worker, nodeUID: "held-worker-uid", providerID: "hcloud://123", machineUUID: "held-hardware-uuid", + phase: "draining", source: ("a" * 40), ownerRun: "123", originalBootID: "12345678-1234-4234-8234-000000000001"} + + if $profile == "host" then {version: 1, recoveredBootID: ""} + else {schema: 2, temporarySelection: "temporary-v1-1", rawVolumeAbsent: true, nativeEmptyDigest: ("b" * 64), originalDisks: {}, ephemeralUUID: "original-ephemeral"} end | tojson)} + }') + printf '%s\n' "$base_nodes" >"$tmp_dir/changed.json" + printf '%s\n' "$base_record" >"$tmp_dir/journal.json" + export MOCK_JOURNAL_FILE="$tmp_dir/journal.json" MOCK_JOURNAL_NAME="$journal_name" + if ! run_readback "$tmp_dir/changed.json" >"$tmp_dir/held.out"; then + printf 'readback refused an identity-bound %s maintenance cordon\n' "$profile" >&2 + exit 1 + fi + grep -Fq "HELD: $worker" "$tmp_dir/held.out" + for phase in reboot-requested returning; do + if [[ "$profile" == temporary ]]; then + phase=reset-requested + fi + phase_nodes=$base_nodes + phase_record=$(jq --arg phase "$phase" '.data.record |= (fromjson | .phase = $phase | tojson)' <<<"$base_record") + if [[ "$phase" == returning || "$profile" == temporary ]]; then + phase_nodes=$(jq --arg worker "$worker" '(.items[] | select(.metadata.name == $worker)).status.nodeInfo.bootID = "12345678-1234-4234-8234-000000000099"' <<<"$base_nodes") + fi + if [[ "$phase" == returning ]]; then + phase_record=$(jq '.data.record |= (fromjson | .recoveredBootID = "12345678-1234-4234-8234-000000000099" | tojson)' <<<"$phase_record") + fi + printf '%s\n' "$phase_nodes" >"$tmp_dir/changed.json" + printf '%s\n' "$phase_record" >"$tmp_dir/journal.json" + run_readback "$tmp_dir/changed.json" >"$tmp_dir/held.out" + [[ "$profile" != temporary ]] || break + done + printf '%s\n' "$base_nodes" >"$tmp_dir/changed.json" + for mutation in \ + '.metadata.name = "foreign"' \ + '.metadata.namespace = "foreign"' \ + '.data.record |= (fromjson | .nodeUID = "replacement" | tojson)' \ + '.data.record |= (fromjson | .providerID = "hcloud://999" | tojson)' \ + '.data.record |= (fromjson | .machineUUID = "replacement" | tojson)' \ + '.data.record |= (fromjson | .source = "unverified" | tojson)' \ + '.data.record |= (fromjson | .ownerRun = "" | tojson)' \ + '.data.record |= (fromjson | .originalBootID = "12345678-1234-4234-8234-000000000099" | tojson)' \ + '.data.record |= (fromjson | .phase = "recovered" | tojson)'; do + jq "$mutation" <<<"$base_record" >"$tmp_dir/journal.json" + assert_rejected "a foreign or completed $profile journal: $mutation" + done + if [[ "$profile" == temporary ]]; then + for mutation in '.rawVolumeAbsent = false' '.temporarySelection = "temporary-v1-2"' '.schema = 1' '.nativeEmptyDigest = ""' '.phase = "prepared"'; do + jq ".data.record |= (fromjson | $mutation | tojson)" <<<"$base_record" >"$tmp_dir/journal.json" + assert_rejected "an invalid temporary preparation proof: $mutation" + done + second_worker=autoscale-longhorn-v1-fedcba9876543210 + second_journal=longhorn-v2-prepare-temporary-v1-2 + jq --arg first "$worker" --arg second "$second_worker" --arg journal "$second_journal" ' + (.items[].metadata.name) |= (if . == $first then $second elif . == $second then $first else . end) | + (.items[] | select(.metadata.name == $second)).metadata.annotations["storage.platform.devantler.tech/longhorn-v2-preparation"] = $journal + ' <<<"$base_nodes" >"$tmp_dir/changed.json" + jq --arg worker "$second_worker" --arg journal "$second_journal" ' + .metadata.name = $journal | .data.record |= (fromjson | .target = $worker | .temporarySelection = "temporary-v1-2" | tojson) + ' <<<"$base_record" >"$tmp_dir/journal.json" + MOCK_JOURNAL_NAME="$second_journal" run_readback "$tmp_dir/changed.json" >"$tmp_dir/held.out" + fi + printf '%s\n' "$base_record" >"$tmp_dir/journal.json" + jq --arg worker "$worker" '(.items[] | select(.metadata.name == $worker)).status.conditions[0].status = "False"' \ + <<<"$base_nodes" >"$tmp_dir/changed.json" + assert_rejected "an unready $profile worker even with an owned journal" + unset MOCK_JOURNAL_FILE MOCK_JOURNAL_NAME +done + printf 'PASS: production Talos version, schematic, and node health are read back fail-closed\n' diff --git a/scripts/verify-talos-schematic-readback.sh b/scripts/verify-talos-schematic-readback.sh index b45fa83ab7..dedcb70095 100755 --- a/scripts/verify-talos-schematic-readback.sh +++ b/scripts/verify-talos-schematic-readback.sh @@ -59,25 +59,71 @@ jq -e 'all(.[]; (.name | test("^autoscale-longhorn-v[12]$")) and exit 1 } -# A failed worker preparation deliberately keeps its cordon and journal for +# A held worker preparation or host reboot keeps its cordon and journal for # explicit recovery. Accept exactly that cordon, so a deploy that may carry the # recovery fix is not evicted by it. Any other cordon still fails this gate. journal_held_worker() { - local nodes_json=$1 candidates worker journal_name record + local nodes_json=$1 candidates worker journal_name record annotation profile selection='' members candidates=$(jq -c '[.items[] | select(.spec.unschedulable == true)]' <<<"$nodes_json") || return 0 [[ $(jq 'length' <<<"$candidates") == 1 ]] || return 0 worker=$(jq -r '.[0].metadata.name' <<<"$candidates") - [[ "$worker" =~ ^prod-worker-[1-9][0-9]*$ ]] || return 0 - journal_name="longhorn-v2-prepare-$worker" - jq -e --arg journal "$journal_name" '.[0].metadata.annotations["storage.platform.devantler.tech/longhorn-v2-preparation"] == $journal' \ - <<<"$candidates" >/dev/null || return 0 + annotation=$(jq -r '.[0].metadata.annotations["storage.platform.devantler.tech/longhorn-v2-preparation"] // ""' <<<"$candidates") + if [[ "$worker" =~ ^prod-worker-[1-9][0-9]*$ && "$annotation" == "longhorn-v2-prepare-$worker" ]]; then + profile=permanent + journal_name="longhorn-v2-prepare-$worker" + elif [[ "$worker" =~ ^prod-worker-[123]$ && "$annotation" == "longhorn-v2-host-memory-$worker" ]]; then + profile=host + journal_name="longhorn-v2-host-memory-$worker" + elif [[ "$worker" =~ ^autoscale-longhorn-v1-[0-9a-f]{16}$ ]]; then + members=$(jq -c '[.items[] | select(.metadata.name | test("^autoscale-longhorn-v1-[0-9a-f]{16}$")) | .metadata.name] | sort' <<<"$nodes_json") || return 0 + [[ $(jq 'length' <<<"$members") == 2 ]] || return 0 + if [[ $(jq -r '.[0]' <<<"$members") == "$worker" ]]; then + selection=temporary-v1-1 + elif [[ $(jq -r '.[1]' <<<"$members") == "$worker" ]]; then + selection=temporary-v1-2 + else + return 0 + fi + profile=temporary + journal_name="longhorn-v2-prepare-$selection" + [[ "$annotation" == "$journal_name" ]] || return 0 + else + return 0 + fi record=$("$kubectl_bin" --context "$kube_context" --request-timeout=15s \ get configmap "$journal_name" --namespace=longhorn-system -o json 2>/dev/null) || return 0 - jq -e --arg worker "$worker" --argjson candidates "$candidates" ' + jq -e --arg worker "$worker" --arg profile "$profile" --arg selection "$selection" \ + --arg journal_name "$journal_name" --argjson candidates "$candidates" ' (.data.record | fromjson) as $journal | - $journal.schema == 1 and $journal.target == $worker and + $journal.target == $worker and $journal.nodeUID == $candidates[0].metadata.uid and - ($journal.phase == "evacuating" or $journal.phase == "draining" or $journal.phase == "reset-requested") + if $profile == "permanent" then + $journal.schema == 1 and + ($journal.phase == "evacuating" or $journal.phase == "draining" or $journal.phase == "reset-requested") + else + .metadata.name == $journal_name and .metadata.namespace == "longhorn-system" and + (.metadata.uid | type == "string" and length > 0) and + (.metadata.resourceVersion | type == "string" and length > 0) and + ($journal.nodeUID | type == "string" and length > 0) and + ($journal.providerID | test("^hcloud://[1-9][0-9]*$")) and + $journal.providerID == $candidates[0].spec.providerID and + ($journal.machineUUID | type == "string" and length > 0) and + $journal.machineUUID == $candidates[0].status.nodeInfo.systemUUID and + ($journal.source | test("^[0-9a-f]{40}$")) and ($journal.ownerRun | test("^[1-9][0-9]*$")) and + ($journal.originalBootID | test("^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$")) and + ($candidates[0].status.nodeInfo.bootID | test("^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$")) and + if $profile == "host" then + $journal.version == 1 and $journal.schema == null and + (($journal.phase == "draining" and $journal.originalBootID == $candidates[0].status.nodeInfo.bootID) or + $journal.phase == "reboot-requested" or + ($journal.phase == "returning" and $journal.recoveredBootID == $candidates[0].status.nodeInfo.bootID and $journal.recoveredBootID != $journal.originalBootID)) + else + $journal.schema == 2 and $journal.version == null and $journal.temporarySelection == $selection and + $journal.rawVolumeAbsent == true and ($journal.nativeEmptyDigest | test("^[0-9a-f]{64}$")) and + ($journal.originalDisks | type == "object" and length == 0) and ($journal.ephemeralUUID | type == "string" and length > 0) and + (($journal.phase == "evacuating" or $journal.phase == "draining") and $journal.originalBootID == $candidates[0].status.nodeInfo.bootID or $journal.phase == "reset-requested") + end + end ' <<<"$record" >/dev/null 2>&1 || return 0 printf '%s' "$worker" } @@ -122,7 +168,7 @@ for ((attempt = 1; attempt <= max_attempts; attempt++)); do printf 'PASS: all Talos nodes run %s with schematic %s and are Ready/schedulable\n' \ "$expected_version" "$expected_schematic" if [[ -n "$held_worker" ]]; then - printf '::warning::HELD: %s stays cordoned by its Longhorn V2 preparation journal; recover it before the next preparation\n' "$held_worker" + printf '::warning::HELD: %s stays cordoned by its Longhorn V2 maintenance journal; recover it before the next worker\n' "$held_worker" fi exit 0 fi From d66a8cbaecaee28b6d3aa90c99836f220926009e Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 09:17:13 +0100 Subject: [PATCH 21/35] fix(storage): wait for verified post-drain attachment transitions --- scripts/migrate-longhorn-v2/README.md | 7 +- scripts/migrate-longhorn-v2/host_reboot.go | 20 ++-- .../migrate-longhorn-v2/host_reboot_test.go | 91 +++++++++++++++++++ scripts/migrate-longhorn-v2/preflight.go | 41 ++++++++- 4 files changed, 148 insertions(+), 11 deletions(-) diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 0922e47a37..c61e969f5a 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -28,8 +28,11 @@ The live `block-if-contains-last-replica` policy and target V1 instance-manager ownership are verified on each health read. Only that ready storage Pod is kept running during drain; a workload impersonating its selector label fails admission. Application evictions keep the normal disruption-budget checks. After -drain, the unchanged database quorum has up to 20 minutes to finish replacement -startup. Changed identities, lost quorum or other safety evidence stop the wait; +drain, known attachment transitions and the unchanged database quorum share up +to 20 minutes to settle. Attaching or detaching volumes must still retain three +healthy copies on ready disks with known engine and replica states. Peer ownership +is checked during this wait. Changed identities, lost quorum, failed runtime +states or other safety evidence stop the wait; a timeout preserves the cordon without requesting a reboot. Two fresh checks of complete redundancy, databases and original storage still precede durable reboot intent. The single reboot request has no reset or disk-wipe selector. diff --git a/scripts/migrate-longhorn-v2/host_reboot.go b/scripts/migrate-longhorn-v2/host_reboot.go index a0afcc6751..17a7f1666b 100644 --- a/scripts/migrate-longhorn-v2/host_reboot.go +++ b/scripts/migrate-longhorn-v2/host_reboot.go @@ -232,9 +232,10 @@ func (h hostMemoryReboot) databases(ctx context.Context, expected map[string]hos return result, nil } -// waitDrainRecovery gives an unchanged database quorum a bounded startup window -// after eviction. Changed ownership, source, target identity, peer maintenance or -// other safety evidence fails immediately; the final two fresh checks still run. +// waitDrainRecovery bounds ordinary attachment and database startup after +// eviction. Attachment transitions require three preserved healthy copies and +// known runtime states. Changed identities, failed evidence or peer maintenance +// fail immediately; the final two fresh checks still run. func (h hostMemoryReboot) waitDrainRecovery(ctx context.Context, r hostRebootRecord) error { return waitReadyWithin(ctx, 20*time.Minute, func(ctx context.Context) error { if _, err := h.owned(ctx, r); err != nil { @@ -245,7 +246,7 @@ func (h hostMemoryReboot) waitDrainRecovery(ctx context.Context, r hostRebootRec return errors.Join(errReadinessTerminal, errHostMemory) } s, _, err := h.healthy(ctx, r.Databases) - if err != nil && !errors.Is(err, errHostDatabasesRecovering) { + if err != nil && !errors.Is(err, errHostDatabasesRecovering) && !errors.Is(err, errHostStorageTransition) { return errors.Join(errReadinessTerminal, err) } if peerErr := h.peersClear(ctx, s.nodes); peerErr != nil { @@ -266,7 +267,11 @@ func (h hostMemoryReboot) healthy(ctx context.Context, expected map[string]hostD if err != nil { return inventory{}, nil, err } - if verifyCapacityRoster(s.nodes, h.pools, false) != nil || verifyRedundancy(s) != nil { + if verifyCapacityRoster(s.nodes, h.pools, false) != nil { + return inventory{}, nil, errHostMemory + } + storageErr := verifyHostDrainRedundancy(s) + if storageErr != nil && !errors.Is(storageErr, errHostStorageTransition) { return inventory{}, nil, errHostMemory } for _, v := range s.volumes { @@ -278,7 +283,10 @@ func (h hostMemoryReboot) healthy(ctx context.Context, expected map[string]hostD return inventory{}, nil, err } databases, err := h.databases(ctx, expected) - return s, databases, err + if err != nil && !errors.Is(err, errHostDatabasesRecovering) { + return inventory{}, nil, err + } + return s, databases, errors.Join(storageErr, err) } // targetState binds the live endpoint to the original hardware, registered disk diff --git a/scripts/migrate-longhorn-v2/host_reboot_test.go b/scripts/migrate-longhorn-v2/host_reboot_test.go index 41dbcc11ae..23f6b4374c 100644 --- a/scripts/migrate-longhorn-v2/host_reboot_test.go +++ b/scripts/migrate-longhorn-v2/host_reboot_test.go @@ -380,6 +380,97 @@ func TestHostRebootWaitsForTheOriginalDatabaseQuorumAfterDrain(t *testing.T) { } } +func setHostDrainAttachmentTransition(f *hostRebootFixture, state string) func() { + volume := f.base.s.volumes[0] + name := text(volume, "metadata", "name") + volume["status"].(object)["state"] = state + volume["status"].(object)["robustness"] = "unknown" + var engine object + for _, candidate := range f.base.s.engines { + if text(candidate, "spec", "volumeName") == name { + engine = candidate + break + } + } + if engine == nil { + f.base.t.Fatal("fixture has no active engine") + } + modes := value(engine, "status", "replicaModeMap") + engine["status"].(object)["currentState"] = "starting" + if state == "detaching" { + engine["status"].(object)["currentState"] = "stopping" + } + engine["status"].(object)["replicaModeMap"] = object{} + return func() { + volume["status"].(object)["state"] = "attached" + volume["status"].(object)["robustness"] = "healthy" + engine["status"].(object)["currentState"] = "running" + engine["status"].(object)["replicaModeMap"] = modes + } +} + +func TestHostRebootWaitsForAProvenPostDrainAttachmentTransition(t *testing.T) { + for _, state := range []string{"attaching", "detaching"} { + t.Run(state, func(t *testing.T) { + f, h := hostRebootAPI(t) + reads := 0 + f.afterDrain = func() { + restore := setHostDrainAttachmentTransition(f, state) + f.afterDatabaseRead = func() { + reads++ + if reads == 2 { + restore() + } + } + } + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + if err := h.run(ctx); err != nil || reads < 2 || f.reboots != 1 { + t.Fatal("normal asynchronous attachment prevented the guarded reboot:", err) + } + }) + } +} + +func TestHostDrainTransitionDoesNotHideOtherStorageOrPeerFailures(t *testing.T) { + for _, mode := range []string{"lost other copy", "engine error", "replica error", "replica mode error", "faulted volume", "unreadable peer"} { + t.Run(mode, func(t *testing.T) { + f, h := hostRebootAPI(t) + reads := 0 + f.afterDatabaseRead = func() { reads++ } + f.afterDrain = func() { + setHostDrainAttachmentTransition(f, "attaching") + switch mode { + case "lost other copy": + other := text(f.base.s.volumes[1], "metadata", "name") + for _, replica := range f.base.s.replicas { + if text(replica, "spec", "volumeName") == other { + replica["spec"].(object)["healthyAt"] = "" + break + } + } + case "engine error": + f.base.s.engines[0]["status"].(object)["currentState"] = "error" + case "replica error": + f.base.s.replicas[0]["status"].(object)["currentState"] = "error" + case "replica mode error": + f.base.s.engines[0]["status"].(object)["replicaModeMap"] = object{text(f.base.s.replicas[0], "metadata", "name"): "ERR"} + case "faulted volume": + f.base.s.volumes[0]["status"].(object)["robustness"] = "faulted" + case "unreadable peer": + f.peerJournal = resource("longhorn-v2-host-memory-prod-worker-2") + } + } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + err := h.run(ctx) + if !errors.Is(err, errReadinessTerminal) || f.reboots != 0 || (mode == "unreadable peer" && reads != 1) || (mode != "unreadable peer" && reads != 0) { + t.Fatal("attachment retry hid a different failed safety check:", err) + } + }) + } +} + func TestHostRebootPreservesItsCordonWhenDrainRecoveryTimesOut(t *testing.T) { f, h := hostRebootAPI(t) f.afterDrain = func() { f.databases[0]["status"].(object)["readyInstances"] = float64(2) } diff --git a/scripts/migrate-longhorn-v2/preflight.go b/scripts/migrate-longhorn-v2/preflight.go index 36b88ad2c6..4d37f370cf 100644 --- a/scripts/migrate-longhorn-v2/preflight.go +++ b/scripts/migrate-longhorn-v2/preflight.go @@ -123,9 +123,31 @@ func verifyRedundancy(s inventory) error { // profile proves two copies outside the single rebooted worker before returning // that verified host to scheduling; it never establishes maintenance completion. func verifyHealthyCopies(s inventory, required int, excludedNode string) error { + return verifyStorageCopies(s, required, excludedNode, false) +} + +var errHostStorageTransition = errors.New("post-drain volume attachment is still settling") + +// verifyHostDrainRedundancy distinguishes an ordinary attachment transition +// from failed evidence without allowing that transition to satisfy readiness. +func verifyHostDrainRedundancy(s inventory) error { + return verifyStorageCopies(s, 3, "", true) +} + +// transitionalInstanceState accepts only known startup/shutdown states while +// preserving existing healthy-copy evidence; error and unknown remain failures. +func transitionalInstanceState(state string) bool { + return state == "running" || state == "stopped" || state == "starting" || state == "stopping" +} + +// verifyStorageCopies checks every volume before reporting a retryable drain +// transition. Transitional volumes still need three distinct healthy recorded +// copies on ready disks, known engine/replica states, and no failed replica mode. +// Final readiness always uses the stable-state profile through verifyHealthyCopies. +func verifyStorageCopies(s inventory, required int, excludedNode string, draining bool) error { fullRedundancy := required == 3 && excludedNode == "" returningHost := required == 2 && inventoryTarget.MatchString(excludedNode) - if !fullRedundancy && !returningHost { + if !fullRedundancy && !returningHost || draining && !fullRedundancy { return errors.New("unsupported storage redundancy profile") } if len(s.volumes) == 0 { @@ -144,14 +166,16 @@ func verifyHealthyCopies(s inventory, required int, excludedNode string) error { return err } } + transitioning := false for _, v := range s.volumes { engine := text(v, "spec", "dataEngine") if engine != "v1" && engine != "v2" || value(v, "spec", "numberOfReplicas") != float64(3) { return errors.New("volume engine or required redundancy is outside the migration contract") } state, health := text(v, "status", "state"), text(v, "status", "robustness") + transition := draining && (state == "attaching" || state == "detaching") attached := state == "attached" && (health == "healthy" || returningHost && health == "degraded") - suitable := attached || state == "detached" && (health == "healthy" || health == "unknown") + suitable := attached || state == "detached" && (health == "healthy" || health == "unknown") || transition && (health == "healthy" || health == "unknown" || health == "degraded") if !suitable { return errors.New("volume health is not suitable for node maintenance") } @@ -166,7 +190,8 @@ func verifyHealthyCopies(s inventory, required int, excludedNode string) error { } if currentEngine == nil || text(currentEngine, "spec", "dataEngine") != engine || state == "attached" && text(currentEngine, "status", "currentState") != "running" || - state == "detached" && text(currentEngine, "status", "currentState") != "stopped" { + state == "detached" && text(currentEngine, "status", "currentState") != "stopped" || + transition && !transitionalInstanceState(text(currentEngine, "status", "currentState")) { return errors.New("the volume's current engine identity and state are not proven") } modes, _ := value(currentEngine, "status", "replicaModeMap").(map[string]any) @@ -195,11 +220,21 @@ func verifyHealthyCopies(s inventory, required int, excludedNode string) error { state == "detached" && current != "stopped" { continue } + if transition { + mode, present := modes[text(r, "metadata", "name")] + if !transitionalInstanceState(current) || present && mode != "RW" && mode != "WO" { + continue + } + } copies[node] = true } if len(copies) < required { return errors.New("the required healthy copies on distinct eligible nodes are not proven for every volume") } + transitioning = transitioning || transition + } + if transitioning { + return errHostStorageTransition } return nil } From 93005ad1b783d14b59b976fb8eefb265d100f121 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 10:10:24 +0100 Subject: [PATCH 22/35] feat(storage): qualify temporary V2 storage with recovery probes --- .../workflows/prepare-longhorn-v2-node.yaml | 26 +- .../controllers/longhorn/helm-release.yaml | 13 + .../controllers/longhorn/kustomization.yaml | 1 + .../controllers/longhorn/limit-range.yaml | 10 +- .../longhorn/storage-class-longhorn-v2.yaml | 19 + scripts/migrate-longhorn-v2/README.md | 62 ++- scripts/migrate-longhorn-v2/authorization.go | 13 + scripts/migrate-longhorn-v2/canary.go | 519 ++++++++++++++++++ scripts/migrate-longhorn-v2/canary_assets.go | 132 +++++ .../migrate-longhorn-v2/canary_assets_test.go | 96 ++++ scripts/migrate-longhorn-v2/canary_cleanup.go | 127 +++++ .../canary_cleanup_test.go | 35 ++ .../migrate-longhorn-v2/canary_cli_test.go | 47 ++ .../migrate-longhorn-v2/canary_csi_assets.go | 21 + .../canary_csi_assets_test.go | 32 ++ .../migrate-longhorn-v2/canary_csi_binding.go | 53 ++ .../canary_csi_binding_test.go | 53 ++ .../migrate-longhorn-v2/canary_csi_capture.go | 220 ++++++++ .../canary_csi_capture_test.go | 78 +++ .../canary_csi_native_test.go | 87 +++ .../migrate-longhorn-v2/canary_csi_runtime.go | 183 ++++++ .../canary_csi_runtime_test.go | 48 ++ .../canary_csi_transfer.go | 65 +++ .../canary_csi_transfer_test.go | 84 +++ .../canary_identity_test.go | 68 +++ .../migrate-longhorn-v2/canary_transfer.go | 59 ++ .../canary_transfer_test.go | 49 ++ scripts/migrate-longhorn-v2/main.go | 44 ++ .../temporary_activation.go | 136 +++++ ...temporary_activation_authorization_test.go | 52 ++ .../temporary_activation_flow_test.go | 276 ++++++++++ .../temporary_activation_native_test.go | 83 +++ .../temporary_activation_record.go | 75 +++ .../temporary_activation_record_test.go | 83 +++ .../temporary_activation_run.go | 281 ++++++++++ .../temporary_activation_test.go | 94 ++++ scripts/migrate-longhorn-v2/v2_readiness.go | 199 +++++++ .../migrate-longhorn-v2/v2_readiness_test.go | 159 ++++++ .../approved-surface.txt | 2 +- 39 files changed, 3674 insertions(+), 10 deletions(-) create mode 100644 k8s/providers/hetzner/infrastructure/controllers/longhorn/storage-class-longhorn-v2.yaml create mode 100644 scripts/migrate-longhorn-v2/canary.go create mode 100644 scripts/migrate-longhorn-v2/canary_assets.go create mode 100644 scripts/migrate-longhorn-v2/canary_assets_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_cleanup.go create mode 100644 scripts/migrate-longhorn-v2/canary_cleanup_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_cli_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_assets.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_assets_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_binding.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_binding_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_capture.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_capture_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_native_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_runtime.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_runtime_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_transfer.go create mode 100644 scripts/migrate-longhorn-v2/canary_csi_transfer_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_identity_test.go create mode 100644 scripts/migrate-longhorn-v2/canary_transfer.go create mode 100644 scripts/migrate-longhorn-v2/canary_transfer_test.go create mode 100644 scripts/migrate-longhorn-v2/temporary_activation.go create mode 100644 scripts/migrate-longhorn-v2/temporary_activation_authorization_test.go create mode 100644 scripts/migrate-longhorn-v2/temporary_activation_flow_test.go create mode 100644 scripts/migrate-longhorn-v2/temporary_activation_native_test.go create mode 100644 scripts/migrate-longhorn-v2/temporary_activation_record.go create mode 100644 scripts/migrate-longhorn-v2/temporary_activation_record_test.go create mode 100644 scripts/migrate-longhorn-v2/temporary_activation_run.go create mode 100644 scripts/migrate-longhorn-v2/temporary_activation_test.go create mode 100644 scripts/migrate-longhorn-v2/v2_readiness.go create mode 100644 scripts/migrate-longhorn-v2/v2_readiness_test.go diff --git a/.github/workflows/prepare-longhorn-v2-node.yaml b/.github/workflows/prepare-longhorn-v2-node.yaml index 8619ec5862..5dd5dda195 100644 --- a/.github/workflows/prepare-longhorn-v2-node.yaml +++ b/.github/workflows/prepare-longhorn-v2-node.yaml @@ -17,7 +17,7 @@ on: description: The guarded migration step to run required: true type: choice - options: [prepare-worker, prepare-unused-temporary-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas, inventory-storage-root, inventory-storage-layout, inspect-host-memory, reboot-host-for-hugepages] + options: [prepare-worker, prepare-unused-temporary-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas, inventory-storage-root, inventory-storage-layout, inspect-host-memory, reboot-host-for-hugepages, canary-v2, canary-csi-v2, register-temporary-v2-storage] node: description: Permanent worker or fixed temporary-worker selection for the chosen operation required: false @@ -47,7 +47,22 @@ jobs: with: go-version-file: go.mod cache: false + - name: Install the approved client for hermetic native watch tests + shell: bash + env: + KUBECTL_VERSION: v1.36.2 + KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 + run: | + set -euo pipefail + kubectl_path="$RUNNER_TEMP/longhorn-v2-test-kubectl" + curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" + printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check + mkdir -p "$RUNNER_TEMP/longhorn-v2-test-tools" + install -m 0755 "$kubectl_path" "$RUNNER_TEMP/longhorn-v2-test-tools/kubectl" + printf '%s\n' "$RUNNER_TEMP/longhorn-v2-test-tools" >> "$GITHUB_PATH" - name: Test complete observations and interrupted recovery + env: + LONGHORN_V2_NATIVE_TEST: '1' run: go test -race -count=1 -timeout=5m ./scripts/migrate-longhorn-v2 ./internal/longhornmigration ./internal/longhorninventory prepare: @@ -104,6 +119,15 @@ jobs: benchmark-v1|benchmark-v2) [[ "${LONGHORN_V2_CONFIRM}" == benchmark-longhorn-storage ]] ;; + canary-v2) + [[ "${LONGHORN_V2_CONFIRM}" == verify-longhorn-v2-restore ]] + ;; + canary-csi-v2) + [[ "${LONGHORN_V2_CONFIRM}" == verify-longhorn-v2-csi-restore ]] + ;; + register-temporary-v2-storage) + [[ "${LONGHORN_V2_CONFIRM}" == register-longhorn-v2-temporary-storage ]] + ;; *) exit 1 ;; esac - name: Checkout reviewed source diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/helm-release.yaml b/k8s/providers/hetzner/infrastructure/controllers/longhorn/helm-release.yaml index df6f88c4a4..790ce0ae8a 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/longhorn/helm-release.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/helm-release.yaml @@ -32,6 +32,19 @@ spec: values: defaultSettings: defaultDataPath: /var/lib/longhorn + # Coexist during migration; retire V1 only after all active and retained + # application data has been recovered and verified on V2. + v1DataEngine: true + v2DataEngine: true + # Each migration worker has eight CPUs. Reserve two polling reactors; + # preserve the V1 reservation while the original replicas remain live. + guaranteedInstanceManagerCPU: '{"v1":"6","v2":"25"}' + dataEngineCPUMask: '{"v2":"0x3"}' + dataEngineNumberOfCPUCores: '{"v2":"0"}' + dataEngineCPUIsolationEnabled: '{"v2":"true"}' + dataEngineInterruptModeEnabled: '{"v2":"false"}' + dataEngineHugepageEnabled: '{"v2":"true"}' + dataEngineMemorySize: '{"v2":"2048"}' # Match the number of workers so every volume is replicated across # all available nodes. defaultReplicaCount: ${longhorn_replica_count:=3} diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/longhorn/kustomization.yaml index ff1d681b2b..793b4dce31 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/longhorn/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/kustomization.yaml @@ -8,6 +8,7 @@ resources: # Default WaitForFirstConsumer StorageClass (see storage-class-longhorn-wffc.yaml); # the chart's `longhorn` class is demoted to non-default in helm-release.yaml. - storage-class-longhorn-wffc.yaml + - storage-class-longhorn-v2.yaml - cilium-network-policy.yaml - http-route.yaml # Default off: no benchmark namespace currently carries the opt-in label. diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/limit-range.yaml b/k8s/providers/hetzner/infrastructure/controllers/longhorn/limit-range.yaml index e7291aa7c2..d455824a3c 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/longhorn/limit-range.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/limit-range.yaml @@ -15,10 +15,10 @@ # # This restores ONLY the CPU limit and deliberately keeps memory unset, so the # OOM-cascade safeguard is preserved: -# * default.cpu "2" is generous and compressible — historically longhorn ran -# fine under the old 500m CPU cap (the incident was memory, never CPU), so a -# 4x-higher ceiling cannot throttle a rebuild into failure; a CPU limit only -# slows, it never OOMKills, so it cannot trigger the IM-restart cascade. +# * default.cpu "4" leaves management and network work headroom above V2's +# two polling reactors. Longhorn sets the CPU request separately; this +# changes its ceiling. Recovery acceptance checks the admitted request and +# limit before registering disks or creating a disposable probe. # * NO default.memory — this is the entire point of the longhorn exclusion and # must stay unset (instance-managers must be free to burst memory during a # rebuild). @@ -33,7 +33,7 @@ spec: limits: - type: Container default: - cpu: "2" + cpu: "4" defaultRequest: cpu: 15m memory: 128Mi diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/storage-class-longhorn-v2.yaml b/k8s/providers/hetzner/infrastructure/controllers/longhorn/storage-class-longhorn-v2.yaml new file mode 100644 index 0000000000..82be0826c0 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/storage-class-longhorn-v2.yaml @@ -0,0 +1,19 @@ +--- +# Opt-in migration destination. The V1 default and existing claims remain until +# each application's recovered bytes and service behavior have been verified. +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: longhorn-v2 +provisioner: driver.longhorn.io +allowVolumeExpansion: true +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +parameters: + numberOfReplicas: "3" + staleReplicaTimeout: "30" + fsType: ext4 + dataLocality: disabled + unmapMarkSnapChainRemoved: disabled + dataEngine: v2 + backupTargetName: default diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index c61e969f5a..b12a7c2f53 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -56,13 +56,69 @@ may adopt it only after the previous run has ended. This operation enables no V2 storage and removes no iSCSI support. The `Prepare Longhorn V2 Migration` workflow runs guarded steps towards -platform#5582. It does not migrate a claim, enable V2, disable V1 or remove iSCSI. +platform#5582. Its recovery probes use disposable claims; application cutover, +V1 retirement and iSCSI removal have separate acceptance gates. This tool lands in stages (platform#5655). It includes `refresh-clients`, `prepare-worker`, `prepare-unused-temporary-worker`, read-only replica inventories and the benchmarks. `refresh-clients` holds until the fixed migration workers are Ready and the host -huge-page settings are loaded. Worker preparation is below and is dormant; enabling temporary -capacity follows in its own change. +huge-page settings are loaded. Host recovery and a cleaned V1 baseline precede +temporary-disk preparation. The V2 activation change must follow those completed +operations, rather than their source PRs alone. + +## Temporary V2 activation and disposable recovery proof + +The activation manifests retain V1 and the existing default StorageClass, enable +V2, and add the opt-in `longhorn-v2` Retain class. Eight-CPU workers reserve two +cores for polling SPDK reactors with mask `0x3`, CPU isolation, and 2048 MiB of +huge pages. The CPU-only LimitRange rises to four cores so management and network +work have headroom above those reactors; it adds no memory limit. The reviewed +Longhorn 1.13.0 instance-manager image and the admitted client resources and +arguments must match before registration. Enabling the setting proves neither +raw-disk readiness nor application recovery. + +After both temporary V1 workers have completed their independently confirmed +raw-layout preparation, select `register-temporary-v2-storage` and confirm +`register-longhorn-v2-temporary-storage`. This operation always selects all three +fixed temporary pool members. It requires intact original V1 replicas, healthy +database quorums, the cleaned baseline, reviewed storage capacity, no competing +maintenance, and ready V2 clients on all six workers before its first write. + +The journal `longhorn-v2-temporary-activation` binds each temporary selection to +the node, provider, hardware, boot, encrypted EPHEMERAL and raw GPT partition +identities. Complete native reads bracket the partition observations. Each +conditional Longhorn Node update registers only the canonical raw disk and adds +the journal UID as its ownership annotation. Matching unowned disks are refused. +An uncertain response stops the run. A new first-attempt main dispatch may resume +an owned partial registration only after the previous workflow is terminal, and +it revalidates all three native bindings and every remaining prerequisite. +Completion requires three independent ready raw disks and all qualified clients; +it grants no application cutover or original-disk reset authority. + +Run `canary-v2` with `verify-longhorn-v2-restore` for filesystem backup recovery, +then `canary-csi-v2` with `verify-longhorn-v2-csi-restore` for CSI snapshot data +movement. Each has a distinct journal and confirmation. Both first revalidate the +completed activation against its current native identities and require its +owning workflow to be terminal. The probes create their own restricted namespaces, +classes and claims, write and checksum 128 MiB on V1, restore to an independent +three-replica V2 backend, compare the original bytes and sentinel, then reattach +the same V2 backend from a second permanent worker and compare again. + +Backup metadata alone cannot pass either probe. Filesystem recovery binds the +native PodVolumeBackup and PodVolumeRestore; CSI recovery binds the DataUpload, +DataDownload, claim, PV and Longhorn backend, including short-lived scratch +identities captured by bounded native list/watch streams. Successful cleanup is +verified before the journal records completion. Any failed or interrupted probe +retains its journal and scratch for diagnosis; it is not rerun or adopted by name. +All original application volumes and historical recovery states remain protected. + +`go run ./scripts/migrate-longhorn-v2 --render-canary 1234` and +`go run ./scripts/migrate-longhorn-v2 --render-csi-canary 1234` render example +disposable assets without production credentials +or mutations. The native watch transport test uses an isolated local HTTP API +fixture and kubeconfig; CI installs the checksum-pinned approved kubectl and runs +it with `LONGHORN_V2_NATIVE_TEST=1`. That protocol test is not a production restore +receipt. Real recovery results come only from the confirmed main dispatches. For the unused temporary-worker route (platform#5787), finish the client refresh and the cleaned V1 baseline first. Select `prepare-unused-temporary-worker`, `temporary-v1-1` or diff --git a/scripts/migrate-longhorn-v2/authorization.go b/scripts/migrate-longhorn-v2/authorization.go index 248f9ede82..60a6cf174a 100644 --- a/scripts/migrate-longhorn-v2/authorization.go +++ b/scripts/migrate-longhorn-v2/authorization.go @@ -45,3 +45,16 @@ func authorizeBenchmark(getenv func(string) string, phase string) error { } return authorizeWorkflow(getenv, "benchmark-longhorn-storage") } + +// authorizeCanary keeps filesystem and CSI data movement behind separate +// confirmations in the same first-attempt protected main workflow. +func authorizeCanary(getenv func(string) string) error { + switch getenv("LONGHORN_V2_OPERATION") { + case "canary-v2": + return authorizeWorkflow(getenv, "verify-longhorn-v2-restore") + case "canary-csi-v2": + return authorizeWorkflow(getenv, "verify-longhorn-v2-csi-restore") + default: + return errors.New("the explicit V2 recovery canary operation is required") + } +} diff --git a/scripts/migrate-longhorn-v2/canary.go b/scripts/migrate-longhorn-v2/canary.go new file mode 100644 index 0000000000..7580d62ff5 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary.go @@ -0,0 +1,519 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "reflect" + "regexp" + "strings" + "time" +) + +type canaryRecord struct { + Version int + Source, Run, Phase, Image, Workload, PayloadSHA256, Snapshot, CompletedAt string + Resources map[string]benchmarkResource + Transport string + CSIScratch map[string]csiScratchEvidence + RestoreVerified, ReattachVerified, Cleaned bool +} + +// The canary is qualified for the instance-manager image shipped by the reviewed +// Longhorn 1.13.0 chart. A chart upgrade requires a new qualification, rather +// than accepting matching but unknown live declarations as image provenance. +const canaryManagerImage = "longhornio/longhorn-instance-manager:v1.13.0" + +type canaryRun struct { + c client + source, run string + record canaryRecord + journalUID string + csi *csiCapture + sourceClaim, sourceVolume, snapshotClass object +} + +// canaryWorkload binds recovery evidence to both reviewed payload commands. +func canaryWorkload() string { + digest := sha256.Sum256([]byte(canaryScript + canaryVerify)) + return hex.EncodeToString(digest[:]) +} + +// save conditionally updates the owned journal without changing previously +// recorded scratch identities, then requires an exact native readback. +func (p *canaryRun) save(ctx context.Context) error { + name, err := canaryJournalName(p.record.Transport) + if err != nil { + return err + } + cm, err := p.c.get(ctx, "configmap", "longhorn-system", name, false) + if err != nil { + return err + } + var old canaryRecord + if p.journalUID == "" || text(cm, "metadata", "uid") != p.journalUID || json.Unmarshal([]byte(text(cm, "data", "record")), &old) != nil || old.Source != p.source || old.Run != p.run || old.Transport != p.record.Transport || old.Cleaned { + return errors.New("canary journal ownership changed") + } + for key, identity := range old.Resources { + if p.record.Resources[key] != identity { + return errors.New("canary scratch identity changed") + } + } + body, err := json.Marshal(p.record) + if err != nil { + return err + } + if err = sourceCurrent(ctx, p.c.run, p.source); err != nil { + return err + } + if err = p.c.patch(ctx, "configmap", "longhorn-system", cm, object{"op": "replace", "path": "/data/record", "value": string(body)}); err != nil { + return err + } + cm, err = p.c.get(ctx, "configmap", "longhorn-system", name, false) + if err != nil || text(cm, "metadata", "uid") != p.journalUID || text(cm, "data", "record") != string(body) { + return errors.New("canary journal update could not be read back") + } + return nil +} + +// remember persists a newly created resource identity before later operations. +func (p *canaryRun) remember(ctx context.Context, key string, o object) error { + if text(o, "metadata", "name") == "" || text(o, "metadata", "uid") == "" { + return errors.New("canary resource identity is incomplete") + } + p.record.Resources[key] = benchmarkIdentity(o) + return p.save(ctx) +} + +// create records the native identity returned by an exclusive scratch creation. +func (p *canaryRun) create(ctx context.Context, key string, desired object) (object, error) { + o, err := p.c.create(ctx, p.source, desired) + if err != nil { + return nil, err + } + if err = p.remember(ctx, key, o); err != nil { + return nil, err + } + return o, nil +} + +// verifyCanaryPod requires the original restricted consumer and executable on +// the selected client before any command reads or writes the payload. +func verifyCanaryPod(p object, uid, node string) error { + status, _ := value(p, "status").(object) + if text(p, "metadata", "uid") != uid || value(p, "metadata", "deletionTimestamp") != nil || text(p, "spec", "nodeName") != node || + value(p, "spec", "automountServiceAccountToken") != false || value(p, "spec", "hostUsers") != false || !ready(status) { + return errors.New("canary consumer identity or placement changed") + } + containers, _ := value(p, "spec", "containers").([]any) + statuses, _ := value(p, "status", "containerStatuses").([]any) + if len(containers) != 1 || len(statuses) != 1 { + return errors.New("canary consumer executable is ambiguous") + } + c, _ := containers[0].(object) + containerStatus, _ := statuses[0].(object) + if text(c, "name") != "fio" || text(c, "image") != fioImage || !reflect.DeepEqual(value(c, "command"), []any{"/bin/sh", "-ec"}) || !reflect.DeepEqual(value(c, "args"), []any{canaryScript}) || + text(containerStatus, "name") != "fio" || !strings.HasSuffix(text(containerStatus, "imageID"), strings.Split(fioImage, "@")[1]) { + return errors.New("canary executable differs from the reviewed probe") + } + return nil +} + +// verifyCanarySettings binds both engine enablement and the effective SPDK +// configuration to the reviewed migration profile before disk registration or IO. +func verifyCanarySettings(ctx context.Context, c client) error { + for name, expected := range map[string]string{ + "v1-data-engine": "true", "v2-data-engine": "true", "guaranteed-instance-manager-cpu": `{"v1":"6","v2":"25"}`, + "data-engine-cpu-mask": `{"v2":"0x3"}`, "data-engine-number-of-cpu-cores": `{"v2":"0"}`, "data-engine-cpu-isolation-enabled": `{"v2":"true"}`, + "data-engine-interrupt-mode-enabled": `{"v2":"false"}`, "data-engine-hugepage-enabled": `{"v2":"true"}`, "data-engine-memory-size": `{"v2":"2048"}`, + } { + o, err := c.get(ctx, "settings.longhorn.io", "longhorn-system", name, false) + if err != nil { + return err + } + actual := text(o, "value") + if strings.HasPrefix(expected, "{") { + var want, got map[string]string + if json.Unmarshal([]byte(expected), &want) != nil || json.Unmarshal([]byte(actual), &got) != nil || !reflect.DeepEqual(want, got) { + return errors.New("live V2 settings have not adopted the reviewed configuration") + } + } else if actual != expected { + return errors.New("both engines must remain enabled during the recovery canary") + } + } + return nil +} + +// canaryEnvironment proves both the actual worker clients and the three ready +// raw disks; enabling V2 alone cannot authorize a recovery probe. +func canaryEnvironment(ctx context.Context, c client, pools map[string]migrationPool) (inventory, error) { + if err := verifyCanarySettings(ctx, c); err != nil { + return inventory{}, err + } + s, err := c.snapshot(ctx) + if err != nil { + return inventory{}, err + } + managers, err := c.list(ctx, "instancemanagers.longhorn.io", "longhorn-system") + if err != nil { + return inventory{}, err + } + if err = verifyV2Runtime(s, managers, pools, canaryManagerImage); err != nil { + return inventory{}, err + } + return s, nil +} + +// verifyCanaryBindingIdentity rejects replacement source objects and target +// claims outside this run's exact native restore. +func verifyCanaryBindingIdentity(phase string, record canaryRecord, class, claim object) error { + group := "source" + if phase == "v2" { + group = "target" + } else if phase != "v1" { + return errors.New("canary engine is outside this procedure") + } + expected := record.Resources[group+"/StorageClass"] + if expected.UID == "" || expected.UID != text(class, "metadata", "uid") || expected.Name != text(class, "metadata", "name") { + return errors.New("canary storage class was replaced") + } + if phase == "v1" { + expected = record.Resources["source/PersistentVolumeClaim"] + if expected.UID == "" || expected.UID != text(claim, "metadata", "uid") || expected.Name != text(claim, "metadata", "name") { + return errors.New("canary source claim was replaced") + } + } else if text(claim, "metadata", "labels", "velero.io/restore-name") != "longhorn-v2-canary-"+record.Run || text(claim, "metadata", "labels", "velero.io/backup-name") != "longhorn-v2-canary-"+record.Run { + return errors.New("V2 claim was not produced by the exact canary restore") + } + return nil +} + +// binding waits for the owned consumer's complete claim/PV/backend graph and +// healthy replicas, preserving each identity for comparison and cleanup. +func (p *canaryRun) binding(ctx context.Context, namespace, class, phase, node, podUID string) (map[string]object, object, object, error) { + var result map[string]object + var pv, volume object + err := waitReady(ctx, func(ctx context.Context) error { + pod, err := p.c.get(ctx, "pod", namespace, "fio", false) + if err != nil || text(pod, "status", "phase") != "Running" { + return errors.New("canary consumer is not running") + } + if err = verifyCanaryPod(pod, podUID, node); err != nil { + return err + } + claim, err := p.c.get(ctx, "persistentvolumeclaim", namespace, "fio-data", false) + if err != nil || text(claim, "status", "phase") != "Bound" || text(claim, "spec", "storageClassName") != class { + return errors.New("canary claim has not bound to its reviewed class") + } + pv, err = p.c.get(ctx, "persistentvolume", "", text(claim, "spec", "volumeName"), false) + if err != nil { + return err + } + volume, err = p.c.get(ctx, "volumes.longhorn.io", "longhorn-system", text(pv, "spec", "csi", "volumeHandle"), false) + if err != nil || text(volume, "spec", "dataEngine") != phase || text(volume, "status", "state") != "attached" || text(volume, "status", "robustness") != "healthy" { + return errors.New("canary backend has not become healthy on the selected engine") + } + classObject, err := p.c.get(ctx, "storageclass", "", class, false) + if err != nil { + return err + } + if err = verifyCanaryBindingIdentity(phase, p.record, classObject, claim); err != nil { + return err + } + if err = verifyScratchBinding(pv, volume, claim, classObject); err != nil { + return err + } + s, err := p.c.snapshot(ctx) + if err != nil { + return err + } + if err = verifyRedundancy(s); err != nil { + return err + } + result = map[string]object{"Pod": pod, "PersistentVolumeClaim": claim, "StorageClass": classObject} + return nil + }) + return result, pv, volume, err +} + +// verifyBytes rebinds the live executable before checking the sentinel, length, +// stored checksum and optional original source digest. +func (p *canaryRun) verifyBytes(ctx context.Context, pod object, node, expected string) (string, error) { + namespace := text(pod, "metadata", "namespace") + fresh, err := p.c.get(ctx, "pod", namespace, "fio", false) + if err != nil { + return "", err + } + if err = verifyCanaryPod(fresh, text(pod, "metadata", "uid"), node); err != nil { + return "", err + } + if _, err = p.c.kubectl(ctx, nil, "exec", "--namespace="+namespace, "fio", "--container=fio", "--", "/bin/sh", "-ec", canaryVerify); err != nil { + return "", errors.New("canary restored bytes failed their checksum or sentinel check") + } + data, err := p.c.kubectl(ctx, nil, "exec", "--namespace="+namespace, "fio", "--container=fio", "--", "sha256sum", "/data/payload.bin") + if err != nil { + return "", err + } + fields := strings.Fields(string(data)) + if len(fields) != 2 || fields[1] != "/data/payload.bin" || !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(fields[0]) || expected != "" && expected != fields[0] { + return "", errors.New("canary payload differs from the original source") + } + return fields[0], nil +} + +// transfer requires a completed owned native transfer; CSI uses its separate +// data-mover proof rather than filesystem-backup fields. +func (p *canaryRun) transfer(ctx context.Context, desired, pod object, expectedSnapshot, sourceNS string) (object, string, error) { + if p.csi != nil { + return p.transferCSI(ctx, desired, expectedSnapshot) + } + kind, query := text(desired, "kind"), "podvolumebackups.velero.io" + if kind == "Restore" { + query = "podvolumerestores.velero.io" + } + created, err := p.create(ctx, kind, desired) + if err != nil { + return nil, "", err + } + var parent object + var snapshot string + err = waitReady(ctx, func(ctx context.Context) error { + parent, err = p.c.get(ctx, strings.ToLower(kind)+"s.velero.io", "velero", text(created, "metadata", "name"), false) + if err != nil || text(parent, "metadata", "uid") != text(created, "metadata", "uid") { + return errors.New("canary backup or restore identity changed") + } + if err = cleanVeleroResult(parent); err != nil { + return err + } + transfers, err := p.c.list(ctx, query, "velero") + if err != nil { + return err + } + if kind == "Restore" { + pod, err = p.c.get(ctx, "pod", "longhorn-canary-v2-"+p.run, "fio", false) + if err != nil { + return err + } + } + snapshot, err = verifyCanaryTransfer(transfers, parent, pod, expectedSnapshot, sourceNS) + return err + }) + return parent, snapshot, err +} + +// runCanaryWithTransport owns a single disposable V1-to-V2 recovery trial and +// records acceptance only after byte comparison, remote reattachment and cleanup. +func runCanaryWithTransport(ctx context.Context, c client, source, run string, pools map[string]migrationPool, transport string) error { + journalName, err := canaryJournalName(transport) + if err != nil { + return err + } + assets, err := canaryResources(run) + if transport == "csi" { + assets, err = csiCanaryResources(run) + } + if err != nil { + return err + } + if err = sourceCurrent(ctx, c.run, source); err != nil { + return err + } + if err = (temporaryV2Activation{c: c, pools: pools}).accepted(ctx); err != nil { + return err + } + s, err := canaryEnvironment(ctx, c, pools) + if err != nil { + return err + } + if err = requireBenchmarkBaseline(ctx, c, s.nodes); err != nil { + return err + } + for _, volume := range s.volumes { + if text(volume, "spec", "dataEngine") != "v1" { + return errors.New("the recovery canary must precede application migration") + } + } + bsl, err := c.get(ctx, "backupstoragelocations.velero.io", "velero", "default", false) + if err != nil || text(bsl, "status", "phase") != "Available" || text(bsl, "spec", "accessMode") == "ReadOnly" { + return errors.New("the existing backup destination is unavailable for the recovery canary") + } + old, err := c.get(ctx, "configmap", "longhorn-system", journalName, true) + if err != nil { + return err + } + if old != nil { + return errors.New("a prior canary journal exists; preserve its resources and diagnose before explicit recovery") + } + p := canaryRun{c: c, source: source, run: run, record: canaryRecord{Version: 2, Source: source, Run: run, Transport: transport, Phase: "created", Image: fioImage, Workload: canaryWorkload(), Resources: map[string]benchmarkResource{}, CSIScratch: map[string]csiScratchEvidence{}}} + body, _ := json.Marshal(p.record) + journal, err := c.create(ctx, source, object{"apiVersion": "v1", "kind": "ConfigMap", "metadata": object{"name": journalName, "namespace": "longhorn-system"}, "data": object{"record": string(body)}}) + if err != nil { + return err + } + p.journalUID = text(journal, "metadata", "uid") + created := map[string]map[string]object{"source": {}, "target": {}} + for i, desired := range assets[:8] { + group := "source" + if i >= 5 { + group = "target" + } + kind := text(desired, "kind") + o, err := p.create(ctx, group+"/"+kind, desired) + if err != nil { + return err + } + created[group][kind] = o + } + sourceNS, targetNS := "longhorn-canary-v1-"+run, "longhorn-canary-v2-"+run + sourceBound, sourcePV, sourceVolume, err := p.binding(ctx, sourceNS, "lh-canary-v1-"+run, "v1", "prod-worker-3", text(created["source"]["Pod"], "metadata", "uid")) + if err != nil { + return err + } + for kind, o := range map[string]object{"PersistentVolume": sourcePV, "LonghornVolume": sourceVolume} { + if err = p.remember(ctx, "source/"+kind, o); err != nil { + return err + } + } + p.record.Phase = "source-write-requested" + if err = p.save(ctx); err != nil { + return err + } + if err = sourceCurrent(ctx, c.run, source); err != nil { + return err + } + freshSource, err := c.get(ctx, "pod", sourceNS, "fio", false) + if err != nil { + return err + } + if err = verifyCanaryPod(freshSource, text(sourceBound["Pod"], "metadata", "uid"), "prod-worker-3"); err != nil { + return err + } + fmt.Println("Canary storage provisioned with three healthy copies; writing the disposable source payload.") + if _, err = c.kubectl(ctx, nil, "exec", "--namespace="+sourceNS, "fio", "--container=fio", "--", "/bin/sh", "-ec", "touch /tmp/canary-start"); err != nil { + return err + } + if err = waitReady(ctx, func(ctx context.Context) error { + logs, err := c.kubectl(ctx, nil, "logs", "--namespace="+sourceNS, "fio", "--container=fio") + if err != nil || strings.TrimSpace(string(logs)) != "CANARY_READY" { + return errors.New("source canary has not completed its verified write") + } + return nil + }); err != nil { + return err + } + p.record.PayloadSHA256, err = p.verifyBytes(ctx, sourceBound["Pod"], "prod-worker-3", "") + if err != nil { + return err + } + if _, err = p.create(ctx, "Modifier", assets[8]); err != nil { + return err + } + if transport == "csi" { + p.sourceClaim, p.sourceVolume = sourceBound["PersistentVolumeClaim"], sourceVolume + p.snapshotClass, err = p.create(ctx, "SnapshotClass", assets[11]) + if err != nil { + return err + } + p.csi = newCSICapture(run) + if err = p.csi.start(ctx, c); err != nil { + return err + } + defer p.csi.stop() + } + backup, snapshot, err := p.transfer(ctx, assets[9], sourceBound["Pod"], "", sourceNS) + if err != nil { + return err + } + p.record.Snapshot, p.record.Phase = snapshot, "backup-verified" + if err = p.save(ctx); err != nil { + return err + } + if p.csi != nil { + if err = p.waitCSIUploadGone(ctx); err != nil { + return err + } + } + fmt.Println("Canary backup data transfer verified; restoring onto an independent V2 volume.") + restore, _, err := p.transfer(ctx, assets[10], nil, snapshot, sourceNS) + if err != nil { + return err + } + restoredPod, err := c.get(ctx, "pod", targetNS, "fio", false) + if err != nil { + return err + } + targetBound, targetPV, targetVolume, err := p.binding(ctx, targetNS, "lh-canary-v2-"+run, "v2", "prod-worker-3", text(restoredPod, "metadata", "uid")) + if err != nil { + return err + } + if text(targetPV, "metadata", "uid") == text(sourcePV, "metadata", "uid") || text(targetVolume, "metadata", "uid") == text(sourceVolume, "metadata", "uid") { + return errors.New("restore reused source storage instead of creating an independent V2 backend") + } + if p.csi != nil { + if err = verifyCSIRestoredBinding(p.record.CSIScratch["DataDownload"], targetBound["PersistentVolumeClaim"], targetPV, targetVolume); err != nil { + return err + } + } + for kind, o := range targetBound { + created["target"][kind] = o + if err = p.remember(ctx, "restored/"+kind, o); err != nil { + return err + } + } + for kind, o := range map[string]object{"PersistentVolume": targetPV, "LonghornVolume": targetVolume} { + if err = p.remember(ctx, "target/"+kind, o); err != nil { + return err + } + } + if _, err = p.verifyBytes(ctx, targetBound["Pod"], "prod-worker-3", p.record.PayloadSHA256); err != nil { + return err + } + p.record.RestoreVerified, p.record.Phase = true, "restore-verified" + if err = p.save(ctx); err != nil { + return err + } + fmt.Println("Restored bytes match the source; checking persistence on another worker.") + if err = p.reattach(ctx, created["target"], targetPV, targetVolume); err != nil { + return err + } + p.record.ReattachVerified, p.record.Phase = true, "reattach-verified" + if err = p.save(ctx); err != nil { + return err + } + if p.csi != nil { + if err = p.removeBackup(ctx, backup, restore, assets[8]); err != nil { + return err + } + if err = p.waitCSIArtifactsGone(ctx); err != nil { + return err + } + } + if err = cleanupBenchmark(ctx, c, source, targetNS, created["target"], targetPV, targetVolume); err != nil { + return err + } + if err = cleanupBenchmark(ctx, c, source, sourceNS, created["source"], sourcePV, sourceVolume); err != nil { + return err + } + if p.csi == nil { + if err = p.removeBackup(ctx, backup, restore, assets[8]); err != nil { + return err + } + } else { + if err = p.removeCSISnapshotClass(ctx); err != nil { + return err + } + p.csi.stop() + if err = p.csi.health(); err != nil { + return err + } + } + p.record.Cleaned, p.record.Phase, p.record.CompletedAt = true, "completed", time.Now().UTC().Format(time.RFC3339) + if err = p.save(ctx); err != nil { + return err + } + fmt.Println("V2 recovery canary passed: independent three-replica restore, original-byte comparison, remote-client reattachment and scratch cleanup verified.") + return nil +} diff --git a/scripts/migrate-longhorn-v2/canary_assets.go b/scripts/migrate-longhorn-v2/canary_assets.go new file mode 100644 index 0000000000..f3ca394264 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_assets.go @@ -0,0 +1,132 @@ +package main + +import ( + "errors" + "fmt" + "regexp" + "time" +) + +const canaryScript = `set -eu +while [ ! -f /tmp/canary-start ]; do sleep 2; done +fio --name=longhorn-v2-canary --filename=/data/payload.bin --size=128M --rw=write --bs=4k --ioengine=libaio --direct=1 --verify=sha256 --do_verify=1 --verify_fatal=1 --verify_state_save=0 --end_fsync=1 --output=/tmp/probe.json --output-format=json +cd /data +sha256sum payload.bin > payload.sha256 +printf 'longhorn-v2-5582\n' > sentinel +sync +printf 'CANARY_READY\n' +while :; do sleep 10; done +` + +const canaryVerify = `set -eu +cd /data +test "$(cat sentinel)" = longhorn-v2-5582 +test "$(wc -c < payload.bin)" -eq 134217728 +sha256sum --check payload.sha256 +` + +// canaryProbeResources reuses the reviewed restricted benchmark workload while +// giving this recovery proof its own names, script, and backup-visible labels. +func canaryProbeResources(namespace, class, runID, phase string) ([]object, error) { + assets, err := benchmarkResources(phase, runID, "longhorn-benchmark-"+phase+"-"+runID) + if err != nil { + return nil, err + } + for _, asset := range assets { + metadata := asset["metadata"].(object) + delete(metadata["labels"].(object), "velero.io/exclude-from-backup") + metadata["labels"].(object)["storage.platform.devantler.tech/canary-run"] = runID + if text(asset, "kind") == "Namespace" { + metadata["name"] = namespace + } else if text(asset, "kind") == "StorageClass" { + metadata["name"] = class + asset["parameters"].(object)["nodeSelector"] = "" + if phase == "v2" { + asset["parameters"].(object)["unmapMarkSnapChainRemoved"] = "disabled" + } + } else { + metadata["namespace"] = namespace + } + } + assets[3]["spec"].(object)["storageClassName"] = class + value(assets[4], "spec", "containers").([]any)[0].(object)["args"] = []any{canaryScript} + return assets, nil +} + +// canaryResources limits backup and restore to this run's disposable workload. +func canaryResources(runID string) ([]object, error) { + if !regexp.MustCompile(`^[1-9][0-9]*$`).MatchString(runID) { + return nil, errors.New("immutable canary run identity is required") + } + sourceNS, targetNS := "longhorn-canary-v1-"+runID, "longhorn-canary-v2-"+runID + sourceClass, targetClass := "lh-canary-v1-"+runID, "lh-canary-v2-"+runID + assets, err := canaryProbeResources(sourceNS, sourceClass, runID, "v1") + if err != nil { + return nil, err + } + pod := assets[4] + pod["metadata"].(object)["annotations"] = object{"backup.velero.io/backup-volumes": "data"} + pod["spec"].(object)["activeDeadlineSeconds"] = 5400 + target, err := canaryProbeResources(targetNS, targetClass, runID, "v2") + if err != nil { + return nil, err + } + assets = append(assets, target[:3]...) + name := "longhorn-v2-canary-" + runID + meta := func() object { + return object{"name": name, "namespace": "velero", "labels": object{"storage.platform.devantler.tech/canary-run": runID}} + } + rules := fmt.Sprintf(`version: v1 +resourceModifierRules: +- conditions: + groupResource: persistentvolumeclaims + resourceNameRegex: "^fio-data$" + namespaces: ["%s"] + patches: + - operation: test + path: /spec/storageClassName + value: "%s" + - operation: replace + path: /spec/storageClassName + value: "%s" +- conditions: + groupResource: pods + resourceNameRegex: "^fio$" + namespaces: ["%s"] + patches: + - operation: test + path: /spec/containers/0/image + value: "%s" + - operation: replace + path: /spec/nodeSelector + value: '{"kubernetes.io/hostname":"prod-worker-3"}' +`, sourceNS, sourceClass, targetClass, sourceNS, fioImage) + assets = append(assets, + object{"apiVersion": "v1", "kind": "ConfigMap", "metadata": meta(), "data": object{"rules.yaml": rules}}, + object{"apiVersion": "velero.io/v1", "kind": "Backup", "metadata": meta(), "spec": object{ + "includedNamespaces": []any{sourceNS}, "includedResources": []any{"pods", "persistentvolumeclaims"}, "includeClusterResources": false, + "storageLocation": "default", "snapshotVolumes": false, "defaultVolumesToFsBackup": false, "ttl": "24h0m0s", "csiSnapshotTimeout": "10m0s", "itemOperationTimeout": "30m0s"}}, + object{"apiVersion": "velero.io/v1", "kind": "Restore", "metadata": meta(), "spec": object{ + "backupName": name, "includedNamespaces": []any{sourceNS}, "includedResources": []any{"pods", "persistentvolumeclaims"}, "includeClusterResources": false, + "namespaceMapping": object{sourceNS: targetNS}, "resourceModifier": object{"kind": "ConfigMap", "name": name}, "itemOperationTimeout": "30m0s"}}, + ) + return assets, nil +} + +// cleanVeleroResult requires native completion without errors or warnings; +// Velero's omitted zero counters remain valid. +func cleanVeleroResult(o object) error { + if text(o, "metadata", "uid") == "" || value(o, "metadata", "deletionTimestamp") != nil || text(o, "status", "phase") != "Completed" { + return errors.New("native backup or restore has not completed") + } + if _, err := time.Parse(time.RFC3339, text(o, "status", "completionTimestamp")); err != nil { + return errors.New("native backup or restore completion is missing") + } + // Velero v1.18 serializes its zero error/warning counters with omitempty. + for _, field := range []string{"errors", "warnings"} { + if count := value(o, "status", field); count != nil && count != float64(0) { + return errors.New("native backup or restore reported warnings or errors") + } + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/canary_assets_test.go b/scripts/migrate-longhorn-v2/canary_assets_test.go new file mode 100644 index 0000000000..b5d8830bcf --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_assets_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "strings" + "testing" + + "sigs.k8s.io/yaml" +) + +func TestCanaryRestoresOnlyItsOwnFilesystemDataOntoV2(t *testing.T) { + assets, err := canaryResources("1234") + if err != nil { + t.Fatal(err) + } + if len(assets) != 11 { + t.Fatalf("incomplete isolated canary assets: %d", len(assets)) + } + var backup, restore, modifier object + classes := map[string]object{} + for _, o := range assets { + switch text(o, "kind") { + case "Backup": + backup = o + case "Restore": + restore = o + case "ConfigMap": + modifier = o + case "StorageClass": + classes[text(o, "parameters", "dataEngine")] = o + case "Pod": + if text(o, "spec", "nodeSelector", "kubernetes.io/hostname") != "prod-worker-3" || value(o, "spec", "automountServiceAccountToken") != false || text(o, "metadata", "annotations", "backup.velero.io/backup-volumes") != "data" { + t.Fatal("probe must exercise the remote V2 client and back up only persistent data") + } + } + } + if len(classes) != 2 || text(classes["v2"], "parameters", "numberOfReplicas") != "3" || text(classes["v2"], "reclaimPolicy") != "Retain" { + t.Fatal("canary storage is not independent retained three-copy storage") + } + if text(classes["v2"], "parameters", "unmapMarkSnapChainRemoved") != "disabled" { + t.Fatal("V2 probe must not inherit unsupported trim behavior") + } + if value(backup, "spec", "snapshotVolumes") != false || value(backup, "spec", "defaultVolumesToFsBackup") != false || value(backup, "spec", "includeClusterResources") != false || value(restore, "spec", "includeClusterResources") != false { + t.Fatal("probe might clone a V1 snapshot or restore cluster resources") + } + if text(restore, "spec", "backupName") != text(backup, "metadata", "name") || text(restore, "spec", "resourceModifier", "name") != text(modifier, "metadata", "name") || text(restore, "spec", "namespaceMapping", "longhorn-canary-v1-1234") != "longhorn-canary-v2-1234" { + t.Fatal("restore escaped the dedicated source and destination") + } + rules := text(modifier, "data", "rules.yaml") + var native struct { + ResourceModifierRules []struct { + Patches []struct{ Operation, Path, Value string } + } + } + if err := yaml.Unmarshal([]byte(rules), &native); err != nil { + t.Fatal("modifier cannot decode Velero's native string-valued JSON patches:", err) + } + for _, required := range []string{"longhorn-canary-v1-1234", "^fio-data$", "^fio$", "/spec/storageClassName", "lh-canary-v2-1234", "operation: test"} { + if !strings.Contains(rules, required) { + t.Fatal("restore modifier lacks exact source or destination binding") + } + } + if strings.Contains(canaryScript, "curl ") || !strings.Contains(canaryScript, "while [ ! -f /tmp/canary-start ]") || !strings.Contains(canaryScript, "--verify=sha256") { + t.Fatal("probe does not wait for verified provisioning or verify persisted bytes") + } + if _, err = canaryResources("../other"); err == nil { + t.Fatal("unbounded canary identity accepted") + } +} + +func TestCompletedBackupCannotStandInForRestoredData(t *testing.T) { + b := resource("canary") + b["status"] = object{"phase": "Completed", "completionTimestamp": "2026-10-29T19:00:00Z", "errors": float64(0), "warnings": float64(0)} + if cleanVeleroResult(b) != nil { + t.Fatal("clean native completion rejected") + } + for _, field := range []string{"errors", "warnings"} { + b["status"].(object)[field] = float64(1) + if cleanVeleroResult(b) == nil { + t.Fatal("partial or warned backup accepted") + } + b["status"].(object)[field] = float64(0) + } + b["status"].(object)["phase"] = "PartiallyFailed" + if cleanVeleroResult(b) == nil { + t.Fatal("partial native completion accepted") + } + if cleanVeleroResult(object{}) == nil { + t.Fatal("missing evidence accepted") + } + b["status"].(object)["phase"] = "Completed" + delete(b["status"].(object), "warnings") + delete(b["status"].(object), "errors") + if cleanVeleroResult(b) != nil { + t.Fatal("native omitempty zero counters rejected") + } +} diff --git a/scripts/migrate-longhorn-v2/canary_cleanup.go b/scripts/migrate-longhorn-v2/canary_cleanup.go new file mode 100644 index 0000000000..c8937eda23 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_cleanup.go @@ -0,0 +1,127 @@ +package main + +import ( + "context" + "errors" +) + +// requireNativeRestoreGone observes Velero's own restore retirement and never +// deletes a replacement that reused the name. +func (p *canaryRun) requireNativeRestoreGone(ctx context.Context, restore object) error { + if text(restore, "metadata", "uid") == "" { + return errors.New("native restore retirement lacks its original identity") + } + // Velero 1.18.4's backup deletion controller retires its Restores too. + // Verify that native result; an unconditional second delete fails after + // successful cleanup and must never target a replacement with the same name. + current, err := p.c.get(ctx, "restores.velero.io", "velero", text(restore, "metadata", "name"), true) + if err != nil || current == nil { + return err + } + if text(current, "metadata", "uid") != text(restore, "metadata", "uid") { + return errors.New("canary restore was replaced; preserve the replacement") + } + return errors.New("native canary restore retirement has not finished") +} + +// reattach moves only this run's restored consumer to a second client while +// requiring the same claim, PV, backend and original payload bytes. +func (p *canaryRun) reattach(ctx context.Context, created map[string]object, originalPV, originalVolume object) error { + namespace := "longhorn-canary-v2-" + p.run + originalClaim := created["PersistentVolumeClaim"] + if err := removeScratch(ctx, p.c, p.source, "pod", namespace, "/api/v1/namespaces/"+namespace+"/pods/fio", created["Pod"]); err != nil { + return err + } + if err := waitReady(ctx, func(ctx context.Context) error { + volume, err := p.c.get(ctx, "volumes.longhorn.io", "longhorn-system", text(originalVolume, "metadata", "name"), false) + if err != nil || text(volume, "metadata", "uid") != text(originalVolume, "metadata", "uid") || text(volume, "status", "state") != "detached" { + return errors.New("canary volume has not detached before changing clients") + } + return nil + }); err != nil { + return err + } + assets, err := canaryProbeResources(namespace, "lh-canary-v2-"+p.run, p.run, "v2") + if err != nil { + return err + } + desired := assets[4] + desired["spec"].(object)["nodeSelector"] = object{"kubernetes.io/hostname": "prod-worker-1"} + desired["spec"].(object)["activeDeadlineSeconds"] = 1800 + pod, err := p.create(ctx, "reattached/Pod", desired) + if err != nil { + return err + } + bound, pv, volume, err := p.binding(ctx, namespace, "lh-canary-v2-"+p.run, "v2", "prod-worker-1", text(pod, "metadata", "uid")) + if err != nil { + return err + } + if text(pv, "metadata", "uid") != text(originalPV, "metadata", "uid") || text(volume, "metadata", "uid") != text(originalVolume, "metadata", "uid") || text(bound["PersistentVolumeClaim"], "metadata", "uid") != text(originalClaim, "metadata", "uid") { + return errors.New("canary reattachment replaced its original restored storage") + } + if _, err = p.verifyBytes(ctx, bound["Pod"], "prod-worker-1", p.record.PayloadSHA256); err != nil { + return err + } + created["Pod"] = bound["Pod"] + return nil +} + +// removeBackup delegates data retirement to Velero's UID-bound deletion request +// and verifies native absence before removing the owned modifier. +func (p *canaryRun) removeBackup(ctx context.Context, original, restore, modifier object) error { + name := "longhorn-v2-canary-" + p.run + backup, err := p.c.get(ctx, "backups.velero.io", "velero", name, false) + if err != nil || text(backup, "metadata", "uid") != text(original, "metadata", "uid") { + return errors.New("canary backup was replaced; refusing retirement") + } + if err = cleanVeleroResult(backup); err != nil { + return err + } + // Velero's native deletion request removes Kopia snapshots and backup + // objects together. Never delete only the Backup CR and strand its data. + request, err := p.create(ctx, "DeleteBackupRequest", object{"apiVersion": "velero.io/v1", "kind": "DeleteBackupRequest", + "metadata": object{"name": name + "-delete", "namespace": "velero", "labels": object{"velero.io/backup-name": name, "velero.io/backup-uid": text(backup, "metadata", "uid")}}, + "spec": object{"backupName": name}}) + if err != nil { + return err + } + if err = waitReady(ctx, func(ctx context.Context) error { + current, err := p.c.get(ctx, "backups.velero.io", "velero", name, true) + if err != nil { + return err + } + if current != nil && text(current, "metadata", "uid") != text(original, "metadata", "uid") { + return errors.New("backup identity changed during retirement") + } + dbr, err := p.c.get(ctx, "deletebackuprequests.velero.io", "velero", text(request, "metadata", "name"), true) + if err != nil { + return err + } + if dbr != nil { + if text(dbr, "metadata", "uid") != text(request, "metadata", "uid") { + return errors.New("backup deletion request was replaced") + } + if errs := value(dbr, "status", "errors"); errs != nil { + if list, ok := errs.([]any); !ok || len(list) != 0 { + return errors.New("native canary backup deletion reported errors") + } + } + } + // Successful native deletion removes its own request after deleting the + // backup and all volume-data snapshots. Both absences are required. + if current != nil || dbr != nil { + return errors.New("native canary backup retirement has not finished") + } + return nil + }); err != nil { + return err + } + if err = p.requireNativeRestoreGone(ctx, restore); err != nil { + return err + } + cm, err := p.c.get(ctx, "configmap", "velero", name, false) + if err != nil || text(cm, "metadata", "uid") != p.record.Resources["Modifier"].UID || text(cm, "data", "rules.yaml") != text(modifier, "data", "rules.yaml") { + return errors.New("canary restore modifier identity changed") + } + return removeScratch(ctx, p.c, p.source, "configmap", "velero", "/api/v1/namespaces/velero/configmaps/"+name, cm) +} diff --git a/scripts/migrate-longhorn-v2/canary_cleanup_test.go b/scripts/migrate-longhorn-v2/canary_cleanup_test.go new file mode 100644 index 0000000000..cd5f183a0c --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_cleanup_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" +) + +func TestNativeRestoreRetirementRequiresAbsenceWithoutDeletingAnything(t *testing.T) { + for _, remaining := range []string{"", "uid-restore", "replacement"} { + original := resource("restore") + c := client{context: "admin@prod", run: func(_ context.Context, _ []byte, program string, args ...string) ([]byte, error) { + request := strings.Join(args, " ") + if program != "kubectl" || !strings.Contains(request, " get restores.velero.io restore ") { + t.Fatal("native restore cleanup issued something other than its fixed read") + } + if remaining == "" { + if !strings.Contains(request, "--ignore-not-found") { + return nil, errors.New("native deletion already removed the Restore") + } + return nil, nil + } + current := resource("restore") + current["metadata"].(object)["uid"] = remaining + return json.Marshal(current) + }} + p := canaryRun{c: c} + err := p.requireNativeRestoreGone(t.Context(), original) + if (err == nil) != (remaining == "") { + t.Fatalf("remaining identity %s: %v", remaining, err) + } + } +} diff --git a/scripts/migrate-longhorn-v2/canary_cli_test.go b/scripts/migrate-longhorn-v2/canary_cli_test.go new file mode 100644 index 0000000000..0afa043a53 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_cli_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "encoding/json" + "os" + "os/exec" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestCSICanaryCanBeRenderedWithoutProductionAuthorization(t *testing.T) { + cmd := exec.CommandContext(t.Context(), "go", "run", ".", "--render-csi-canary", "1234") + cmd.Env = append(os.Environ(), "GITHUB_ACTIONS=false", "LONGHORN_V2_CONFIRM=", "LONGHORN_V2_OPERATION=") + output, err := cmd.Output() + if err != nil { + t.Fatalf("offline CSI rendering failed: %v", err) + } + var rendered struct{ Items []object } + if json.Unmarshal(output, &rendered) != nil || len(rendered.Items) != 12 || text(rendered.Items[11], "kind") != "VolumeSnapshotClass" || value(rendered.Items[9], "spec", "snapshotMoveData") != true { + t.Fatal("offline CSI output is not the native Data Mover canary") + } +} + +func TestWorkflowRequiresTheCSIConfirmationAndRejectsTheFSBConfirmation(t *testing.T) { + data, err := os.ReadFile("../../.github/workflows/prepare-longhorn-v2-node.yaml") + if err != nil { + t.Fatal(err) + } + var workflow struct { + Jobs map[string]struct { + Steps []struct{ Run string } + } + } + if err = yaml.Unmarshal(data, &workflow); err != nil { + t.Fatal(err) + } + guard := workflow.Jobs["prepare"].Steps[0].Run + for _, confirmation := range []string{"verify-longhorn-v2-csi-restore", "verify-longhorn-v2-restore", "prepare-longhorn-v2-prod-worker-1"} { + cmd := exec.CommandContext(t.Context(), "bash", "-c", guard) + cmd.Env = append(os.Environ(), "DISPATCH_REF=refs/heads/main", "GITHUB_RUN_ATTEMPT=1", "LONGHORN_V2_OPERATION=canary-csi-v2", "LONGHORN_V2_CONFIRM="+confirmation) + if err = cmd.Run(); (err == nil) != strings.Contains(confirmation, "-csi-") { + t.Fatalf("CSI workflow confirmation %s: %v", confirmation, err) + } + } +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_assets.go b/scripts/migrate-longhorn-v2/canary_csi_assets.go new file mode 100644 index 0000000000..b09d9125cf --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_assets.go @@ -0,0 +1,21 @@ +package main + +// csiCanaryResources selects native snapshot data movement for the same +// disposable byte-recovery probe and gives it a run-owned snapshot class. +func csiCanaryResources(run string) ([]object, error) { + assets, err := canaryResources(run) + if err != nil { + return nil, err + } + delete(assets[4]["metadata"].(object), "annotations") + class := "lh-csi-canary-" + run + assets[9]["metadata"].(object)["annotations"] = object{"velero.io/csi-volumesnapshot-class_driver.longhorn.io": class} + spec := assets[9]["spec"].(object) + spec["snapshotVolumes"], spec["snapshotMoveData"], spec["dataMover"] = true, true, "velero" + assets[10]["spec"].(object)["restorePVs"] = true + return append(assets, object{ + "apiVersion": "snapshot.storage.k8s.io/v1", "kind": "VolumeSnapshotClass", + "metadata": object{"name": class, "labels": object{"storage.platform.devantler.tech/canary-run": run}}, + "driver": "driver.longhorn.io", "deletionPolicy": "Delete", "parameters": object{"type": "snap"}, + }), nil +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_assets_test.go b/scripts/migrate-longhorn-v2/canary_csi_assets_test.go new file mode 100644 index 0000000000..44b32ae926 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_assets_test.go @@ -0,0 +1,32 @@ +package main + +import "testing" + +func TestCSICanaryUsesItsOwnDisposableSnapshotAndNativeDataMover(t *testing.T) { + assets, err := csiCanaryResources("1234") + if err != nil || len(assets) != 12 { + t.Fatalf("CSI fixture invalid: %v", err) + } + pod, backup, restore, class := assets[4], assets[9], assets[10], assets[11] + if value(pod, "metadata", "annotations", "backup.velero.io/backup-volumes") != nil || + value(backup, "spec", "snapshotVolumes") != true || value(backup, "spec", "snapshotMoveData") != true || text(backup, "spec", "dataMover") != "velero" || + value(backup, "spec", "defaultVolumesToFsBackup") != false || value(restore, "spec", "restorePVs") != true { + t.Fatal("CSI data movement was replaced by FSB or a metadata-only restore") + } + if text(class, "kind") != "VolumeSnapshotClass" || text(class, "metadata", "name") != "lh-csi-canary-1234" || text(class, "driver") != "driver.longhorn.io" || text(class, "deletionPolicy") != "Delete" || text(class, "parameters", "type") != "snap" || + text(backup, "metadata", "annotations", "velero.io/csi-volumesnapshot-class_driver.longhorn.io") != text(class, "metadata", "name") { + t.Fatal("backup could select an original historical snapshot or a default snapshot class") + } + if value(class, "metadata", "labels", "velero.io/csi-volumesnapshot-class") != nil || value(class, "metadata", "annotations", "snapshot.storage.kubernetes.io/is-default-class") != nil { + t.Fatal("disposable canary changed global snapshot-class selection") + } + for _, i := range []int{2, 7} { + if text(assets[i], "reclaimPolicy") != "Retain" { + t.Fatal("source or restored data class lost Retain") + } + } + fsb, err := canaryResources("1234") + if err != nil || value(fsb[9], "spec", "snapshotVolumes") != false || value(fsb[9], "spec", "snapshotMoveData") != nil { + t.Fatal("FSB transport was changed by CSI mode") + } +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_binding.go b/scripts/migrate-longhorn-v2/canary_csi_binding.go new file mode 100644 index 0000000000..a54e833931 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_binding.go @@ -0,0 +1,53 @@ +package main + +import ( + "errors" + "strings" +) + +type csiScratchEvidence struct { + Claim, PV, Volume benchmarkResource + OperationUID, NativeSource string +} + +// Accepts observed watch objects, including deletion events. These identities +// are proof for later absence checks, never authority to delete by name. +func verifyCSIScratchBinding(claim, pv, volume, operation, sourceVolume object, class string) (csiScratchEvidence, error) { + proof := csiScratchEvidence{Claim: benchmarkIdentity(claim), PV: benchmarkIdentity(pv), Volume: benchmarkIdentity(volume), OperationUID: text(operation, "metadata", "uid")} + kind := text(operation, "kind") + if proof.OperationUID == "" || (kind != "DataUpload" && kind != "DataDownload") || text(operation, "metadata", "namespace") != "velero" || proof.Claim.Namespace != "velero" || + proof.Claim.UID == "" || proof.PV.UID == "" || proof.Volume.UID == "" || proof.PV.Namespace != "" || proof.Volume.Namespace != "longhorn-system" || text(sourceVolume, "metadata", "uid") == "" { + return proof, errors.New("CSI scratch resource identity or operation scope is incomplete") + } + owned := false + refs, _ := value(claim, "metadata", "ownerReferences").([]any) + for _, raw := range refs { + r, _ := raw.(object) + if text(r, "apiVersion") == "velero.io/v2alpha1" && text(r, "kind") == kind && text(r, "name") == text(operation, "metadata", "name") && text(r, "uid") == proof.OperationUID && value(r, "controller") == true { + owned = true + } + } + policy := text(pv, "spec", "persistentVolumeReclaimPolicy") + if !owned || class == "" || text(claim, "spec", "storageClassName") != class || text(pv, "spec", "storageClassName") != class || (policy != "Retain" && policy != "Delete") || + text(claim, "spec", "volumeName") != proof.PV.Name || text(pv, "spec", "claimRef", "name") != proof.Claim.Name || text(pv, "spec", "claimRef", "namespace") != "velero" || text(pv, "spec", "claimRef", "uid") != proof.Claim.UID || + text(pv, "spec", "csi", "driver") != "driver.longhorn.io" || text(pv, "spec", "csi", "volumeHandle") != proof.Volume.Name || proof.Volume.UID == text(sourceVolume, "metadata", "uid") { + return proof, errors.New("CSI scratch PVC, PV and independent Longhorn backend are not bound to this operation") + } + engine := "v1" + if kind == "DataDownload" { + engine = "v2" + } + if text(volume, "spec", "dataEngine") != engine { + return proof, errors.New("CSI scratch backend uses the wrong data engine") + } + proof.NativeSource = text(volume, "spec", "dataSource") + if kind == "DataUpload" { + snapshot, ok := strings.CutPrefix(proof.NativeSource, "snap://"+text(sourceVolume, "metadata", "name")+"/") + if !ok || !kubeName.MatchString(snapshot) { + return proof, errors.New("temporary CSI backup clone is not a snapshot of the exact canary source") + } + } else if proof.NativeSource != "" { + return proof, errors.New("CSI recovery target was pre-populated from another data source") + } + return proof, nil +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_binding_test.go b/scripts/migrate-longhorn-v2/canary_csi_binding_test.go new file mode 100644 index 0000000000..a84a2a9e2a --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_binding_test.go @@ -0,0 +1,53 @@ +package main + +import "testing" + +func csiScratchFixture(kind string) (object, object, object, object, object, string) { + _, _, _, op := csiTransferFixture("Backup") + op["kind"] = kind + claim, pv, backend, source := resource("transfer"), resource("temporary-pv"), resource("temporary-backend"), resource("source-backend") + claim["metadata"].(object)["namespace"] = "velero" + claim["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "velero.io/v2alpha1", "kind": kind, "name": text(op, "metadata", "name"), "uid": text(op, "metadata", "uid"), "controller": true}} + class, engine := "lh-canary-v1-1234", "v1" + if kind == "DataDownload" { + class, engine = "lh-canary-v2-1234", "v2" + } + claim["spec"] = object{"volumeName": text(pv, "metadata", "name"), "storageClassName": class} + pv["spec"] = object{"claimRef": object{"name": text(claim, "metadata", "name"), "namespace": "velero", "uid": text(claim, "metadata", "uid")}, "storageClassName": class, "persistentVolumeReclaimPolicy": "Retain", "csi": object{"driver": "driver.longhorn.io", "volumeHandle": text(backend, "metadata", "name")}} + backend["metadata"].(object)["namespace"] = "longhorn-system" + backend["spec"] = object{"dataEngine": engine} + if kind == "DataUpload" { + backend["spec"].(object)["dataSource"] = "snap://source-backend/native-snapshot" + } + return claim, pv, backend, op, source, class +} + +func TestCSIScratchProofRequiresTheObservedOwnedClaimAndExactBackend(t *testing.T) { + for _, kind := range []string{"DataUpload", "DataDownload"} { + claim, pv, volume, op, source, class := csiScratchFixture(kind) + proof, err := verifyCSIScratchBinding(claim, pv, volume, op, source, class) + if err != nil || proof.Claim.UID == "" || proof.PV.UID == "" || proof.Volume.UID == "" { + t.Fatalf("valid owned scratch rejected: %v", err) + } + for name, change := range map[string]func(object, object, object){ + "claim belongs to another operation": func(c, _, _ object) { value(c, "metadata", "ownerReferences").([]any)[0].(object)["uid"] = "foreign" }, + "PV rebound to another claim": func(_, p, _ object) { value(p, "spec", "claimRef").(object)["uid"] = "replacement" }, + "backend replaced": func(_, _, v object) { v["metadata"].(object)["name"] = "other-backend" }, + "another storage driver": func(_, p, _ object) { value(p, "spec", "csi").(object)["driver"] = "another-driver" }, + "wrong data engine": func(_, _, v object) { v["spec"].(object)["dataEngine"] = "other" }, + } { + t.Run(kind+"/"+name, func(t *testing.T) { + c, p, v, o, s, sc := csiScratchFixture(kind) + change(c, p, v) + if _, err := verifyCSIScratchBinding(c, p, v, o, s, sc); err == nil { + t.Fatal("unowned temporary storage accepted") + } + }) + } + } + c, p, v, o, s, sc := csiScratchFixture("DataUpload") + v["spec"].(object)["dataSource"] = "snap://original-historical-volume/do-not-delete" + if _, err := verifyCSIScratchBinding(c, p, v, o, s, sc); err == nil { + t.Fatal("a snapshot of another source volume was accepted") + } +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_capture.go b/scripts/migrate-longhorn-v2/canary_csi_capture.go new file mode 100644 index 0000000000..01a0c408b6 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_capture.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/url" + "os/exec" + "sync" +) + +// A completed DataUpload can outlive its temporary PVC, PV and backend. Start +// gap-free API watches before creating the backup, and retain their observed +// identities. This evidence never authorizes deletion of an observed name. +type csiCapture struct { + mu sync.Mutex + run string + objects map[string]map[string]object + err error + cancel context.CancelFunc + wg sync.WaitGroup +} + +var csiCollections = map[string]string{ + "persistentvolumeclaims": "/api/v1/namespaces/velero/persistentvolumeclaims", + "persistentvolumes": "/api/v1/persistentvolumes", + "volumes.longhorn.io": "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes", +} + +// csiInventory anchors the following fixed-scope watch to a complete native +// collection snapshot rather than kubectl's synthetic List printer. +func csiInventory(ctx context.Context, c client, query string) ([]object, string, error) { + path, ok := csiCollections[query] + if !ok { + return nil, "", errors.New("CSI inventory is outside its fixed scope") + } + // kubectl's ordinary JSON printer synthesizes kind: List with an empty + // resourceVersion. Read the native collection so its snapshot can anchor + // the following watch without missing transient provisioning resources. + data, err := c.kubectl(ctx, nil, "get", "--raw="+path+"?limit=500") + if err != nil { + return nil, "", err + } + var list object + if json.Unmarshal(data, &list) != nil || text(list, "metadata", "resourceVersion") == "" || len(text(list, "metadata", "resourceVersion")) > 256 { + return nil, "", errors.New("CSI watch cannot start without a complete native list resource version") + } + items, err := decodeList(data) + return items, text(list, "metadata", "resourceVersion"), err +} + +// newCSICapture creates a run-scoped inventory for the three storage collections. +func newCSICapture(run string) *csiCapture { + return &csiCapture{run: run, objects: map[string]map[string]object{ + "persistentvolumeclaims": {}, "persistentvolumes": {}, "volumes.longhorn.io": {}, + }} +} + +// observe retains bounded original identities, including short-lived bound +// scratch objects that native transfer cleanup may remove before the next poll. +func (p *csiCapture) observe(query string, o object) error { + p.mu.Lock() + defer p.mu.Unlock() + if p.err != nil { + return p.err + } + uid := text(o, "metadata", "uid") + if uid == "" || text(o, "metadata", "resourceVersion") == "" { + return errors.New("CSI watch object has no complete identity") + } + class := text(o, "spec", "storageClassName") + if query != "volumes.longhorn.io" { + if class != "lh-canary-v1-"+p.run && class != "lh-canary-v2-"+p.run { + return nil + } + if query == "persistentvolumeclaims" && (text(o, "metadata", "namespace") != "velero" || text(o, "spec", "volumeName") == "") { + return nil + } + if query == "persistentvolumes" && text(o, "spec", "claimRef", "namespace") != "velero" { + return nil + } + } + items, ok := p.objects[query] + if !ok { + return errors.New("CSI capture request is outside its fixed scope") + } + if len(items) >= 512 { + return errors.New("CSI capture exceeded its bounded identity inventory") + } + // Keep the original bound graph, including a clone's original dataSource, + // before native cleanup or download rebinding changes those fields. + if _, exists := items[uid]; !exists { + items[uid] = o + } + return nil +} + +// proof joins exactly one owned CSI scratch graph and rejects PV name reuse; +// these observations are evidence and never deletion authority. +func (p *csiCapture) proof(operation, source object, class string) (csiScratchEvidence, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.err != nil { + return csiScratchEvidence{}, p.err + } + var proofs []csiScratchEvidence + for _, claim := range p.objects["persistentvolumeclaims"] { + for _, pv := range p.objects["persistentvolumes"] { + if text(pv, "metadata", "name") != text(claim, "spec", "volumeName") { + continue + } + for _, volume := range p.objects["volumes.longhorn.io"] { + if text(volume, "metadata", "name") != text(pv, "spec", "csi", "volumeHandle") { + continue + } + if proof, err := verifyCSIScratchBinding(claim, pv, volume, operation, source, class); err == nil { + proofs = append(proofs, proof) + } + } + } + } + if len(proofs) != 1 { + return csiScratchEvidence{}, errors.New("the CSI operation's independent storage graph was not observed exactly once") + } + proof := proofs[0] + for _, pv := range p.objects["persistentvolumes"] { + if text(pv, "metadata", "name") == proof.PV.Name && text(pv, "metadata", "uid") != proof.PV.UID { + return proof, errors.New("the CSI temporary PV name was reused") + } + } + return proof, nil +} + +// readCSIWatch rejects stream gaps, incomplete objects and unsupported events +// instead of accepting a partial identity history. +func readCSIWatch(input io.Reader, observe func(object) error) error { + decoder := json.NewDecoder(input) + for { + var event struct { + Type string `json:"type"` + Object object `json:"object"` + } + if decoder.Decode(&event) != nil { + return errors.New("CSI watch ended before the recovery proof completed") + } + if event.Type == "BOOKMARK" { + continue + } + if event.Type != "ADDED" && event.Type != "MODIFIED" && event.Type != "DELETED" { + return errors.New("CSI watch reported an error or unsupported event") + } + if text(event.Object, "metadata", "uid") == "" || text(event.Object, "metadata", "resourceVersion") == "" { + return errors.New("CSI watch event identity is incomplete") + } + if err := observe(event.Object); err != nil { + return err + } + } +} + +// start lists then streams each fixed collection from its resource version; +// the parent deadline bounds native watches and any failed stream poisons proof. +func (p *csiCapture) start(ctx context.Context, c client) error { + watchCtx, cancel := context.WithCancel(ctx) + p.cancel = cancel + for _, query := range []string{"persistentvolumeclaims", "persistentvolumes", "volumes.longhorn.io"} { + items, rv, err := csiInventory(watchCtx, c, query) + if err != nil { + p.stop() + return err + } + for _, item := range items { + if err = p.observe(query, item); err != nil { + p.stop() + return err + } + } + args := []string{"--context=" + c.context, "--request-timeout=0", "get", "--raw=" + csiCollections[query] + "?watch=true&allowWatchBookmarks=true&resourceVersion=" + url.QueryEscape(rv)} + cmd := exec.CommandContext(watchCtx, "kubectl", args...) + cmd.Stderr = io.Discard + stdout, err := cmd.StdoutPipe() + if err != nil { + p.stop() + return err + } + if err = cmd.Start(); err != nil { + p.stop() + return errors.New("CSI identity watch could not start") + } + p.wg.Go(func() { + err := readCSIWatch(stdout, func(o object) error { return p.observe(query, o) }) + _ = cmd.Process.Kill() + _ = cmd.Wait() + if watchCtx.Err() == nil { + p.mu.Lock() + if p.err == nil { + p.err = err + } + p.mu.Unlock() + } + }) + } + return nil +} + +// stop cancels and joins only the native watch processes owned by this capture. +func (p *csiCapture) stop() { + if p.cancel != nil { + p.cancel() + } + p.wg.Wait() +} + +// health returns any observed stream failure after synchronized shutdown. +func (p *csiCapture) health() error { + p.mu.Lock() + defer p.mu.Unlock() + return p.err +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_capture_test.go b/scripts/migrate-longhorn-v2/canary_csi_capture_test.go new file mode 100644 index 0000000000..cd7af2a9de --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_capture_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" +) + +func TestCSIInventoryPreservesNativeListResourceVersion(t *testing.T) { + for _, continuation := range []string{"", "unread-page"} { + calls := 0 + c := client{context: "admin@prod", run: func(_ context.Context, _ []byte, program string, args ...string) ([]byte, error) { + calls++ + if program != "kubectl" || strings.Join(args, " ") != "--context=admin@prod --request-timeout=20s get --raw=/api/v1/persistentvolumes?limit=500" { + t.Fatal("watch inventory did not read the fixed native collection") + } + return []byte(`{"apiVersion":"v1","kind":"PersistentVolumeList","metadata":{"resourceVersion":"12345","continue":"` + continuation + `"},"items":[]}`), nil + }} + items, rv, err := csiInventory(t.Context(), c, "persistentvolumes") + if continuation == "" && (err != nil || rv != "12345" || len(items) != 0 || calls != 1) { + t.Fatalf("lost native list version: rv=%s err=%v", rv, err) + } + if continuation != "" && err == nil { + t.Fatal("a partial list was accepted as a gap-free watch origin") + } + } +} + +func TestCSICaptureKeepsTheOwnedBindingAfterNativeCleanup(t *testing.T) { + for _, kind := range []string{"DataUpload", "DataDownload"} { + claim, pv, volume, op, source, class := csiScratchFixture(kind) + capture := newCSICapture("1234") + // The backend may arrive before its PV and claim. Native cleanup may + // remove all three before a backup's completion becomes observable. + for _, event := range []struct { + query string + value object + }{ + {"volumes.longhorn.io", volume}, {"persistentvolumes", pv}, {"persistentvolumeclaims", claim}, + } { + if err := capture.observe(event.query, event.value); err != nil { + t.Fatal(err) + } + } + proof, err := capture.proof(op, source, class) + if err != nil || proof.Volume.UID != text(volume, "metadata", "uid") { + t.Fatalf("owned transient binding lost: %v", err) + } + encoded, _ := json.Marshal(pv) + var foreign object + if err := json.Unmarshal(encoded, &foreign); err != nil { + t.Fatal(err) + } + foreign["metadata"].(object)["uid"] = "replacement-pv" + foreign["spec"].(object)["csi"].(object)["volumeHandle"] = "unrelated-volume" + if err := capture.observe("persistentvolumes", foreign); err != nil { + t.Fatal(err) + } + if _, err := capture.proof(op, source, class); err == nil { + t.Fatal("a replaced or ambiguous PV name was accepted") + } + } +} + +func TestCSIWatchRefusesGapsAndMalformedEvents(t *testing.T) { + for _, input := range []string{ + `{"type":"ERROR","object":{"code":410}}`, + `{"type":"ADDED","object":{"metadata":{"name":"claim"}}}`, + `{"type":"MODIFIED","object":`, + `{"type":"UNRECOGNIZED","object":{}}`, + "", + } { + if err := readCSIWatch(strings.NewReader(input), func(object) error { return nil }); err == nil { + t.Fatalf("incomplete watch accepted: %q", input) + } + } +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_native_test.go b/scripts/migrate-longhorn-v2/canary_csi_native_test.go new file mode 100644 index 0000000000..7d6b01fe42 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_native_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" +) + +// Exercise the actual kubectl raw collection/watch transport, including its +// streaming output. The API fixture has no credentials and is not a cluster. +func TestCSIIdentityCaptureThroughNativeKubectl(t *testing.T) { + if os.Getenv("LONGHORN_V2_NATIVE_TEST") != "1" { + t.Skip("set LONGHORN_V2_NATIVE_TEST=1 with the repository-approved kubectl") + } + claim, pv, volume, op, source, class := csiScratchFixture("DataUpload") + objects := map[string]object{ + "/api/v1/namespaces/velero/persistentvolumeclaims": claim, + "/api/v1/persistentvolumes": pv, + "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes": volume, + } + watching := make(chan struct{}, 3) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + o, ok := objects[r.URL.Path] + if !ok || r.Method != http.MethodGet { + http.Error(w, "unexpected fixture request", http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("watch") != "true" { + if r.URL.Query().Get("limit") != "500" { + http.Error(w, "missing collection bound", http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(object{"apiVersion": "v1", "kind": "List", "metadata": object{"resourceVersion": "12345"}, "items": []any{}}) + return + } + if r.URL.Query().Get("resourceVersion") != "12345" || r.URL.Query().Get("allowWatchBookmarks") != "true" { + http.Error(w, "watch lost its collection snapshot", http.StatusBadRequest) + return + } + _ = json.NewEncoder(w).Encode(object{"type": "ADDED", "object": o}) + w.(http.Flusher).Flush() + watching <- struct{}{} + <-r.Context().Done() + })) + defer server.Close() + config := fmt.Sprintf("apiVersion: v1\nkind: Config\nclusters:\n- name: fixture\n cluster:\n server: %s\ncontexts:\n- name: admin@prod\n context:\n cluster: fixture\ncurrent-context: admin@prod\n", server.URL) + path := filepath.Join(t.TempDir(), "kubeconfig") + if err := os.WriteFile(path, []byte(config), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("KUBECONFIG", path) + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + capture := newCSICapture("1234") + if err := capture.start(ctx, client{run: execute, context: "admin@prod"}); err != nil { + t.Fatal(err) + } + defer capture.stop() + for range 3 { + select { + case <-watching: + case <-ctx.Done(): + t.Fatal("native watches did not stream all three storage identities") + } + } + deadline := time.NewTimer(5 * time.Second) + defer deadline.Stop() + tick := time.NewTicker(10 * time.Millisecond) + defer tick.Stop() + for { + if proof, err := capture.proof(op, source, class); err == nil && proof.Volume.UID == text(volume, "metadata", "uid") { + return + } + select { + case <-tick.C: + case <-deadline.C: + t.Fatal("native watch events did not produce the bound CSI storage proof") + } + } +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_runtime.go b/scripts/migrate-longhorn-v2/canary_csi_runtime.go new file mode 100644 index 0000000000..0fe30fa14e --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_runtime.go @@ -0,0 +1,183 @@ +package main + +import ( + "context" + "errors" + "strings" +) + +// canaryJournalName separates filesystem and CSI proof ownership at fixed names. +func canaryJournalName(transport string) (string, error) { + switch transport { + case "filesystem": + return "longhorn-v2-canary", nil + case "csi": + return "longhorn-v2-csi-canary", nil + default: + return "", errors.New("canary transport is outside the reviewed procedure") + } +} + +// verifyCSIRestoredBinding requires the destination claim to adopt the exact +// independent backend downloaded by the native data mover. +func verifyCSIRestoredBinding(proof csiScratchEvidence, claim, pv, volume object) error { + if proof.OperationUID == "" || proof.Claim.UID == "" || proof.PV.UID == "" || proof.Volume.UID == "" || + text(claim, "metadata", "namespace") == "velero" || text(claim, "metadata", "uid") == "" || text(claim, "metadata", "uid") == proof.Claim.UID || + text(pv, "metadata", "name") != proof.PV.Name || text(pv, "metadata", "uid") != proof.PV.UID || + text(volume, "metadata", "name") != proof.Volume.Name || text(volume, "metadata", "uid") != proof.Volume.UID { + return errors.New("the restored claim did not adopt the exact downloaded PV and independent backend") + } + return nil +} + +// transferCSI joins the completed native operation to its snapshot class, +// source claim, captured scratch storage and parent backup or restore. +func (p *canaryRun) transferCSI(ctx context.Context, desired object, expectedSnapshot string) (object, string, error) { + kind := text(desired, "kind") + query := "datauploads.velero.io" + if kind == "Restore" { + query = "datadownloads.velero.io" + } + created, err := p.create(ctx, kind, desired) + if err != nil { + return nil, "", err + } + var parent object + var snapshot string + err = waitReady(ctx, func(ctx context.Context) error { + if err = p.csi.health(); err != nil { + return err + } + parent, err = p.c.get(ctx, strings.ToLower(kind)+"s.velero.io", "velero", text(created, "metadata", "name"), false) + if err != nil || text(parent, "metadata", "uid") != text(created, "metadata", "uid") { + return errors.New("CSI backup or restore identity changed") + } + if err = cleanVeleroResult(parent); err != nil { + return err + } + class, readErr := p.c.get(ctx, "volumesnapshotclasses.snapshot.storage.k8s.io", "", text(p.snapshotClass, "metadata", "name"), false) + if readErr != nil || text(class, "metadata", "uid") != text(p.snapshotClass, "metadata", "uid") || text(class, "driver") != "driver.longhorn.io" || text(class, "deletionPolicy") != "Delete" || text(class, "parameters", "type") != "snap" { + return errors.New("the disposable CSI snapshot class changed") + } + sourceClaim, readErr := p.c.get(ctx, "persistentvolumeclaim", text(p.sourceClaim, "metadata", "namespace"), text(p.sourceClaim, "metadata", "name"), false) + if readErr != nil || text(sourceClaim, "metadata", "uid") != text(p.sourceClaim, "metadata", "uid") { + return errors.New("the CSI source claim was replaced") + } + var targetClaim object + if kind == "Restore" { + targetClaim, readErr = p.c.get(ctx, "persistentvolumeclaim", "longhorn-canary-v2-"+p.run, "fio-data", false) + if readErr != nil { + return readErr + } + } + operations, readErr := p.c.list(ctx, query, "velero") + if readErr != nil { + return readErr + } + op, snap, verifyErr := verifyCSITransfer(operations, parent, sourceClaim, targetClaim, expectedSnapshot, text(sourceClaim, "spec", "storageClassName"), text(class, "metadata", "name")) + if verifyErr != nil { + return verifyErr + } + storageClass := text(sourceClaim, "spec", "storageClassName") + if kind == "Restore" { + storageClass = text(targetClaim, "spec", "storageClassName") + } + proof, verifyErr := p.csi.proof(op, p.sourceVolume, storageClass) + if verifyErr != nil { + return verifyErr + } + opKind := text(op, "kind") + if old := p.record.CSIScratch[opKind]; old.OperationUID != "" && old != proof { + return errors.New("the observed CSI operation storage graph changed") + } + p.record.CSIScratch[opKind] = proof + if verifyErr = p.remember(ctx, "csi/"+opKind, op); verifyErr != nil { + return verifyErr + } + snapshot = snap + return nil + }) + return parent, snapshot, err +} + +// requireCSIGone observes native scratch retirement without adopting or deleting +// a replacement that reused a recorded resource name. +func (p *canaryRun) requireCSIGone(ctx context.Context, kind string, identity benchmarkResource) error { + if identity.Name == "" || identity.UID == "" { + return errors.New("CSI cleanup lacks a captured resource identity") + } + current, err := p.c.get(ctx, kind, identity.Namespace, identity.Name, true) + if err != nil { + return err + } + if current != nil { + if text(current, "metadata", "uid") != identity.UID { + return errors.New("a CSI scratch name was reused; do not adopt or delete its replacement") + } + return errors.New("native CSI scratch cleanup has not completed") + } + return nil +} + +// waitCSIUploadGone requires native retirement of all captured upload storage +// before beginning download, while keeping the identity streams healthy. +func (p *canaryRun) waitCSIUploadGone(ctx context.Context) error { + proof := p.record.CSIScratch["DataUpload"] + return waitReady(ctx, func(ctx context.Context) error { + if err := p.csi.health(); err != nil { + return err + } + for _, item := range []struct { + kind string + identity benchmarkResource + }{ + {"persistentvolumeclaim", proof.Claim}, {"persistentvolume", proof.PV}, {"volumes.longhorn.io", proof.Volume}, + } { + if err := p.requireCSIGone(ctx, item.kind, item.identity); err != nil { + return err + } + } + return nil + }) +} + +// waitCSIArtifactsGone verifies native operation and snapshot cleanup while +// retaining the downloaded backend for its separate owned cleanup. +func (p *canaryRun) waitCSIArtifactsGone(ctx context.Context) error { + return waitReady(ctx, func(ctx context.Context) error { + if err := p.csi.health(); err != nil { + return err + } + for _, kind := range []string{"DataUpload", "DataDownload"} { + if err := p.requireCSIGone(ctx, strings.ToLower(kind)+"s.velero.io", p.record.Resources["csi/"+kind]); err != nil { + return err + } + } + if err := p.requireCSIGone(ctx, "persistentvolumeclaim", p.record.CSIScratch["DataDownload"].Claim); err != nil { + return err + } + for _, scope := range []struct{ kind, namespace string }{ + {"volumesnapshots.snapshot.storage.k8s.io", "*"}, {"volumesnapshotcontents.snapshot.storage.k8s.io", ""}, + } { + items, err := p.c.list(ctx, scope.kind, scope.namespace) + if err != nil { + return err + } + for _, item := range items { + if text(item, "spec", "volumeSnapshotClassName") == text(p.snapshotClass, "metadata", "name") { + return errors.New("native CSI snapshot cleanup has not completed") + } + } + } + return nil + }) +} + +// removeCSISnapshotClass conditionally removes only this run's original class. +func (p *canaryRun) removeCSISnapshotClass(ctx context.Context) error { + current, err := p.c.get(ctx, "volumesnapshotclasses.snapshot.storage.k8s.io", "", text(p.snapshotClass, "metadata", "name"), false) + if err != nil || text(current, "metadata", "uid") != text(p.snapshotClass, "metadata", "uid") { + return errors.New("the CSI snapshot class was replaced; preserve it for diagnosis") + } + return removeScratch(ctx, p.c, p.source, "volumesnapshotclasses.snapshot.storage.k8s.io", "", "/apis/snapshot.storage.k8s.io/v1/volumesnapshotclasses/"+text(current, "metadata", "name"), current) +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_runtime_test.go b/scripts/migrate-longhorn-v2/canary_csi_runtime_test.go new file mode 100644 index 0000000000..47e2714ed1 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_runtime_test.go @@ -0,0 +1,48 @@ +package main + +import "testing" + +func TestCSIAndFilesystemCanariesHaveSeparateConfirmationsAndJournals(t *testing.T) { + env := dispatchFixture() + env["LONGHORN_V2_OPERATION"] = "canary-csi-v2" + env["LONGHORN_V2_CONFIRM"] = "verify-longhorn-v2-restore" + getenv := func(k string) string { return env[k] } + if authorizeCanary(getenv) == nil { + t.Fatal("filesystem confirmation authorized CSI data movement") + } + env["LONGHORN_V2_CONFIRM"] = "verify-longhorn-v2-csi-restore" + if err := authorizeCanary(getenv); err != nil { + t.Fatal(err) + } + if authorize(getenv, "prod-worker-1") == nil || authorizeTemporaryPreparation(getenv, "temporary-v1-1") == nil || authorizeHostReboot(getenv, "prod-worker-1") == nil { + t.Fatal("CSI canary authorized a reset or host reboot") + } + fsb, err := canaryJournalName("filesystem") + if err != nil { + t.Fatal(err) + } + csi, err := canaryJournalName("csi") + if err != nil || csi == fsb { + t.Fatal("one canary could overwrite the other transport's evidence") + } + if _, err = canaryJournalName("arbitrary-journal"); err == nil { + t.Fatal("transport selected an undeclared journal") + } +} + +func TestCSIRestoreKeepsTheDownloadedPVAndIndependentBackend(t *testing.T) { + claim, pv, volume, op, source, class := csiScratchFixture("DataDownload") + proof, err := verifyCSIScratchBinding(claim, pv, volume, op, source, class) + if err != nil { + t.Fatal(err) + } + claim["metadata"].(object)["uid"] = "restored-claim-uid" + claim["metadata"].(object)["namespace"] = "longhorn-canary-v2-1234" + if err = verifyCSIRestoredBinding(proof, claim, pv, volume); err != nil { + t.Fatal(err) + } + volume["metadata"].(object)["uid"] = "replacement-backend" + if verifyCSIRestoredBinding(proof, claim, pv, volume) == nil { + t.Fatal("a different backend substituted for the downloaded data") + } +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_transfer.go b/scripts/migrate-longhorn-v2/canary_csi_transfer.go new file mode 100644 index 0000000000..3517a09701 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_transfer.go @@ -0,0 +1,65 @@ +package main + +import "errors" + +// CSI DataUpload/DataDownload have different schemas from PVB/PVR. Bind the +// actual native fields instead of inferring a CSI recovery from FSB evidence. +func verifyCSITransfer(transfers []object, parent, sourceClaim, targetClaim object, expectedSnapshot, sourceClass, snapshotClass string) (object, string, error) { + kind, uid := text(parent, "kind"), text(parent, "metadata", "uid") + if uid == "" || text(parent, "metadata", "namespace") != "velero" || text(sourceClaim, "metadata", "uid") == "" || (kind != "Backup" && kind != "Restore") { + return nil, "", errors.New("CSI transfer parent or source claim identity is missing") + } + label, expectedKind := "velero.io/backup-uid", "DataUpload" + if kind == "Restore" { + label, expectedKind = "velero.io/restore-uid", "DataDownload" + if text(targetClaim, "metadata", "uid") == "" || text(targetClaim, "metadata", "uid") == text(sourceClaim, "metadata", "uid") { + return nil, "", errors.New("CSI restore requires an independent target claim") + } + } + var found object + snapshot := "" + for _, op := range transfers { + if !controlledBy(op, parent) && text(op, "metadata", "labels", label) != uid { + continue + } + if found != nil || !controlledBy(op, parent) || text(op, "metadata", "namespace") != "velero" || text(op, "kind") != expectedKind || text(op, "metadata", "labels", label) != uid || + text(op, "metadata", "uid") == "" || text(op, "spec", "backupStorageLocation") != "default" || text(op, "spec", "datamover") != "velero" || text(op, "spec", "sourceNamespace") != text(sourceClaim, "metadata", "namespace") { + return nil, "", errors.New("CSI transfer ownership, repository or source scope is ambiguous") + } + if err := cleanVeleroResult(op); err != nil { + return nil, "", err + } + done, doneOK := value(op, "status", "progress", "bytesDone").(float64) + total, totalOK := value(op, "status", "progress", "totalBytes").(float64) + if !doneOK || !totalOK || done < 134217728 || done != total { + return nil, "", errors.New("the CSI operation has not transferred the complete canary payload") + } + if kind == "Backup" { + if text(op, "metadata", "labels", "velero.io/pvc-uid") != text(sourceClaim, "metadata", "uid") || text(op, "spec", "snapshotType") != "CSI" || + text(op, "spec", "sourcePVC") != text(sourceClaim, "metadata", "name") || text(op, "spec", "csiSnapshot", "driver") != "driver.longhorn.io" || + text(op, "spec", "csiSnapshot", "storageClass") != sourceClass || text(op, "spec", "csiSnapshot", "snapshotClass") != snapshotClass || + !kubeName.MatchString(text(op, "spec", "csiSnapshot", "volumeSnapshot")) { + return nil, "", errors.New("CSI upload does not bind the exact source claim and disposable snapshot class") + } + snapshot = text(op, "status", "snapshotID") + } else { + if text(op, "spec", "targetVolume", "namespace") != text(targetClaim, "metadata", "namespace") || text(op, "spec", "targetVolume", "pvc") != text(targetClaim, "metadata", "name") || + text(op, "spec", "snapshotID") != expectedSnapshot || expectedSnapshot == "" { + return nil, "", errors.New("CSI download does not bind the exact repository snapshot and target claim") + } + size, ok := value(op, "spec", "snapshotSize").(float64) + if !ok || size < 134217728 { + return nil, "", errors.New("CSI download source size is incomplete") + } + snapshot = text(op, "spec", "snapshotID") + } + if snapshot == "" { + return nil, "", errors.New("CSI repository snapshot identity is missing") + } + found = op + } + if found == nil { + return nil, "", errors.New("exactly one completed CSI volume-data operation is required") + } + return found, snapshot, nil +} diff --git a/scripts/migrate-longhorn-v2/canary_csi_transfer_test.go b/scripts/migrate-longhorn-v2/canary_csi_transfer_test.go new file mode 100644 index 0000000000..84928cfe79 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_csi_transfer_test.go @@ -0,0 +1,84 @@ +package main + +import "testing" + +func csiTransferFixture(kind string) (object, object, object, object) { + parent := resource("longhorn-v2-canary-1234") + parent["kind"] = kind + parent["metadata"].(object)["namespace"] = "velero" + source, target := resource("fio-data"), resource("fio-data") + source["metadata"].(object)["namespace"], source["metadata"].(object)["uid"] = "longhorn-canary-v1-1234", "source-claim" + target["metadata"].(object)["namespace"], target["metadata"].(object)["uid"] = "longhorn-canary-v2-1234", "target-claim" + op := resource("transfer") + op["metadata"].(object)["namespace"] = "velero" + op["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "velero.io/v1", "kind": kind, "name": text(parent, "metadata", "name"), "uid": text(parent, "metadata", "uid"), "controller": true}} + op["status"] = object{"phase": "Completed", "completionTimestamp": "2026-10-29T22:00:00Z", "snapshotID": "repository-snapshot", "progress": object{"bytesDone": float64(134217800), "totalBytes": float64(134217800)}} + op["spec"] = object{"backupStorageLocation": "default", "datamover": "velero", "sourceNamespace": text(source, "metadata", "namespace")} + spec := op["spec"].(object) + if kind == "Backup" { + op["kind"] = "DataUpload" + op["metadata"].(object)["labels"] = object{"velero.io/backup-uid": text(parent, "metadata", "uid"), "velero.io/pvc-uid": text(source, "metadata", "uid")} + spec["snapshotType"], spec["sourcePVC"] = "CSI", "fio-data" + spec["csiSnapshot"] = object{"driver": "driver.longhorn.io", "storageClass": "lh-canary-v1-1234", "snapshotClass": "lh-csi-canary-1234", "volumeSnapshot": "native-snapshot"} + } else { + op["kind"] = "DataDownload" + op["metadata"].(object)["labels"] = object{"velero.io/restore-uid": text(parent, "metadata", "uid")} + spec["targetVolume"] = object{"pvc": "fio-data", "namespace": text(target, "metadata", "namespace")} + spec["snapshotID"], spec["snapshotSize"] = "repository-snapshot", float64(134217800) + delete(op["status"].(object), "snapshotID") + } + return parent, source, target, op +} + +func TestCSICanaryBindsBothNativeDataMovementOperations(t *testing.T) { + for _, kind := range []string{"Backup", "Restore"} { + parent, source, target, op := csiTransferFixture(kind) + found, snapshot, err := verifyCSITransfer([]object{op}, parent, source, target, "repository-snapshot", "lh-canary-v1-1234", "lh-csi-canary-1234") + if err != nil || found == nil || snapshot != "repository-snapshot" { + t.Fatalf("valid %s transfer rejected: %v", kind, err) + } + } +} + +func TestCSICanaryRejectsForeignOrIncompleteVolumeTransfers(t *testing.T) { + for _, kind := range []string{"Backup", "Restore"} { + for name, change := range map[string]func(object){ + "foreign owner": func(o object) { value(o, "metadata", "ownerReferences").([]any)[0].(object)["uid"] = "another-backup" }, + "foreign repository": func(o object) { o["spec"].(object)["backupStorageLocation"] = "another-destination" }, + "wrong source namespace": func(o object) { o["spec"].(object)["sourceNamespace"] = "production" }, + "incomplete payload": func(o object) { value(o, "status", "progress").(object)["bytesDone"] = float64(128) }, + "uncompleted operation": func(o object) { o["status"].(object)["phase"] = "InProgress" }, + } { + t.Run(kind+"/"+name, func(t *testing.T) { + parent, source, target, op := csiTransferFixture(kind) + change(op) + if _, _, err := verifyCSITransfer([]object{op}, parent, source, target, "repository-snapshot", "lh-canary-v1-1234", "lh-csi-canary-1234"); err == nil { + t.Fatal("unproven CSI recovery accepted") + } + }) + } + } + for _, change := range []func(object){ + func(o object) { o["metadata"].(object)["labels"].(object)["velero.io/pvc-uid"] = "replaced-claim" }, + func(o object) { value(o, "spec", "csiSnapshot").(object)["driver"] = "another-driver" }, + func(o object) { value(o, "spec", "csiSnapshot").(object)["snapshotClass"] = "original-history" }, + func(o object) { o["status"].(object)["snapshotID"] = "" }, + } { + parent, source, target, op := csiTransferFixture("Backup") + change(op) + if _, _, err := verifyCSITransfer([]object{op}, parent, source, target, "repository-snapshot", "lh-canary-v1-1234", "lh-csi-canary-1234"); err == nil { + t.Fatal("foreign upload accepted") + } + } + for _, change := range []func(object){ + func(o object) { value(o, "spec", "targetVolume").(object)["namespace"] = "production" }, + func(o object) { value(o, "spec", "targetVolume").(object)["pvc"] = "another-claim" }, + func(o object) { o["spec"].(object)["snapshotID"] = "another-point-in-time" }, + } { + parent, source, target, op := csiTransferFixture("Restore") + change(op) + if _, _, err := verifyCSITransfer([]object{op}, parent, source, target, "repository-snapshot", "lh-canary-v1-1234", "lh-csi-canary-1234"); err == nil { + t.Fatal("foreign download accepted") + } + } +} diff --git a/scripts/migrate-longhorn-v2/canary_identity_test.go b/scripts/migrate-longhorn-v2/canary_identity_test.go new file mode 100644 index 0000000000..be5b406ba4 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_identity_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "strings" + "testing" +) + +func canaryPodFixture() object { + assets, err := canaryProbeResources("scratch", "class", "1234", "v1") + if err != nil { + panic(err) + } + o := assets[4] + o["metadata"].(object)["uid"] = "probe" + o["spec"].(object)["nodeName"] = "prod-worker-3" + o["status"] = object{"conditions": readyConditions(), "containerStatuses": []any{object{"name": "fio", "imageID": strings.Split(fioImage, "@")[1]}}} + return o +} + +func TestCanaryCannotExecuteAgainstAReplacedOrUnreadyPod(t *testing.T) { + if err := verifyCanaryPod(canaryPodFixture(), "probe", "prod-worker-3"); err != nil { + t.Fatal(err) + } + for name, change := range map[string]func(object){ + "unready": func(p object) { p["status"].(object)["conditions"] = []any{} }, + "replaced": func(p object) { p["metadata"].(object)["uid"] = "replacement" }, + "wrong client": func(p object) { p["spec"].(object)["nodeName"] = "other" }, + "changed script": func(p object) { value(p, "spec", "containers").([]any)[0].(object)["args"] = []any{"other"} }, + "wrong executable": func(p object) { value(p, "status", "containerStatuses").([]any)[0].(object)["imageID"] = "unpinned" }, + "host users": func(p object) { p["spec"].(object)["hostUsers"] = true }, + } { + t.Run(name, func(t *testing.T) { + p := canaryPodFixture() + change(p) + if verifyCanaryPod(p, "probe", "prod-worker-3") == nil { + t.Fatal("canary may execute against an unverified consumer") + } + }) + } +} + +func TestCanaryBindingCannotAdoptAnUnrelatedClassOrClaim(t *testing.T) { + for _, phase := range []string{"v1", "v2"} { + group := "source" + if phase == "v2" { + group = "target" + } + class, claim := resource("class"), resource("fio-data") + claim["metadata"].(object)["labels"] = object{"velero.io/restore-name": "longhorn-v2-canary-1234", "velero.io/backup-name": "longhorn-v2-canary-1234"} + r := canaryRecord{Run: "1234", Resources: map[string]benchmarkResource{group + "/StorageClass": benchmarkIdentity(class), "source/PersistentVolumeClaim": benchmarkIdentity(claim)}} + if err := verifyCanaryBindingIdentity(phase, r, class, claim); err != nil { + t.Fatal(err) + } + class["metadata"].(object)["uid"] = "replacement" + if verifyCanaryBindingIdentity(phase, r, class, claim) == nil { + t.Fatal("foreign storage class adopted") + } + class["metadata"].(object)["uid"] = "uid-class" + if phase == "v1" { + claim["metadata"].(object)["uid"] = "replacement" + } else { + claim["metadata"].(object)["labels"] = object{} + } + if verifyCanaryBindingIdentity(phase, r, class, claim) == nil { + t.Fatal("unrelated source or restored claim adopted") + } + } +} diff --git a/scripts/migrate-longhorn-v2/canary_transfer.go b/scripts/migrate-longhorn-v2/canary_transfer.go new file mode 100644 index 0000000000..861fdb4d7e --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_transfer.go @@ -0,0 +1,59 @@ +package main + +import "errors" + +// controlledBy requires the exact native Velero parent as controller owner. +func controlledBy(o, parent object) bool { + refs, _ := value(o, "metadata", "ownerReferences").([]any) + for _, ref := range refs { + r, _ := ref.(object) + if text(r, "apiVersion") == "velero.io/v1" && text(r, "kind") == text(parent, "kind") && text(r, "name") == text(parent, "metadata", "name") && + text(r, "uid") == text(parent, "metadata", "uid") && text(r, "uid") != "" && value(r, "controller") == true { + return true + } + } + return false +} + +// verifyCanaryTransfer binds one complete Kopia payload transfer to the original +// pod and backup destination; restores must name that exact source snapshot. +func verifyCanaryTransfer(transfers []object, parent, pod object, expectedSnapshot, sourceNS string) (string, error) { + if text(parent, "metadata", "uid") == "" || text(pod, "metadata", "uid") == "" || (text(parent, "kind") != "Backup" && text(parent, "kind") != "Restore") { + return "", errors.New("canary transfer ownership is missing") + } + matched, snapshot := 0, "" + for _, transfer := range transfers { + owned := controlledBy(transfer, parent) + if !owned && text(transfer, "spec", "pod", "uid") != text(pod, "metadata", "uid") { + continue + } + matched++ + if !owned || text(transfer, "metadata", "namespace") != "velero" || text(transfer, "spec", "pod", "uid") != text(pod, "metadata", "uid") || + text(transfer, "spec", "pod", "namespace") != text(pod, "metadata", "namespace") || text(transfer, "spec", "pod", "name") != "fio" || + text(transfer, "spec", "volume") != "data" || text(transfer, "spec", "backupStorageLocation") != "default" || text(transfer, "spec", "repoIdentifier") == "" || text(transfer, "spec", "uploaderType") != "kopia" { + return "", errors.New("canary data transfer belongs to another pod, volume, repository or operation") + } + if err := cleanVeleroResult(transfer); err != nil { + return "", err + } + done, doneOK := value(transfer, "status", "progress", "bytesDone").(float64) + total, totalOK := value(transfer, "status", "progress", "totalBytes").(float64) + if !doneOK || !totalOK || done < 134217728 || done != total { + return "", errors.New("complete canary payload transfer is unproven") + } + snapshot = text(transfer, "status", "snapshotID") + if text(parent, "kind") == "Restore" { + snapshot = text(transfer, "spec", "snapshotID") + if snapshot != expectedSnapshot || text(transfer, "spec", "sourceNamespace") != sourceNS { + return "", errors.New("canary restore did not use the exact source snapshot") + } + } + if snapshot == "" { + return "", errors.New("canary volume-data snapshot identity is missing") + } + } + if matched != 1 { + return "", errors.New("exactly one completed canary volume-data transfer is required") + } + return snapshot, nil +} diff --git a/scripts/migrate-longhorn-v2/canary_transfer_test.go b/scripts/migrate-longhorn-v2/canary_transfer_test.go new file mode 100644 index 0000000000..42b986dd37 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_transfer_test.go @@ -0,0 +1,49 @@ +package main + +import "testing" + +func transferFixture(kind string) (object, object, object) { + parent, pod, transfer := resource("canary"), resource("fio"), resource("transfer") + parent["kind"] = kind + pod["metadata"].(object)["namespace"] = "longhorn-canary-v1-1234" + transfer["metadata"].(object)["namespace"] = "velero" + transfer["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "velero.io/v1", "kind": kind, "name": text(parent, "metadata", "name"), "uid": text(parent, "metadata", "uid"), "controller": true}} + transfer["spec"] = object{"pod": object{"name": "fio", "namespace": text(pod, "metadata", "namespace"), "uid": text(pod, "metadata", "uid")}, "volume": "data", "backupStorageLocation": "default", "repoIdentifier": "private-repository", "uploaderType": "kopia", "snapshotID": "snapshot", "sourceNamespace": "longhorn-canary-v1-1234"} + transfer["status"] = object{"phase": "Completed", "completionTimestamp": "2026-10-29T19:00:00Z", "snapshotID": "snapshot", "progress": object{"bytesDone": float64(134217800), "totalBytes": float64(134217800)}} + return parent, pod, transfer +} + +func TestCanaryRequiresTheExactCompletedDataTransfer(t *testing.T) { + for _, kind := range []string{"Backup", "Restore"} { + parent, pod, transfer := transferFixture(kind) + if _, err := verifyCanaryTransfer([]object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err != nil { + t.Fatal(err) + } + for name, mutate := range map[string]func(object){ + "unrelated parent": func(o object) { o["metadata"].(object)["ownerReferences"] = []any{} }, + "wrong pod uid": func(o object) { o["spec"].(object)["pod"].(object)["uid"] = "replacement" }, + "wrong volume": func(o object) { o["spec"].(object)["volume"] = "scratch" }, + "no repository": func(o object) { o["spec"].(object)["repoIdentifier"] = "" }, + "unproven progress": func(o object) { o["status"].(object)["progress"] = object{} }, + "incomplete transfer": func(o object) { o["status"].(object)["progress"].(object)["bytesDone"] = float64(32) }, + "failed transfer": func(o object) { o["status"].(object)["phase"] = "Failed" }, + "restic instead of kopia": func(o object) { o["spec"].(object)["uploaderType"] = "restic" }, + } { + t.Run(kind+"/"+name, func(t *testing.T) { + p, pod, transfer := transferFixture(kind) + mutate(transfer) + if _, err := verifyCanaryTransfer([]object{transfer}, p, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { + t.Fatal("backup metadata was mistaken for actual volume-data recovery") + } + }) + } + if _, err := verifyCanaryTransfer([]object{transfer, transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { + t.Fatal("ambiguous transfer inventory accepted") + } + } + p, pod, transfer := transferFixture("Restore") + transfer["spec"].(object)["snapshotID"] = "older-backup" + if _, err := verifyCanaryTransfer([]object{transfer}, p, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { + t.Fatal("restore from a different snapshot accepted") + } +} diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index 831a068118..e00b56e93e 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -11,6 +11,25 @@ import ( // main admits explicit migration operations through their distinct confirmations // and protected workflow identities before constructing any production client. func main() { + if len(os.Args) > 1 { + if len(os.Args) != 3 || (os.Args[1] != "--render-canary" && os.Args[1] != "--render-csi-canary") { + fmt.Fprintln(os.Stderr, "Only the explicit offline canary rendering arguments are supported.") + os.Exit(1) + } + render := canaryResources + if os.Args[1] == "--render-csi-canary" { + render = csiCanaryResources + } + assets, err := render(os.Args[2]) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err = json.NewEncoder(os.Stdout).Encode(object{"apiVersion": "v1", "kind": "List", "items": assets}); err != nil { + os.Exit(1) + } + return + } operation := os.Getenv("LONGHORN_V2_OPERATION") target := os.Getenv("LONGHORN_V2_NODE") var authorization error @@ -21,6 +40,10 @@ func main() { authorization = authorizeTemporaryPreparation(os.Getenv, target) case "benchmark-v1", "benchmark-v2": authorization = authorizeBenchmark(os.Getenv, operation[len("benchmark-"):]) + case "canary-v2", "canary-csi-v2": + authorization = authorizeCanary(os.Getenv) + case "register-temporary-v2-storage": + authorization = authorizeTemporaryActivation(os.Getenv) case "inventory-replicas": authorization = authorizeInventory(os.Getenv, target) case "inspect-host-memory": @@ -86,6 +109,27 @@ func main() { fmt.Fprintln(os.Stderr, err) os.Exit(1) } + if operation == "canary-v2" || operation == "canary-csi-v2" { + transport := "filesystem" + if operation == "canary-csi-v2" { + transport = "csi" + } + if err = runCanaryWithTransport(ctx, client{run: execute, context: "admin@prod", inventoryPools: pools}, os.Getenv("GITHUB_SHA"), os.Getenv("GITHUB_RUN_ID"), pools, transport); err != nil { + fmt.Fprintln(os.Stderr, "V2 recovery canary held:", err) + fmt.Fprintln(os.Stderr, "Preserve the canary journal and scratch resources; no application cutover is authorized.") + os.Exit(1) + } + return + } + if operation == "register-temporary-v2-storage" { + activation := temporaryV2Activation{c: client{run: execute, context: "admin@prod", inventoryPools: pools}, source: os.Getenv("GITHUB_SHA"), owner: os.Getenv("GITHUB_RUN_ID"), pools: pools} + if err = activation.run(ctx); err != nil { + fmt.Fprintln(os.Stderr, "Temporary V2 registration held:", err) + fmt.Fprintln(os.Stderr, "Preserve its identity journal and any partial registration for explicit recovery.") + os.Exit(1) + } + return + } if operation == "prepare-unused-temporary-worker" { c := client{run: execute, context: "admin@prod", inventoryPools: pools} nodes, readErr := c.list(ctx, "nodes", "") diff --git a/scripts/migrate-longhorn-v2/temporary_activation.go b/scripts/migrate-longhorn-v2/temporary_activation.go new file mode 100644 index 0000000000..defd842b0f --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "errors" + "regexp" + "strings" + "time" +) + +type temporaryV2Binding struct { + Selection string `json:"selection"` + Name string `json:"name"` + NodeUID string `json:"nodeUID"` + ProviderID string `json:"providerID"` + MachineUUID string `json:"machineUUID"` + BootID string `json:"bootID"` + OSImage string `json:"osImage"` + LonghornUID string `json:"longhornUID"` + EphemeralUUID string `json:"ephemeralUUID"` + RawPartitionUUID string `json:"rawPartitionUUID"` +} + +var errTemporaryActivation = errors.New("temporary raw storage identity or registration ownership is unproven") + +const temporaryActivationOwner = "storage.platform.devantler.tech/v2-activation" + +// authorizeTemporaryActivation requires a distinct confirmation for registering +// the fixed three-disk pool through the protected first-attempt main workflow. +func authorizeTemporaryActivation(getenv func(string) string) error { + if getenv("LONGHORN_V2_OPERATION") != "register-temporary-v2-storage" { + return errTemporaryActivation + } + return authorizeWorkflow(getenv, "register-longhorn-v2-temporary-storage") +} + +type temporaryV2Activation struct { + c client + source, owner string + pools map[string]migrationPool + reader inventoryReaderFactory +} + +// readBinding brackets the exact native partition reads with hardware and boot +// identity, then repeats the Kubernetes projection. An incomplete read or failed +// close cannot produce a binding for a subsequent registration mutation. +func (a temporaryV2Activation) readBinding(ctx context.Context, selection string, node, storage object, pool migrationPool) (binding temporaryV2Binding, err error) { + ctx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + if !commandContextActive(ctx) { + return temporaryV2Binding{}, errTemporaryActivation + } + address, err := verifyNode(node, journal{Target: text(node, "metadata", "name"), NodeUID: text(node, "metadata", "uid"), ProviderID: text(node, "spec", "providerID"), MachineUUID: text(node, "status", "nodeInfo", "systemUUID")}, true) + if err != nil || a.c.verifyHost(ctx, address, node) != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + factory := a.reader + if factory == nil { + factory = newInventoryReader + } + reader, closeReader, err := factory(ctx, address) + if err != nil || reader == nil || closeReader == nil { + return temporaryV2Binding{}, errTemporaryActivation + } + defer func() { + if closeReader() != nil || !commandContextActive(ctx) { + binding, err = temporaryV2Binding{}, errTemporaryActivation + } + }() + boot, err := reader.BootID(ctx) + if err != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + raw, err := a.c.talosVolume(ctx, address, "r-longhorn") + if err != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + ephemeral, err := a.c.talosVolume(ctx, address, "EPHEMERAL") + if err != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + before, err := projectTemporaryV2Binding(selection, node, storage, raw, ephemeral, pool, boot) + if err != nil { + return temporaryV2Binding{}, err + } + fresh, err := a.c.get(ctx, "node", "", before.Name, false) + if err != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + boot, err = reader.BootID(ctx) + if err != nil || a.c.verifyHost(ctx, address, fresh) != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + after, err := projectTemporaryV2Binding(selection, fresh, storage, raw, ephemeral, pool, boot) + if err != nil || before != after { + return temporaryV2Binding{}, errTemporaryActivation + } + return after, nil +} + +// temporaryRegistrationState accepts an empty fence or this journal's exact +// partial activation. A matching disk path without the journal UID is not owned. +func temporaryRegistrationState(storage object, journalUID string) (bool, error) { + owner := text(storage, "metadata", "annotations", temporaryActivationOwner) + if owner == "" && unusedStorageFenced(storage) { + return false, nil + } + if journalUID == "" || owner != journalUID || text(storage, "metadata", "labels", migrationEnabledLabel) != "true" || + value(storage, "spec", "allowScheduling") != true || value(storage, "spec", "evictionRequested") != false || !reviewedBlockDisks(value(storage, "spec", "disks")) { + return false, errTemporaryActivation + } + return true, nil +} + +// projectTemporaryV2Binding binds a fixed temporary selection to its pinned +// hardware and native encrypted partitions. Both legacy pool roles receive raw +// V2 storage; this profile never selects an original data-bearing worker. +func projectTemporaryV2Binding(selection string, node, storage, raw, ephemeral object, pool migrationPool, nativeBoot string) (temporaryV2Binding, error) { + b := temporaryV2Binding{Selection: selection, Name: text(node, "metadata", "name"), NodeUID: text(node, "metadata", "uid"), + ProviderID: text(node, "spec", "providerID"), MachineUUID: text(node, "status", "nodeInfo", "systemUUID"), + BootID: text(node, "status", "nodeInfo", "bootID"), OSImage: text(node, "status", "nodeInfo", "osImage"), + LonghornUID: text(storage, "metadata", "uid"), EphemeralUUID: text(ephemeral, "spec", "uuid"), RawPartitionUUID: text(raw, "spec", "partitionUUID")} + if !inventoryTemporarySelection.MatchString(selection) || !strings.HasPrefix(selection, "temporary-"+pool.role+"-") || + !inventoryTemporaryNode.MatchString(b.Name) || verifyMigrationNode(node, pool) != nil || verifyEncryptedLocalLayout(raw, ephemeral) != nil || + b.NodeUID == "" || !regexp.MustCompile(`^hcloud://[1-9][0-9]*$`).MatchString(b.ProviderID) || b.MachineUUID == "" || b.OSImage == "" || + !inventoryBoot.MatchString(b.BootID) || nativeBoot != b.BootID || !inventoryBoot.MatchString(strings.ToLower(b.RawPartitionUUID)) || b.RawPartitionUUID == b.EphemeralUUID || + text(node, "status", "nodeInfo", "architecture") != "amd64" || !hugepagesReady(node) || text(node, "status", "allocatable", "cpu") != "8" || + value(node, "spec", "unschedulable") == true || text(node, "metadata", "annotations", fenceAnnotation) != "" || + text(node, "metadata", "annotations", capacityOwnerAnnotation) != capacityOwnerValue || text(node, "metadata", "annotations", scaleDownDisabled) != "true" || + text(node, "metadata", "labels", "node.longhorn.io/create-default-disk") != "" || + text(storage, "metadata", "name") != b.Name || text(storage, "metadata", "namespace") != "longhorn-system" || b.LonghornUID == "" || + text(storage, "metadata", "resourceVersion") == "" || value(storage, "metadata", "deletionTimestamp") != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + return b, nil +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_authorization_test.go b/scripts/migrate-longhorn-v2/temporary_activation_authorization_test.go new file mode 100644 index 0000000000..69fff80a29 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation_authorization_test.go @@ -0,0 +1,52 @@ +package main + +import ( + "os" + "os/exec" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestTemporaryActivationRequiresItsOwnFirstAttemptMainConfirmation(t *testing.T) { + env := dispatchFixture() + env["LONGHORN_V2_OPERATION"] = "register-temporary-v2-storage" + env["LONGHORN_V2_CONFIRM"] = "register-longhorn-v2-temporary-storage" + getenv := func(key string) string { return env[key] } + if err := authorizeTemporaryActivation(getenv); err != nil { + t.Fatal(err) + } + for key, value := range map[string]string{"LONGHORN_V2_CONFIRM": "prepare-longhorn-v2-prod-worker-1", "GITHUB_RUN_ATTEMPT": "2", "GITHUB_REF": "refs/heads/topic", "GITHUB_EVENT_NAME": "pull_request", "GITHUB_WORKFLOW_REF": "other/workflow@main"} { + t.Run(key, func(t *testing.T) { + old := env[key] + env[key] = value + defer func() { env[key] = old }() + if authorizeTemporaryActivation(getenv) == nil { + t.Fatal("unreviewed or differently confirmed activation was accepted") + } + }) + } + if authorizeCanary(getenv) == nil || authorizeHostReboot(getenv, "prod-worker-1") == nil { + t.Fatal("disk registration authorized a canary or host disruption") + } +} + +func TestTemporaryActivationWorkflowRequiresTheDistinctConfirmation(t *testing.T) { + data, err := os.ReadFile("../../.github/workflows/prepare-longhorn-v2-node.yaml") + if err != nil { + t.Fatal(err) + } + var workflow struct { + Jobs map[string]struct{ Steps []struct{ Run string } } + } + if yaml.Unmarshal(data, &workflow) != nil { + t.Fatal("invalid workflow") + } + for _, confirmation := range []string{"register-longhorn-v2-temporary-storage", "verify-longhorn-v2-restore", "prepare-unused-longhorn-v2-temporary-v1-1"} { + cmd := exec.CommandContext(t.Context(), "bash", "-c", workflow.Jobs["prepare"].Steps[0].Run) + cmd.Env = append(os.Environ(), "DISPATCH_REF=refs/heads/main", "GITHUB_RUN_ATTEMPT=1", "LONGHORN_V2_OPERATION=register-temporary-v2-storage", "LONGHORN_V2_CONFIRM="+confirmation) + if err = cmd.Run(); (err == nil) != (confirmation == "register-longhorn-v2-temporary-storage") { + t.Fatalf("registration confirmation %s: %v", confirmation, err) + } + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go new file mode 100644 index 0000000000..371dbdf97e --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go @@ -0,0 +1,276 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strconv" + "strings" + "testing" + "time" +) + +type activationAPI struct { + t *testing.T + s inventory + managers, databases []object + baseline, record object + writes int + lostResponse bool + priorLive bool +} + +func newActivationAPI(t *testing.T) (*activationAPI, temporaryV2Activation) { + t.Helper() + s, managers := v2ReadinessFixture() + for i, node := range s.nodes { + node["metadata"].(object)["uid"] = fmt.Sprintf("node-%d", i) + if strings.HasPrefix(text(node, "metadata", "name"), "prod-worker-") { + node["metadata"].(object)["labels"].(object)["node.kubernetes.io/instance-type"] = "cx43" + } + node["spec"] = object{"providerID": fmt.Sprintf("hcloud://%d", i+100), "unschedulable": false} + node["status"].(object)["nodeInfo"] = object{"systemUUID": fmt.Sprintf("hardware-%d", i), "bootID": fmt.Sprintf("12345678-1234-4234-8234-%012x", i+1), "osImage": "Talos fixture", "architecture": "amd64"} + node["status"].(object)["addresses"] = []any{object{"type": "InternalIP", "address": fmt.Sprintf("10.0.99.%d", i+10)}} + } + for _, lh := range s.storageNodes { + lh["metadata"].(object)["namespace"] = "longhorn-system" + lh["spec"].(object)["allowScheduling"] = true + lh["spec"].(object)["evictionRequested"] = false + if strings.HasPrefix(text(lh, "metadata", "name"), "autoscale-longhorn-") { + lh["spec"] = object{"allowScheduling": false, "evictionRequested": true, "disks": object{}} + lh["status"] = object{"diskStatus": object{}} + } + } + f := &activationAPI{t: t, s: s, managers: managers, baseline: unusedBaselineFixture(t, s)} + for _, selected := range []struct{ namespace, name string }{{"observability", "coroot-db"}, {"umami", "umami-db"}, {"wedding-app", "wedding-db"}} { + db := resource(selected.name) + db["metadata"].(object)["namespace"] = selected.namespace + db["spec"] = object{"instances": float64(3)} + db["status"] = object{"readyInstances": float64(3), "systemID": "123456789", "phase": "Cluster in healthy state"} + f.databases = append(f.databases, db) + } + pools, err := configuredMigrationPools(poolConfigFixture()) + if err != nil { + t.Fatal(err) + } + a := temporaryV2Activation{c: client{context: "fixture", run: f.run}, source: strings.Repeat("a", 40), owner: "456", pools: pools} + a.reader = func(_ context.Context, address string) (nativeReplicaReader, func() error, error) { + for _, node := range f.s.nodes { + addresses, _ := value(node, "status", "addresses").([]any) + if len(addresses) == 1 && text(addresses[0].(object), "address") == address { + boot := text(node, "status", "nodeInfo", "bootID") + return &activationBootReader{before: boot, after: boot}, func() error { return nil }, nil + } + } + return nil, nil, errTemporaryActivation + } + return f, a +} + +func (f *activationAPI) run(_ context.Context, input []byte, program string, args ...string) ([]byte, error) { + f.t.Helper() + if program == "git" && reflect.DeepEqual(args, []string{"ls-remote", "--exit-code", "https://github.com/devantler-tech/platform.git", "refs/heads/main"}) { + return []byte(strings.Repeat("a", 40) + "\trefs/heads/main\n"), nil + } + if program == "gh" && len(args) == 2 && args[0] == "api" { + source := strings.Repeat("1", 40) + status := "completed" + if !strings.Contains(args[1], "/5678/") { + source = strings.Repeat("a", 40) + if f.priorLive { + status = "in_progress" + } + } + return json.Marshal(object{"status": status, "head_sha": source, "head_branch": "main", "event": "workflow_dispatch", "run_attempt": 1, "path": ".github/workflows/prepare-longhorn-v2-node.yaml"}) + } + if program == "talosctl" && len(args) == 5 && args[1] == "get" && args[4] == "--output=json" { + for i, node := range f.s.nodes { + if args[0] != "--nodes="+fmt.Sprintf("10.0.99.%d", i+10) { + continue + } + if args[2] == "systeminformation" && args[3] == "systeminformation" { + return json.Marshal(object{"metadata": object{"id": "systeminformation", "namespace": "hardware"}, "spec": object{"uuid": text(node, "status", "nodeInfo", "systemUUID")}}) + } + raw, eph := rawFixture() + raw["spec"].(object)["partitionUUID"] = fmt.Sprintf("12345678-1234-4234-8234-%012x", i+100) + eph["spec"].(object)["uuid"] = fmt.Sprintf("ephemeral-%d", i) + if args[2] == "volumestatus" && args[3] == "r-longhorn" { + return json.Marshal(raw) + } + if args[2] == "volumestatus" && args[3] == "EPHEMERAL" { + return json.Marshal(eph) + } + } + } + if program != "kubectl" || len(args) < 4 || args[0] != "--context=fixture" || args[1] != "--request-timeout=20s" { + f.t.Fatalf("activation escaped its native scope: %s %v", program, args) + } + args = args[2:] + if args[0] == "get" { + kind := args[1] + if args[2] == "--output=json" { + collections := map[string][]object{"nodes": f.s.nodes, "nodes.longhorn.io": f.s.storageNodes, "volumes.longhorn.io": f.s.volumes, "replicas.longhorn.io": f.s.replicas, "engines.longhorn.io": f.s.engines, "orphans.longhorn.io": f.s.orphans, "pods": f.s.pods, "instancemanagers.longhorn.io": f.managers, "clusters.postgresql.cnpg.io": f.databases} + items, ok := collections[kind] + if !ok { + f.t.Fatalf("unexpected collection %s", kind) + } + if items == nil { + items = []object{} + } + return json.Marshal(object{"items": items}) + } + name := args[2] + if kind == "configmap" { + if name == temporaryActivationJournal { + if f.record == nil { + return nil, nil + } + return json.Marshal(f.record) + } + if name == "longhorn-v2-benchmark-v1" { + return json.Marshal(f.baseline) + } + } + if kind == "settings.longhorn.io" { + settings := map[string]string{"v1-data-engine": "true", "v2-data-engine": "true", "storage-over-provisioning-percentage": "100", "storage-minimal-available-percentage": "25", "guaranteed-instance-manager-cpu": `{"v1":"6","v2":"25"}`, "data-engine-cpu-mask": `{"v2":"0x3"}`, "data-engine-number-of-cpu-cores": `{"v2":"0"}`, "data-engine-cpu-isolation-enabled": `{"v2":"true"}`, "data-engine-interrupt-mode-enabled": `{"v2":"false"}`, "data-engine-hugepage-enabled": `{"v2":"true"}`, "data-engine-memory-size": `{"v2":"2048"}`} + setting, ok := settings[name] + if !ok { + f.t.Fatalf("unexpected setting %s", name) + } + o := resource(name) + o["value"] = setting + return json.Marshal(o) + } + for _, o := range append(append([]object{}, f.s.nodes...), f.s.storageNodes...) { + if text(o, "metadata", "name") == name && (kind == "node" && text(o, "metadata", "namespace") == "" || kind == "nodes.longhorn.io" && text(o, "metadata", "namespace") == "longhorn-system") { + return json.Marshal(o) + } + } + } + if args[0] == "create" { + if f.record != nil { + return nil, errConflict + } + var created object + if json.Unmarshal(input, &created) != nil || text(created, "kind") != "ConfigMap" || text(created, "metadata", "name") != temporaryActivationJournal || text(created, "metadata", "namespace") != "longhorn-system" { + f.t.Fatal("activation created a resource outside its fixed journal") + } + created["metadata"].(object)["uid"] = "activation-journal-uid" + created["metadata"].(object)["resourceVersion"] = "1" + f.record = created + return json.Marshal(created) + } + if args[0] == "patch" { + var target object + if args[1] == "configmap" && args[2] == temporaryActivationJournal { + target = f.record + } + if args[1] == "nodes.longhorn.io" { + for _, lh := range f.s.storageNodes { + if text(lh, "metadata", "name") == args[2] && strings.HasPrefix(args[2], "autoscale-longhorn-") { + target = lh + } + } + } + var changes []object + if target == nil || json.Unmarshal(input, &changes) != nil || len(changes) < 3 || !reflect.DeepEqual(changes[:2], identityTests(target)) { + f.t.Fatal("activation write lost its exact native identity or exceeded temporary scope") + } + for _, change := range changes[2:] { + parts := strings.Split(strings.TrimPrefix(text(change, "path"), "/"), "/") + parent := target + for _, part := range parts[:len(parts)-1] { + parent = parent[part].(object) + } + parent[strings.ReplaceAll(parts[len(parts)-1], "~1", "/")] = change["value"] + } + version, _ := strconv.Atoi(text(target, "metadata", "resourceVersion")) + target["metadata"].(object)["resourceVersion"] = strconv.Itoa(version + 1) + if args[1] == "nodes.longhorn.io" { + f.writes++ + target["status"] = object{"conditions": readyConditions(), "diskStatus": object{"longhorn-v2": object{"diskUUID": "disk-" + args[2], "conditions": readyConditions()}}} + if f.lostResponse { + f.lostResponse = false + return nil, errors.New("response lost after registration") + } + } + return nil, nil + } + f.t.Fatalf("unexpected activation command: %s %v", program, args) + return nil, nil +} + +func TestTemporaryActivationRegistersOnlyThreeJournalOwnedRawDisks(t *testing.T) { + f, a := newActivationAPI(t) + if err := a.run(t.Context()); err != nil { + t.Fatal(err) + } + _, record, err := a.read(t.Context()) + if err != nil || !record.Completed || f.writes != 3 { + t.Fatal("activation did not prove all three owned raw disks:", err) + } + if err = a.run(t.Context()); err != nil || f.writes != 3 { + t.Fatal("completed activation repeated a registration:", err) + } +} + +func TestTemporaryActivationResumesItsUncertainWriteWithoutAdoptingForeignStorage(t *testing.T) { + f, a := newActivationAPI(t) + f.lostResponse = true + if a.run(t.Context()) == nil || f.writes != 1 { + t.Fatal("uncertain registration was accepted or retried") + } + a.owner = "789" + if err := a.run(t.Context()); err != nil || f.writes != 3 { + t.Fatal("owned partial activation could not safely resume:", err) + } + f, a = newActivationAPI(t) + f.s.storageNodes[3]["metadata"].(object)["annotations"] = object{temporaryActivationOwner: "foreign"} + if a.run(t.Context()) == nil || f.writes != 0 { + t.Fatal("unowned storage was adopted") + } +} + +func TestTemporaryActivationRejectsUnqualifiedClientsBeforeRegisteringDisks(t *testing.T) { + f, a := newActivationAPI(t) + f.managers[0]["spec"].(object)["image"] = "example.invalid/unreviewed:latest" + value(f.s.pods[0], "spec", "containers").([]any)[0].(object)["image"] = "example.invalid/unreviewed:latest" + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + if a.run(ctx) == nil || f.writes != 0 { + t.Fatal("unqualified V2 clients reached raw-disk registration") + } +} + +func TestCanaryRequiresCompletedStillOwnedTemporaryActivation(t *testing.T) { + f, a := newActivationAPI(t) + if a.accepted(t.Context()) == nil || f.writes != 0 { + t.Fatal("an unregistered pool authorized the canary") + } + for _, mode := range []string{"valid", "journal replaced", "worker changed", "workflow live", "disk fenced"} { + t.Run(mode, func(t *testing.T) { + f, a := newActivationAPI(t) + if err := a.run(t.Context()); err != nil { + t.Fatal(err) + } + switch mode { + case "journal replaced": + f.record["metadata"].(object)["uid"] = "replacement" + case "worker changed": + f.s.nodes[3]["spec"].(object)["providerID"] = "hcloud://999" + case "workflow live": + f.priorLive = true + case "disk fenced": + f.s.storageNodes[3]["spec"].(object)["allowScheduling"] = false + } + before, _ := json.Marshal(f.record) + err := a.accepted(t.Context()) + after, _ := json.Marshal(f.record) + if (err == nil) != (mode == "valid") || f.writes != 3 || string(before) != string(after) { + t.Fatal("canary acceptance ignored changed evidence or mutated the plan:", err) + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_native_test.go b/scripts/migrate-longhorn-v2/temporary_activation_native_test.go new file mode 100644 index 0000000000..c01c2e9309 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation_native_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "testing" +) + +type activationBootReader struct { + nativeReplicaReader + before, after string + reads int +} + +func (r *activationBootReader) BootID(context.Context) (string, error) { + r.reads++ + if r.reads == 1 { + return r.before, nil + } + return r.after, nil +} + +func TestTemporaryRawBindingBracketsNativeReadsAndClosesItsReader(t *testing.T) { + for _, mode := range []string{"valid", "boot changed", "hardware changed", "node changed", "partition unreadable", "close failed"} { + t.Run(mode, func(t *testing.T) { + n, storage, raw, eph, pool := temporaryRawBindingFixture() + n["status"].(object)["addresses"] = []any{object{"type": "InternalIP", "address": "10.0.99.10"}} + boot := text(n, "status", "nodeInfo", "bootID") + reader := &activationBootReader{before: boot, after: boot} + if mode == "boot changed" { + reader.after = "12345678-1234-4234-8234-000000000099" + } + closed, volumes := 0, 0 + activation := temporaryV2Activation{reader: func(context.Context, string) (nativeReplicaReader, func() error, error) { + return reader, func() error { + closed++ + if mode == "close failed" { + return errors.New("reader close failed") + } + return nil + }, nil + }} + activation.c = client{context: "fixture", run: func(_ context.Context, _ []byte, program string, args ...string) ([]byte, error) { + if program == "talosctl" && len(args) == 5 && args[0] == "--nodes=10.0.99.10" && args[1] == "get" && args[4] == "--output=json" { + switch { + case args[2] == "systeminformation" && args[3] == "systeminformation": + uuid := "hardware" + if mode == "hardware changed" { + uuid = "other" + } + return json.Marshal(object{"metadata": object{"namespace": "hardware", "id": "systeminformation"}, "spec": object{"uuid": uuid}}) + case args[2] == "volumestatus" && args[3] == "r-longhorn": + volumes++ + if mode == "partition unreadable" { + return nil, errors.New("native read failed") + } + return json.Marshal(raw) + case args[2] == "volumestatus" && args[3] == "EPHEMERAL": + volumes++ + return json.Marshal(eph) + } + } + if program == "kubectl" && reflect.DeepEqual(args, []string{"--context=fixture", "--request-timeout=20s", "get", "node", text(n, "metadata", "name"), "--output=json"}) { + if mode == "node changed" { + n["metadata"].(object)["uid"] = "replacement" + } + return json.Marshal(n) + } + t.Fatalf("native binding escaped its read-only fixed scope: %s %v", program, args) + return nil, nil + }} + binding, err := activation.readBinding(t.Context(), "temporary-v1-1", n, storage, pool) + if (err == nil) != (mode == "valid") { + t.Fatal("native binding validity did not match its complete identity bracket:", err) + } + if mode == "valid" && (closed != 1 || reader.reads != 2 || volumes != 2 || binding.BootID != boot) { + t.Fatal("successful binding omitted a native read or close") + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_record.go b/scripts/migrate-longhorn-v2/temporary_activation_record.go new file mode 100644 index 0000000000..376068dea6 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation_record.go @@ -0,0 +1,75 @@ +package main + +import ( + "bytes" + "encoding/json" + "regexp" + "strings" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +const temporaryActivationJournal = "longhorn-v2-temporary-activation" + +var temporaryV2Selections = [...]string{"temporary-v1-1", "temporary-v1-2", "temporary-v2-1"} + +type temporaryActivationRecord struct { + Version int `json:"version"` + Source string `json:"source"` + OwnerRun string `json:"ownerRun"` + Targets map[string]temporaryV2Binding `json:"targets"` + Completed bool `json:"completed"` +} + +// decodeTemporaryActivationRecord rejects ambiguous JSON before its typed +// decode, including duplicate members and case aliases in every fixed target. +func decodeTemporaryActivationRecord(data []byte) (temporaryActivationRecord, error) { + if len(data) == 0 || len(data) > 32<<10 { + return temporaryActivationRecord{}, errTemporaryActivation + } + canonical := map[string][]string{"": {"version", "source", "ownerRun", "targets", "completed"}} + fields := []string{"selection", "name", "nodeUID", "providerID", "machineUUID", "bootID", "osImage", "longhornUID", "ephemeralUUID", "rawPartitionUUID"} + for _, selection := range temporaryV2Selections { + canonical["targets."+selection] = fields + } + if strictjson.Validate(data, canonical) != nil { + return temporaryActivationRecord{}, errTemporaryActivation + } + var record temporaryActivationRecord + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if decoder.Decode(&record) != nil || record.validate() != nil { + return temporaryActivationRecord{}, errTemporaryActivation + } + return record, nil +} + +// validate requires three independent fixed pool members and complete native +// partition identities. An existing journal cannot substitute another worker or +// reuse one physical disk under multiple selections when a run resumes. +func (r temporaryActivationRecord) validate() error { + if r.Version != 1 || !regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString(r.Source) || !regexp.MustCompile(`^[1-9][0-9]*$`).MatchString(r.OwnerRun) || len(r.Targets) != len(temporaryV2Selections) { + return errTemporaryActivation + } + seen := [7]map[string]bool{} + for i := range seen { + seen[i] = map[string]bool{} + } + for _, selection := range temporaryV2Selections { + b, ok := r.Targets[selection] + if !ok || b.Selection != selection || !inventoryTemporaryNode.MatchString(b.Name) || + !strings.HasPrefix(b.Name, "autoscale-longhorn-"+strings.Split(selection, "-")[1]+"-") || + b.NodeUID == "" || !regexp.MustCompile(`^hcloud://[1-9][0-9]*$`).MatchString(b.ProviderID) || b.MachineUUID == "" || + !inventoryBoot.MatchString(b.BootID) || b.OSImage == "" || b.LonghornUID == "" || b.EphemeralUUID == "" || + !inventoryBoot.MatchString(strings.ToLower(b.RawPartitionUUID)) || strings.EqualFold(b.RawPartitionUUID, b.EphemeralUUID) { + return errTemporaryActivation + } + for i, identity := range []string{b.Name, b.NodeUID, b.ProviderID, strings.ToLower(b.MachineUUID), b.LonghornUID, strings.ToLower(b.EphemeralUUID), strings.ToLower(b.RawPartitionUUID)} { + if seen[i][identity] { + return errTemporaryActivation + } + seen[i][identity] = true + } + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_record_test.go b/scripts/migrate-longhorn-v2/temporary_activation_record_test.go new file mode 100644 index 0000000000..ce04507c5a --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation_record_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" +) + +func temporaryActivationRecordFixture(t *testing.T) temporaryActivationRecord { + t.Helper() + n, lh, raw, eph, pool := temporaryRawBindingFixture() + binding, err := projectTemporaryV2Binding("temporary-v1-1", n, lh, raw, eph, pool, text(n, "status", "nodeInfo", "bootID")) + if err != nil { + t.Fatal(err) + } + record := temporaryActivationRecord{Version: 1, Source: strings.Repeat("a", 40), OwnerRun: "123", Targets: map[string]temporaryV2Binding{}} + for i, selection := range []string{"temporary-v1-1", "temporary-v1-2", "temporary-v2-1"} { + b := binding + b.Selection = selection + b.Name = fmt.Sprintf("autoscale-longhorn-%s-%016x", strings.Split(selection, "-")[1], i) + b.NodeUID, b.ProviderID, b.MachineUUID = fmt.Sprintf("node-%d", i), fmt.Sprintf("hcloud://%d", 100+i), fmt.Sprintf("hardware-%d", i) + b.LonghornUID, b.EphemeralUUID = fmt.Sprintf("longhorn-%d", i), fmt.Sprintf("ephemeral-%d", i) + b.RawPartitionUUID = fmt.Sprintf("12345678-1234-4234-8234-%012x", i+10) + record.Targets[selection] = b + } + return record +} + +func TestTemporaryActivationRecordRequiresThreeDistinctCompleteBindings(t *testing.T) { + record := temporaryActivationRecordFixture(t) + data, _ := json.Marshal(record) + if _, err := decodeTemporaryActivationRecord(data); err != nil { + t.Fatal("complete fixed temporary activation record rejected:", err) + } + for name, change := range map[string]func(*temporaryActivationRecord){ + "unreviewed source": func(r *temporaryActivationRecord) { r.Source = "unknown" }, + "missing owner": func(r *temporaryActivationRecord) { r.OwnerRun = "" }, + "missing third disk": func(r *temporaryActivationRecord) { delete(r.Targets, "temporary-v2-1") }, + "wrong target pool": func(r *temporaryActivationRecord) { + b := r.Targets["temporary-v2-1"] + b.Name = r.Targets["temporary-v1-1"].Name + r.Targets["temporary-v2-1"] = b + }, + "duplicate provider": func(r *temporaryActivationRecord) { + b := r.Targets["temporary-v1-2"] + b.ProviderID = r.Targets["temporary-v1-1"].ProviderID + r.Targets["temporary-v1-2"] = b + }, + "duplicate partition": func(r *temporaryActivationRecord) { + b := r.Targets["temporary-v1-2"] + b.RawPartitionUUID = r.Targets["temporary-v1-1"].RawPartitionUUID + r.Targets["temporary-v1-2"] = b + }, + "missing native identity": func(r *temporaryActivationRecord) { + b := r.Targets["temporary-v1-2"] + b.RawPartitionUUID = "" + r.Targets["temporary-v1-2"] = b + }, + } { + t.Run(name, func(t *testing.T) { + r := temporaryActivationRecordFixture(t) + change(&r) + data, _ := json.Marshal(r) + if _, err := decodeTemporaryActivationRecord(data); err == nil { + t.Fatal("unproven activation record was accepted") + } + }) + } + for name, altered := range map[string]string{ + "duplicate field": strings.Replace(string(data), `"completed":false`, `"completed":false,"completed":true`, 1), + "root case alias": strings.Replace(string(data), `"source":`, `"Source":`, 1), + "nested case alias": strings.Replace(string(data), `"nodeUID":`, `"NodeUID":`, 1), + "unknown field": strings.Replace(string(data), `"version":1`, `"version":1,"extra":true`, 1), + "trailing document": string(data) + ` {}`, + } { + t.Run(name, func(t *testing.T) { + if _, err := decodeTemporaryActivationRecord([]byte(altered)); err == nil { + t.Fatal("ambiguous activation record was accepted") + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_run.go b/scripts/migrate-longhorn-v2/temporary_activation_run.go new file mode 100644 index 0000000000..d00ac5bf46 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation_run.go @@ -0,0 +1,281 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" +) + +// read accepts only the fixed activation journal with a complete native API +// identity and a strictly decoded three-member plan. +func (a temporaryV2Activation) read(ctx context.Context) (object, temporaryActivationRecord, error) { + cm, err := a.c.get(ctx, "configmap", "longhorn-system", temporaryActivationJournal, true) + if err != nil || cm == nil { + return cm, temporaryActivationRecord{}, err + } + if text(cm, "metadata", "name") != temporaryActivationJournal || text(cm, "metadata", "namespace") != "longhorn-system" || text(cm, "metadata", "uid") == "" || text(cm, "metadata", "resourceVersion") == "" || value(cm, "metadata", "deletionTimestamp") != nil { + return nil, temporaryActivationRecord{}, errTemporaryActivation + } + record, err := decodeTemporaryActivationRecord([]byte(text(cm, "data", "record"))) + return cm, record, err +} + +// persist conditionally writes the plan on current main, then verifies the +// same journal identity and complete record through a fresh native read. +func (a temporaryV2Activation) persist(ctx context.Context, cm object, record temporaryActivationRecord) (object, error) { + if record.validate() != nil || record.Source != a.source || record.OwnerRun != a.owner { + return nil, errTemporaryActivation + } + data, err := json.Marshal(record) + if err != nil || sourceCurrent(ctx, a.c.run, a.source) != nil { + return nil, errTemporaryActivation + } + if cm == nil { + cm, err = a.c.create(ctx, a.source, object{"apiVersion": "v1", "kind": "ConfigMap", "metadata": object{"name": temporaryActivationJournal, "namespace": "longhorn-system"}, "data": object{"record": string(data)}}) + } else { + err = a.c.patch(ctx, "configmap", "longhorn-system", cm, object{"op": "replace", "path": "/data/record", "value": string(data)}) + } + if err != nil { + return nil, err + } + readback, actual, err := a.read(ctx) + if err != nil || text(readback, "metadata", "uid") != text(cm, "metadata", "uid") || !reflect.DeepEqual(record, actual) { + return nil, errTemporaryActivation + } + return readback, nil +} + +// owned rebinds journal identity, owner, plan and current main immediately +// before every registration or completion write. +func (a temporaryV2Activation) owned(ctx context.Context, uid string, expected temporaryActivationRecord) (object, error) { + cm, actual, err := a.read(ctx) + if err != nil || uid == "" || text(cm, "metadata", "uid") != uid || actual.Source != a.source || actual.OwnerRun != a.owner || !reflect.DeepEqual(expected, actual) { + return nil, errTemporaryActivation + } + return cm, sourceCurrent(ctx, a.c.run, a.source) +} + +// environment preserves the original V1 data and database quorums throughout +// raw registration. All six clients need their reserved capacity and every +// temporary worker must retain its owned autoscaler pin. +func (a temporaryV2Activation) environment(ctx context.Context) (inventory, error) { + if err := verifyBudgetSettings(ctx, a.c); err != nil { + return inventory{}, err + } + if err := verifyCanarySettings(ctx, a.c); err != nil { + return inventory{}, err + } + s, err := a.c.snapshot(ctx) + if err != nil || verifyMigrationCapacity(s.nodes, a.pools) != nil || verifyRedundancy(s) != nil { + return inventory{}, errTemporaryActivation + } + managers, err := a.c.list(ctx, "instancemanagers.longhorn.io", "longhorn-system") + if err != nil || verifyV2Clients(s, managers, a.pools, canaryManagerImage) != nil { + return inventory{}, errTemporaryActivation + } + for _, node := range s.nodes { + if strings.HasPrefix(text(node, "metadata", "name"), "prod-control-plane-") { + continue + } + if value(node, "spec", "unschedulable") == true || text(node, "metadata", "annotations", fenceAnnotation) != "" || text(node, "status", "allocatable", "cpu") != "8" { + return inventory{}, errTemporaryActivation + } + } + for _, volume := range s.volumes { + if text(volume, "spec", "dataEngine") != "v1" { + return inventory{}, errTemporaryActivation + } + } + if _, err = (hostMemoryReboot{c: a.c}).databases(ctx, nil); err != nil { + return inventory{}, err + } + if err = requireBenchmarkBaseline(ctx, a.c, s.nodes); err != nil { + return inventory{}, err + } + return s, nil +} + +// targets collects all native bindings before any disk is enabled. A resumed +// plan must still name the same physical workers and partitions, and no existing +// replica may be assigned to a temporary member during this pre-cutover step. +func (a temporaryV2Activation) targets(ctx context.Context, s inventory, journalUID string, expected map[string]temporaryV2Binding) (map[string]temporaryV2Binding, error) { + nodes, err := indexed(s.nodes) + if err != nil { + return nil, err + } + storage, err := indexed(s.storageNodes) + if err != nil { + return nil, err + } + result := map[string]temporaryV2Binding{} + for _, selection := range temporaryV2Selections { + name, _, err := resolveTemporaryInventoryTarget(selection, s.nodes, a.pools) + if err != nil { + return nil, fmt.Errorf("temporary raw registration held [stage=pool-selection]: %w", err) + } + for _, replica := range s.replicas { + if text(replica, "spec", "nodeID") == name { + return nil, errTemporaryActivation + } + } + if _, err = temporaryRegistrationState(storage[name], journalUID); err != nil { + return nil, err + } + role := text(nodes[name], "metadata", "labels", migrationPoolLabel) + binding, err := a.readBinding(ctx, selection, nodes[name], storage[name], a.pools[role]) + if err != nil || expected != nil && expected[selection] != binding { + return nil, errTemporaryActivation + } + result[selection] = binding + } + return result, nil +} + +// registrationChanges adds ownership and the canonical raw disk in one +// conditional Longhorn-node patch, preserving unrelated metadata. +func registrationChanges(storage object, journalUID string) []object { + changes := []object{} + if value(storage, "metadata", "labels") == nil { + changes = append(changes, object{"op": "add", "path": "/metadata/labels", "value": object{}}) + } + if value(storage, "metadata", "annotations") == nil { + changes = append(changes, object{"op": "add", "path": "/metadata/annotations", "value": object{}}) + } + changes = append(changes, + object{"op": "add", "path": "/metadata/labels/" + strings.ReplaceAll(migrationEnabledLabel, "/", "~1"), "value": "true"}, + object{"op": "add", "path": "/metadata/annotations/" + strings.ReplaceAll(temporaryActivationOwner, "/", "~1"), "value": journalUID}, + object{"op": "replace", "path": "/spec/disks", "value": blockDisks()}, + object{"op": "replace", "path": "/spec/allowScheduling", "value": true}, + object{"op": "replace", "path": "/spec/evictionRequested", "value": false}) + return changes +} + +// run registers the three temporary raw disks under one durable identity plan. +// An uncertain patch stops the invocation; only a later confirmed owner can +// adopt its exact partial result after the previous workflow has terminated. +func (a temporaryV2Activation) run(ctx context.Context) error { + if err := sourceCurrent(ctx, a.c.run, a.source); err != nil { + return err + } + s, err := a.environment(ctx) + if err != nil { + return err + } + cm, record, err := a.read(ctx) + if err != nil { + return err + } + if cm == nil { + targets, err := a.targets(ctx, s, "", nil) + if err != nil { + return err + } + record = temporaryActivationRecord{Version: 1, Source: a.source, OwnerRun: a.owner, Targets: targets} + cm, err = a.persist(ctx, nil, record) + if err != nil { + return err + } + } else { + if _, err = a.targets(ctx, s, text(cm, "metadata", "uid"), record.Targets); err != nil { + return err + } + if record.OwnerRun != a.owner || record.Source != a.source { + if err = previousRunCompleted(ctx, a.c, record.OwnerRun, record.Source); err != nil { + return err + } + record.OwnerRun, record.Source = a.owner, a.source + cm, err = a.persist(ctx, cm, record) + if err != nil { + return err + } + } + } + uid := text(cm, "metadata", "uid") + for _, selection := range temporaryV2Selections { + s, err = a.environment(ctx) + if err != nil { + return err + } + if _, err = a.targets(ctx, s, uid, record.Targets); err != nil { + return err + } + storage, err := a.c.get(ctx, "nodes.longhorn.io", "longhorn-system", record.Targets[selection].Name, false) + if err != nil || text(storage, "metadata", "uid") != record.Targets[selection].LonghornUID { + return errTemporaryActivation + } + registered, err := temporaryRegistrationState(storage, uid) + if err != nil { + return err + } + if registered { + continue + } + if _, err = a.owned(ctx, uid, record); err != nil { + return err + } + if err = a.c.patch(ctx, "nodes.longhorn.io", "longhorn-system", storage, registrationChanges(storage, uid)...); err != nil { + return err + } + } + if err = waitReady(ctx, func(ctx context.Context) error { + if _, err := a.owned(ctx, uid, record); err != nil { + return errors.Join(errReadinessTerminal, err) + } + s, err := a.environment(ctx) + if err != nil { + return err + } + if _, err = a.targets(ctx, s, uid, record.Targets); err != nil { + return errors.Join(errReadinessTerminal, err) + } + _, err = canaryEnvironment(ctx, a.c, a.pools) + return err + }); err != nil { + return err + } + if !record.Completed { + cm, err = a.owned(ctx, uid, record) + if err != nil { + return err + } + record.Completed = true + if _, err = a.persist(ctx, cm, record); err != nil { + return err + } + } + fmt.Println("Temporary V2 storage is registered on three identity-bound raw disks; clients and original V1 copies are healthy.") + return nil +} + +// accepted is a read-only canary admission check. The completed journal must +// come from a terminated protected workflow and still match all three native +// partitions and journal-owned registrations; its cached completion bit alone +// never authorizes IO. +func (a temporaryV2Activation) accepted(ctx context.Context) error { + cm, record, err := a.read(ctx) + if err != nil || cm == nil || !record.Completed || previousRunCompleted(ctx, a.c, record.OwnerRun, record.Source) != nil { + return errTemporaryActivation + } + s, err := a.environment(ctx) + if err != nil { + return err + } + uid := text(cm, "metadata", "uid") + if _, err = a.targets(ctx, s, uid, record.Targets); err != nil { + return err + } + storage, err := indexed(s.storageNodes) + if err != nil { + return err + } + for _, binding := range record.Targets { + registered, err := temporaryRegistrationState(storage[binding.Name], uid) + if err != nil || !registered { + return errTemporaryActivation + } + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_test.go b/scripts/migrate-longhorn-v2/temporary_activation_test.go new file mode 100644 index 0000000000..d98421135a --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "testing" +) + +func temporaryRawBindingFixture() (object, object, object, object, migrationPool) { + n := capacityFixture()[3] + n["spec"] = object{"providerID": "hcloud://123", "unschedulable": false} + n["status"].(object)["nodeInfo"] = object{"systemUUID": "hardware", "bootID": "12345678-1234-4234-8234-000000000001", "osImage": "Talos fixture", "architecture": "amd64"} + n["status"].(object)["allocatable"].(object)["cpu"] = "8" + lh := resource(text(n, "metadata", "name")) + lh["metadata"].(object)["namespace"] = "longhorn-system" + raw, eph := rawFixture() + raw["spec"].(object)["partitionUUID"] = "12345678-1234-4234-8234-000000000002" + return n, lh, raw, eph, migrationPool{name: "autoscale-longhorn-v1", role: "v1", serverType: "cpx42", count: 2} +} + +func TestTemporaryRawRegistrationBindsTheNativePartitionsAndPoolMember(t *testing.T) { + n, lh, raw, eph, pool := temporaryRawBindingFixture() + binding, err := projectTemporaryV2Binding("temporary-v1-1", n, lh, raw, eph, pool, text(n, "status", "nodeInfo", "bootID")) + if err != nil || binding.Name != text(n, "metadata", "name") || binding.RawPartitionUUID != text(raw, "spec", "partitionUUID") || binding.EphemeralUUID != text(eph, "spec", "uuid") { + t.Fatal("qualified temporary raw storage was not bound to its native identities:", err) + } + for name, change := range map[string]func(object, object, object, object){ + "foreign storage identity": func(_ object, lh object, _, _ object) { lh["metadata"].(object)["name"] = "other" }, + "missing raw identity": func(_, _ object, raw, _ object) { delete(raw["spec"].(object), "partitionUUID") }, + "raw and filesystem identity reused": func(_, _ object, raw, eph object) { raw["spec"].(object)["partitionUUID"] = text(eph, "spec", "uuid") }, + "unencrypted raw disk": func(_, _ object, raw, _ object) { raw["spec"].(object)["encryptionProvider"] = "" }, + "wrong raw capacity": func(_, _ object, raw, _ object) { raw["spec"].(object)["size"] = float64(95 << 30) }, + "cordoned member": func(n, _, _, _ object) { n["spec"].(object)["unschedulable"] = true }, + "unowned capacity": func(n, _, _, _ object) { + delete(n["metadata"].(object)["annotations"].(object), capacityOwnerAnnotation) + }, + "automatic disk creation enabled": func(n, _, _, _ object) { + n["metadata"].(object)["labels"].(object)["node.longhorn.io/create-default-disk"] = "config" + }, + } { + t.Run(name, func(t *testing.T) { + n, lh, raw, eph, pool := temporaryRawBindingFixture() + change(n, lh, raw, eph) + if _, err := projectTemporaryV2Binding("temporary-v1-1", n, lh, raw, eph, pool, text(n, "status", "nodeInfo", "bootID")); err == nil { + t.Fatal("unproven temporary registration binding was accepted") + } + }) + } + if _, err := projectTemporaryV2Binding("temporary-v1-1", n, lh, raw, eph, pool, "12345678-1234-4234-8234-000000000099"); err == nil { + t.Fatal("a native boot mismatch was accepted") + } + if _, err := projectTemporaryV2Binding("temporary-v2-1", n, lh, raw, eph, pool, text(n, "status", "nodeInfo", "bootID")); err == nil { + t.Fatal("a selection from another pool was accepted") + } +} + +func TestTemporaryRawRegistrationAcceptsOnlyItsOwnPartialActivation(t *testing.T) { + storage := func(enabled bool) object { + _, lh, _, _, _ := temporaryRawBindingFixture() + lh["metadata"].(object)["annotations"] = object{} + lh["metadata"].(object)["labels"] = object{} + lh["spec"] = object{"allowScheduling": false, "evictionRequested": true, "disks": object{}} + lh["status"] = object{"diskStatus": object{}} + if enabled { + lh["metadata"].(object)["annotations"].(object)[temporaryActivationOwner] = "journal-uid" + lh["metadata"].(object)["labels"].(object)[migrationEnabledLabel] = "true" + lh["spec"] = object{"allowScheduling": true, "evictionRequested": false, "disks": blockDisks()} + } + return lh + } + if registered, err := temporaryRegistrationState(storage(false), "journal-uid"); err != nil || registered { + t.Fatal("empty fenced storage could not be registered:", err) + } + if registered, err := temporaryRegistrationState(storage(true), "journal-uid"); err != nil || !registered { + t.Fatal("the journal's exact existing activation could not resume:", err) + } + for name, change := range map[string]func(object){ + "foreign journal": func(lh object) { lh["metadata"].(object)["annotations"].(object)[temporaryActivationOwner] = "foreign" }, + "missing ownership": func(lh object) { delete(lh["metadata"].(object)["annotations"].(object), temporaryActivationOwner) }, + "filesystem substitution": func(lh object) { + lh["spec"].(object)["disks"] = object{"filesystem": object{"path": "/var/lib/longhorn"}} + }, + "unknown eviction state": func(lh object) { delete(lh["spec"].(object), "evictionRequested") }, + } { + t.Run(name, func(t *testing.T) { + lh := storage(true) + change(lh) + if _, err := temporaryRegistrationState(lh, "journal-uid"); err == nil { + t.Fatal("a foreign or changed disk registration was adopted") + } + }) + } + if _, err := temporaryRegistrationState(storage(true), ""); err == nil { + t.Fatal("an unjournaled existing registration was adopted") + } +} diff --git a/scripts/migrate-longhorn-v2/v2_readiness.go b/scripts/migrate-longhorn-v2/v2_readiness.go new file mode 100644 index 0000000000..d13d496e03 --- /dev/null +++ b/scripts/migrate-longhorn-v2/v2_readiness.go @@ -0,0 +1,199 @@ +package main + +import ( + "errors" + "strings" +) + +// verifyV2Runtime requires ready SPDK clients on every worker and three +// independent raw disks on the temporary workers. The original workers retain +// the V1 replicas throughout the disposable recovery proof. +func verifyV2Runtime(s inventory, managers []object, pools map[string]migrationPool, expectedManagerImage string) error { + if expectedManagerImage == "" || strings.TrimSpace(expectedManagerImage) != expectedManagerImage { + return errors.New("the reviewed instance-manager image is unresolved") + } + if err := verifyMigrationCapacity(s.nodes, pools); err != nil { + return err + } + if err := verifyRedundancy(s); err != nil { + return err + } + nodes, err := indexed(s.nodes) + if err != nil { + return err + } + storage, err := indexed(s.storageNodes) + if err != nil { + return err + } + if _, err = indexed(managers); err != nil { + return err + } + workers, raw := map[string]bool{}, map[string]bool{} + for name, n := range nodes { + if strings.HasPrefix(name, "prod-control-plane-") { + continue + } + if value(n, "spec", "unschedulable") == true || text(n, "status", "allocatable", "cpu") != "8" { + return errors.New("V2 migration workers must be schedulable with eight allocatable CPUs") + } + workers[name] = true + if _, temporary := pools[text(n, "metadata", "labels", migrationPoolLabel)]; temporary { + raw[name] = true + } + } + if len(workers) != 6 || len(raw) != 3 { + return errors.New("V2 canary storage and client roster is incomplete") + } + diskIDs := map[string]bool{} + for name := range raw { + lh := storage[name] + status, _ := value(lh, "status").(object) + diskStatus, _ := value(lh, "status", "diskStatus").(object) + disk, _ := diskStatus["longhorn-v2"].(object) + id := text(disk, "diskUUID") + if !reviewedBlockDisks(value(lh, "spec", "disks")) || value(lh, "spec", "allowScheduling") != true || value(lh, "spec", "evictionRequested") != false || + !ready(status) || len(diskStatus) != 1 || !ready(disk) || id == "" || diskIDs[id] { + return errors.New("three distinct ready canonical raw disks are not proven") + } + if override := value(lh, "spec", "instanceManagerCPURequest"); override != nil && override != float64(0) { + return errors.New("a node CPU override supersedes the reviewed V2 reservation") + } + diskIDs[id] = true + } + return verifyV2ClientPods(s, managers, workers, expectedManagerImage) +} + +// verifyV2Clients qualifies the effective clients before any raw disk is +// registered, independently of the later three-disk readiness requirement. +func verifyV2Clients(s inventory, managers []object, pools map[string]migrationPool, expectedManagerImage string) error { + if err := verifyMigrationCapacity(s.nodes, pools); err != nil { + return err + } + nodes, err := indexed(s.nodes) + if err != nil { + return err + } + workers := map[string]bool{} + for name, node := range nodes { + if strings.HasPrefix(name, "prod-control-plane-") { + continue + } + if value(node, "spec", "unschedulable") == true || text(node, "status", "allocatable", "cpu") != "8" { + return errors.New("V2 migration workers must be schedulable with eight allocatable CPUs") + } + workers[name] = true + } + if len(workers) != 6 { + return errors.New("V2 client roster is incomplete") + } + return verifyV2ClientPods(s, managers, workers, expectedManagerImage) +} + +// verifyV2ClientPods binds each client to its controller and reviewed image, +// reservations, reactor arguments and isolation on the already checked roster. +func verifyV2ClientPods(s inventory, managers []object, workers map[string]bool, expectedManagerImage string) error { + if expectedManagerImage == "" || strings.TrimSpace(expectedManagerImage) != expectedManagerImage { + return errors.New("the reviewed instance-manager image is unresolved") + } + if _, err := indexed(managers); err != nil { + return err + } + pods := map[string]object{} + for _, p := range s.pods { + if text(p, "metadata", "namespace") != "longhorn-system" { + continue + } + name := text(p, "metadata", "name") + if pods[name] != nil { + return errors.New("instance-manager pod inventory is duplicated") + } + pods[name] = p + } + seen := map[string]bool{} + for _, im := range managers { + if text(im, "spec", "dataEngine") == "v1" { + continue + } + if text(im, "spec", "dataEngine") != "v2" { + return errors.New("an instance manager has an unrecognized data engine") + } + node := text(im, "spec", "nodeID") + if !workers[node] || seen[node] || text(im, "spec", "type") != "aio" || value(im, "metadata", "deletionTimestamp") != nil || + text(im, "status", "currentState") != "running" || text(im, "metadata", "namespace") != "longhorn-system" { + return errors.New("V2 instance managers do not match the ready worker roster") + } + p := pods[text(im, "metadata", "name")] + status, _ := value(p, "status").(object) + if text(p, "metadata", "uid") == "" || value(p, "metadata", "deletionTimestamp") != nil || text(p, "spec", "nodeName") != node || text(p, "status", "phase") != "Running" || !ready(status) { + return errors.New("V2 instance-manager pod is missing, misplaced or unready") + } + owned := false + refs, _ := value(p, "metadata", "ownerReferences").([]any) + for _, ref := range refs { + r, _ := ref.(object) + if text(r, "apiVersion") == "longhorn.io/v1beta2" && text(r, "kind") == "InstanceManager" && text(r, "name") == text(im, "metadata", "name") && text(r, "uid") == text(im, "metadata", "uid") && value(r, "controller") == true { + owned = true + } + } + containers, _ := value(p, "spec", "containers").([]any) + if !owned || len(containers) != 1 { + return errors.New("V2 instance-manager workload ownership is unproven") + } + container, _ := containers[0].(object) + if command := value(container, "command"); command != nil { + items, ok := command.([]any) + if !ok || len(items) != 0 { + return errors.New("V2 workload overrides the reviewed image entrypoint") + } + } + cpu := text(container, "resources", "requests", "cpu") + cpuLimit := text(container, "resources", "limits", "cpu") + hugepages := text(container, "resources", "limits", "hugepages-2Mi") + if text(container, "name") != "instance-manager" || text(container, "image") != expectedManagerImage || text(container, "image") != text(im, "spec", "image") || (cpu != "2" && cpu != "2000m") || (cpuLimit != "4" && cpuLimit != "4000m") || (hugepages != "2Gi" && hugepages != "2048Mi") { + return errors.New("V2 workload lacks its effective two-core and two-GiB huge-page reservation") + } + args, _ := value(container, "args").([]any) + if len(args) == 0 || args[0] != "start-spdk-tgt" { + return errors.New("V2 workload does not launch the reviewed SPDK target wrapper") + } + flags := map[string]int{} + positions := map[string]int{} + for i, arg := range args { + a, ok := arg.(string) + if !ok { + return errors.New("V2 workload arguments are malformed") + } + for _, flag := range []string{"--spdk-cpumask", "--spdk-core-number", "--spdk-memory-size", "--enable-spdk", "--spdk-enabled", "--enable-irq-affinity", "--enable-workqueue-affinity", "--enable-rps", "--spdk-no-hugepage", "--spdk-interrupt-mode"} { + if strings.HasPrefix(a, flag+"=") { + return errors.New("V2 workload overrides a reviewed flag using an assignment") + } + } + flags[a]++ + positions[a] = i + } + paired := func(flag, expected string) bool { + count := 0 + for i, arg := range args { + if arg == flag { + if i+1 >= len(args) || args[i+1] != expected { + return false + } + count++ + } + } + return count == 1 + } + if !paired("--spdk-cpumask", "0x3") || !paired("--spdk-core-number", "0") || !paired("--spdk-memory-size", "2048") || + flags["start-spdk-tgt"] != 1 || flags["daemon"] != 1 || flags["--enable-spdk"] != 1 || flags["--spdk-enabled"] != 1 || + positions["--enable-spdk"] >= positions["daemon"] || positions["--spdk-enabled"] <= positions["daemon"] || + flags["--enable-irq-affinity"] != 1 || flags["--enable-workqueue-affinity"] != 1 || flags["--enable-rps"] != 1 || flags["--spdk-no-hugepage"] != 0 || flags["--spdk-interrupt-mode"] != 0 { + return errors.New("V2 workload has not adopted the reviewed polling reactors, huge pages and CPU isolation") + } + seen[node] = true + } + if len(seen) != len(workers) { + return errors.New("V2 volume clients are not ready on every worker") + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/v2_readiness_test.go b/scripts/migrate-longhorn-v2/v2_readiness_test.go new file mode 100644 index 0000000000..a7701cb3c3 --- /dev/null +++ b/scripts/migrate-longhorn-v2/v2_readiness_test.go @@ -0,0 +1,159 @@ +package main + +import ( + "fmt" + "testing" +) + +func v2ReadinessFixture() (inventory, []object) { + return v2ReadinessFixtureForRoute(false) +} + +func v2ReadinessFixtureForRoute(legacy bool) (inventory, []object) { + s := evacuatedFixture() + s.nodes = capacityFixture() + old := []string{"storage-1", "storage-2", "storage-3"} + v1Nodes := []string{"prod-worker-1", "prod-worker-2", "prod-worker-3"} + if legacy { + v1Nodes = []string{"prod-worker-3", text(s.nodes[3], "metadata", "name"), text(s.nodes[4], "metadata", "name")} + } + for i, name := range v1Nodes { + s.storageNodes[i]["metadata"].(object)["name"] = name + for _, r := range s.replicas { + if text(r, "spec", "nodeID") == old[i] { + r["spec"].(object)["nodeID"] = name + } + } + } + for _, n := range s.nodes[:6] { + n["status"].(object)["allocatable"].(object)["cpu"] = "8" + role := text(n, "metadata", "labels", migrationPoolLabel) + if legacy && (text(n, "metadata", "name") == "prod-worker-3" || role == "v1") || !legacy && role == "" { + continue + } + lh := resource(text(n, "metadata", "name")) + lh["spec"] = object{"disks": blockDisks(), "allowScheduling": true, "evictionRequested": false, "instanceManagerCPURequest": float64(0)} + lh["status"] = object{"conditions": readyConditions(), "diskStatus": object{"longhorn-v2": object{"diskUUID": "raw-" + text(n, "metadata", "name"), "conditions": readyConditions()}}} + s.storageNodes = append(s.storageNodes, lh) + } + managers := []object{} + for i, n := range s.nodes[:6] { + im := resource(fmt.Sprintf("im-%d", i)) + im["metadata"].(object)["namespace"] = "longhorn-system" + im["spec"] = object{"nodeID": text(n, "metadata", "name"), "dataEngine": "v2", "type": "aio", "image": "longhornio/longhorn-instance-manager:v1.13.0"} + im["status"] = object{"currentState": "running"} + managers = append(managers, im) + p := resource(text(im, "metadata", "name")) + p["metadata"].(object)["namespace"] = "longhorn-system" + p["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "longhorn.io/v1beta2", "kind": "InstanceManager", "name": text(im, "metadata", "name"), "uid": text(im, "metadata", "uid"), "controller": true}} + p["spec"] = object{"nodeName": text(n, "metadata", "name"), "containers": []any{object{"name": "instance-manager", "image": text(im, "spec", "image"), + "args": []any{"start-spdk-tgt", "--spdk-cpumask", "0x3", "--spdk-core-number", "0", "--spdk-memory-size", "2048", "--enable-spdk", "daemon", "--spdk-enabled", "--enable-irq-affinity", "--enable-workqueue-affinity", "--enable-rps"}, + "resources": object{"requests": object{"cpu": "2"}, "limits": object{"cpu": "4", "hugepages-2Mi": "2Gi"}}}}} + p["status"] = object{"phase": "Running", "conditions": readyConditions()} + s.pods = append(s.pods, p) + } + return s, managers +} + +func TestV2RuntimeRejectsTheEarlierRebuiltPermanentWorkerRoute(t *testing.T) { + pools, err := configuredMigrationPools(poolConfigFixture()) + if err != nil { + t.Fatal(err) + } + s, managers := v2ReadinessFixtureForRoute(true) + if verifyV2Runtime(s, managers, pools, "longhornio/longhorn-instance-manager:v1.13.0") == nil { + t.Fatal("the temporary-only route accepted raw storage on original workers") + } +} + +func TestV2CanaryRequiresRealRawDisksAndReservedReactors(t *testing.T) { + pools, err := configuredMigrationPools(poolConfigFixture()) + if err != nil { + t.Fatal(err) + } + s, managers := v2ReadinessFixture() + if err = verifyV2Runtime(s, managers, pools, "longhornio/longhorn-instance-manager:v1.13.0"); err != nil { + t.Fatal(err) + } + for name, change := range map[string]func(*inventory, *[]object){ + "unrebuilt disk": func(s *inventory, _ *[]object) { s.storageNodes[3]["spec"].(object)["disks"] = object{} }, + "raw disk not ready": func(s *inventory, _ *[]object) { s.storageNodes[3]["status"].(object)["diskStatus"] = object{} }, + "raw disk identity reused": func(s *inventory, _ *[]object) { + s.storageNodes[4]["status"].(object)["diskStatus"].(object)["longhorn-v2"].(object)["diskUUID"] = text(s.storageNodes[3], "status", "diskStatus", "longhorn-v2", "diskUUID") + }, + "retained copy lost": func(s *inventory, _ *[]object) { s.replicas[3]["spec"].(object)["healthyAt"] = "" }, + "client manager missing": func(_ *inventory, m *[]object) { *m = (*m)[1:] }, + "client manager not running": func(_ *inventory, m *[]object) { (*m)[2]["status"].(object)["currentState"] = "starting" }, + "manager pod unready": func(s *inventory, _ *[]object) { s.pods[0]["status"].(object)["conditions"] = []any{} }, + "manager type replaced": func(_ *inventory, m *[]object) { (*m)[0]["spec"].(object)["type"] = "engine" }, + "manager deletion pending": func(_ *inventory, m *[]object) { + (*m)[0]["metadata"].(object)["deletionTimestamp"] = "2026-11-03T09:00:00Z" + }, + "manager identity missing": func(_ *inventory, m *[]object) { delete((*m)[0]["metadata"].(object), "uid") }, + "foreign manager pod": func(s *inventory, _ *[]object) { s.pods[0]["metadata"].(object)["ownerReferences"] = []any{} }, + "entrypoint overridden": func(s *inventory, _ *[]object) { + value(s.pods[0], "spec", "containers").([]any)[0].(object)["command"] = []any{"/bin/true"} + }, + "SPDK bootstrap absent": func(s *inventory, _ *[]object) { + value(s.pods[0], "spec", "containers").([]any)[0].(object)["args"].([]any)[0] = "daemon" + }, + "SPDK launch disabled": func(s *inventory, _ *[]object) { + value(s.pods[0], "spec", "containers").([]any)[0].(object)["args"].([]any)[7] = "--debug" + }, + "V2 daemon disabled": func(s *inventory, _ *[]object) { + value(s.pods[0], "spec", "containers").([]any)[0].(object)["args"].([]any)[9] = "--debug" + }, + "insufficient cpu": func(s *inventory, _ *[]object) { + value(s.pods[0], "spec", "containers").([]any)[0].(object)["resources"].(object)["requests"] = object{"cpu": "480m"} + }, + "reactors consume whole cpu limit": func(s *inventory, _ *[]object) { + value(s.pods[0], "spec", "containers").([]any)[0].(object)["resources"].(object)["limits"].(object)["cpu"] = "2" + }, + "hugepages not reserved": func(s *inventory, _ *[]object) { + value(s.pods[0], "spec", "containers").([]any)[0].(object)["resources"].(object)["limits"] = object{"hugepages-2Mi": "0"} + }, + "one busy reactor": func(s *inventory, _ *[]object) { + value(s.pods[0], "spec", "containers").([]any)[0].(object)["args"].([]any)[2] = "0x1" + }, + "cpu isolation missing": func(s *inventory, _ *[]object) { + c := value(s.pods[0], "spec", "containers").([]any)[0].(object) + c["args"] = c["args"].([]any)[:7] + }, + "node pressured": func(s *inventory, _ *[]object) { s.nodes[0]["status"].(object)["conditions"] = readyConditions() }, + "raw cpu override": func(s *inventory, _ *[]object) { + s.storageNodes[3]["spec"].(object)["instanceManagerCPURequest"] = float64(480) + }, + "raw eviction status missing": func(s *inventory, _ *[]object) { + delete(s.storageNodes[3]["spec"].(object), "evictionRequested") + }, + "matching but unreviewed image": func(s *inventory, m *[]object) { + (*m)[0]["spec"].(object)["image"] = "example.invalid/foreign:latest" + value(s.pods[0], "spec", "containers").([]any)[0].(object)["image"] = "example.invalid/foreign:latest" + }, + "interrupt mode assignment": func(s *inventory, _ *[]object) { + c := value(s.pods[0], "spec", "containers").([]any)[0].(object) + c["args"] = append(c["args"].([]any), "--spdk-interrupt-mode=true") + }, + "hugepage bypass assignment": func(s *inventory, _ *[]object) { + c := value(s.pods[0], "spec", "containers").([]any)[0].(object) + c["args"] = append(c["args"].([]any), "--spdk-no-hugepage=true") + }, + "isolation disabled by later assignment": func(s *inventory, _ *[]object) { + c := value(s.pods[0], "spec", "containers").([]any)[0].(object) + c["args"] = append(c["args"].([]any), "--enable-irq-affinity=false") + }, + "unknown extra manager engine": func(_ *inventory, m *[]object) { + im := resource("unknown-engine") + im["spec"] = object{"dataEngine": "v3"} + *m = append(*m, im) + }, + } { + t.Run(name, func(t *testing.T) { + s, m := v2ReadinessFixture() + change(&s, &m) + if verifyV2Runtime(s, m, pools, "longhornio/longhorn-instance-manager:v1.13.0") == nil { + t.Fatal("V2 canary admitted without its real runtime prerequisites") + } + }) + } +} diff --git a/scripts/validate-eks-ci-role-policy/approved-surface.txt b/scripts/validate-eks-ci-role-policy/approved-surface.txt index 9c05eabc53..b94ac08672 100644 --- a/scripts/validate-eks-ci-role-policy/approved-surface.txt +++ b/scripts/validate-eks-ci-role-policy/approved-surface.txt @@ -84,7 +84,7 @@ helm.toolkit.fluxcd.io/v2|HelmRelease|kubescape|kubescape 6a352e2f5cb05f56d3558d helm.toolkit.fluxcd.io/v2|HelmRelease|kyverno|kyverno b910afb7d7bcbb10e14430ec948732454c95e649bcdb71d16a3855fea07e7fbe -helm.toolkit.fluxcd.io/v2|HelmRelease|longhorn-system|longhorn 3687bc9ddaa8177bc383eb7d25fc8f1bff732a935e2874656fdd8b438aea6e03 +helm.toolkit.fluxcd.io/v2|HelmRelease|longhorn-system|longhorn 10154822248452b0a2b5419cd8d898204fc4f75554b2805407a6cb9bef15ca75 helm.toolkit.fluxcd.io/v2|HelmRelease|oauth2-proxy|oauth2-proxy d16150bb9a7d264f002fd3cb8d82d459a368d6428207e23fd0b8479e4194f60d From 845c59834d3d870bf7747a18ba24b2abd6c9f016 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 10:33:43 +0100 Subject: [PATCH 23/35] fix(storage): bind activation to successful disk preparation --- scripts/migrate-longhorn-v2/README.md | 9 ++- .../temporary_activation.go | 1 + .../temporary_activation_flow_test.go | 66 ++++++++++++++++++- .../temporary_activation_prepared.go | 61 +++++++++++++++++ .../temporary_activation_prepared_test.go | 63 ++++++++++++++++++ .../temporary_activation_record.go | 13 +++- .../temporary_activation_record_test.go | 13 ++++ .../temporary_activation_run.go | 4 ++ 8 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 scripts/migrate-longhorn-v2/temporary_activation_prepared.go create mode 100644 scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index b12a7c2f53..2c3d12a28d 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -86,8 +86,13 @@ maintenance, and ready V2 clients on all six workers before its first write. The journal `longhorn-v2-temporary-activation` binds each temporary selection to the node, provider, hardware, boot, encrypted EPHEMERAL and raw GPT partition -identities. Complete native reads bracket the partition observations. Each -conditional Longhorn Node update registers only the canonical raw disk and adds +identities. Complete native reads bracket the partition observations. Each rebuilt +worker also binds its completed preparation journal UID and the successful +first-attempt main workflow that produced it. A prepared-looking disk without that +record, a replaced journal or a failed preparation workflow cannot authorize +registration. The V2-born worker has no reset operation and requires its native +encrypted raw-layout proof. Each conditional Longhorn Node update registers only +the canonical raw disk and adds the journal UID as its ownership annotation. Matching unowned disks are refused. An uncertain response stops the run. A new first-attempt main dispatch may resume an owned partial registration only after the previous workflow is terminal, and diff --git a/scripts/migrate-longhorn-v2/temporary_activation.go b/scripts/migrate-longhorn-v2/temporary_activation.go index defd842b0f..2f6952d475 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation.go +++ b/scripts/migrate-longhorn-v2/temporary_activation.go @@ -19,6 +19,7 @@ type temporaryV2Binding struct { LonghornUID string `json:"longhornUID"` EphemeralUUID string `json:"ephemeralUUID"` RawPartitionUUID string `json:"rawPartitionUUID"` + PreparationUID string `json:"preparationUID,omitempty"` } var errTemporaryActivation = errors.New("temporary raw storage identity or registration ownership is unproven") diff --git a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go index 371dbdf97e..53f985f107 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go @@ -20,6 +20,8 @@ type activationAPI struct { writes int lostResponse bool priorLive bool + preparations map[string]object + preparationFailed bool } func newActivationAPI(t *testing.T) (*activationAPI, temporaryV2Activation) { @@ -55,6 +57,29 @@ func newActivationAPI(t *testing.T) (*activationAPI, temporaryV2Activation) { if err != nil { t.Fatal(err) } + f.preparations = map[string]object{} + for ordinal, selection := range []string{"temporary-v1-1", "temporary-v1-2"} { + name, _, err := resolveTemporaryInventoryTarget(selection, s.nodes, pools) + if err != nil { + t.Fatal(err) + } + for _, node := range s.nodes { + if text(node, "metadata", "name") != name { + continue + } + var lh object + for _, storage := range s.storageNodes { + if text(storage, "metadata", "name") == name { + lh = storage + } + } + binding := temporaryV2Binding{Selection: selection, Name: name, NodeUID: text(node, "metadata", "uid"), ProviderID: text(node, "spec", "providerID"), + MachineUUID: text(node, "status", "nodeInfo", "systemUUID"), LonghornUID: text(lh, "metadata", "uid")} + cm := temporaryPreparedFixture(t, binding) + cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), `"ownerRun":"7777"`, fmt.Sprintf(`"ownerRun":"%d"`, 7777+ordinal), 1) + f.preparations[selection] = cm + } + } a := temporaryV2Activation{c: client{context: "fixture", run: f.run}, source: strings.Repeat("a", 40), owner: "456", pools: pools} a.reader = func(_ context.Context, address string) (nativeReplicaReader, func() error, error) { for _, node := range f.s.nodes { @@ -77,13 +102,18 @@ func (f *activationAPI) run(_ context.Context, input []byte, program string, arg if program == "gh" && len(args) == 2 && args[0] == "api" { source := strings.Repeat("1", 40) status := "completed" - if !strings.Contains(args[1], "/5678/") { + conclusion := "success" + if strings.Contains(args[1], "/7777/") || strings.Contains(args[1], "/7778/") { + if f.preparationFailed { + conclusion = "failure" + } + } else if !strings.Contains(args[1], "/5678/") { source = strings.Repeat("a", 40) if f.priorLive { status = "in_progress" } } - return json.Marshal(object{"status": status, "head_sha": source, "head_branch": "main", "event": "workflow_dispatch", "run_attempt": 1, "path": ".github/workflows/prepare-longhorn-v2-node.yaml"}) + return json.Marshal(object{"status": status, "conclusion": conclusion, "head_sha": source, "head_branch": "main", "event": "workflow_dispatch", "run_attempt": 1, "path": ".github/workflows/prepare-longhorn-v2-node.yaml"}) } if program == "talosctl" && len(args) == 5 && args[1] == "get" && args[4] == "--output=json" { for i, node := range f.s.nodes { @@ -123,6 +153,14 @@ func (f *activationAPI) run(_ context.Context, input []byte, program string, arg } name := args[2] if kind == "configmap" { + for selection, prepared := range f.preparations { + if name == "longhorn-v2-prepare-"+selection { + if prepared == nil { + return nil, nil + } + return json.Marshal(prepared) + } + } if name == temporaryActivationJournal { if f.record == nil { return nil, nil @@ -216,6 +254,26 @@ func TestTemporaryActivationRegistersOnlyThreeJournalOwnedRawDisks(t *testing.T) } } +func TestTemporaryActivationHoldsBeforeWritesWithoutSuccessfulPreparation(t *testing.T) { + for _, mode := range []string{"missing preparation", "failed preparation", "unfinished preparation"} { + t.Run(mode, func(t *testing.T) { + f, a := newActivationAPI(t) + switch mode { + case "missing preparation": + f.preparations["temporary-v1-1"] = nil + case "failed preparation": + f.preparationFailed = true + case "unfinished preparation": + cm := f.preparations["temporary-v1-1"] + cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), `"phase":"prepared"`, `"phase":"reset-requested"`, 1) + } + if err := a.run(t.Context()); err == nil || f.writes != 0 || f.record != nil { + t.Fatal("activation wrote before successful native preparation was bound:", err) + } + }) + } +} + func TestTemporaryActivationResumesItsUncertainWriteWithoutAdoptingForeignStorage(t *testing.T) { f, a := newActivationAPI(t) f.lostResponse = true @@ -249,7 +307,7 @@ func TestCanaryRequiresCompletedStillOwnedTemporaryActivation(t *testing.T) { if a.accepted(t.Context()) == nil || f.writes != 0 { t.Fatal("an unregistered pool authorized the canary") } - for _, mode := range []string{"valid", "journal replaced", "worker changed", "workflow live", "disk fenced"} { + for _, mode := range []string{"valid", "journal replaced", "worker changed", "workflow live", "disk fenced", "preparation replaced"} { t.Run(mode, func(t *testing.T) { f, a := newActivationAPI(t) if err := a.run(t.Context()); err != nil { @@ -264,6 +322,8 @@ func TestCanaryRequiresCompletedStillOwnedTemporaryActivation(t *testing.T) { f.priorLive = true case "disk fenced": f.s.storageNodes[3]["spec"].(object)["allowScheduling"] = false + case "preparation replaced": + f.preparations["temporary-v1-1"]["metadata"].(object)["uid"] = "replacement-preparation" } before, _ := json.Marshal(f.record) err := a.accepted(t.Context()) diff --git a/scripts/migrate-longhorn-v2/temporary_activation_prepared.go b/scripts/migrate-longhorn-v2/temporary_activation_prepared.go new file mode 100644 index 0000000000..2bd8c54aa8 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation_prepared.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +// preparedTemporaryBinding validates a completed unused-storage reset journal +// against the current worker and its changed boot and EPHEMERAL partition. +func preparedTemporaryBinding(cm object, binding temporaryV2Binding) (journal, error) { + name := "longhorn-v2-prepare-" + binding.Selection + if text(cm, "metadata", "name") != name || text(cm, "metadata", "namespace") != "longhorn-system" || + text(cm, "metadata", "uid") == "" || text(cm, "metadata", "resourceVersion") == "" || value(cm, "metadata", "deletionTimestamp") != nil { + return journal{}, errTemporaryActivation + } + data := []byte(text(cm, "data", "record")) + fields := map[string][]string{"": {"schema", "target", "source", "ownerRun", "phase", "nodeUID", "providerID", "machineUUID", "longhornUID", "ephemeralUUID", "originalDisks", "temporarySelection", "nativeEmptyDigest", "originalBootID", "rawVolumeAbsent"}} + var record journal + if len(data) == 0 || len(data) > 32<<10 || strictjson.Validate(data, fields) != nil || json.Unmarshal(data, &record) != nil { + return journal{}, errTemporaryActivation + } + p := preparation{target: binding.Name, name: name, temporarySelection: binding.Selection} + if p.validateRecord(record) != nil || record.Schema != 2 || record.Phase != "prepared" || record.NodeUID != binding.NodeUID || + record.ProviderID != binding.ProviderID || record.MachineUUID != binding.MachineUUID || record.LonghornUID != binding.LonghornUID || + record.OriginalBootID == binding.BootID || strings.EqualFold(record.EphemeralUUID, binding.EphemeralUUID) { + return journal{}, errTemporaryActivation + } + return record, nil +} + +// prepared requires successful completion of the original protected preparation +// before enabling a rebuilt worker's raw disk. The V2-born pool has no reset +// operation; its independently checked native raw layout remains mandatory. +func (a temporaryV2Activation) prepared(ctx context.Context, binding temporaryV2Binding) (temporaryV2Binding, error) { + if binding.Selection == "temporary-v2-1" { + return binding, nil + } + if binding.Selection != "temporary-v1-1" && binding.Selection != "temporary-v1-2" { + return temporaryV2Binding{}, errTemporaryActivation + } + cm, err := a.c.get(ctx, "configmap", "longhorn-system", "longhorn-v2-prepare-"+binding.Selection, false) + if err != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + record, err := preparedTemporaryBinding(cm, binding) + if err != nil { + return temporaryV2Binding{}, err + } + data, err := a.c.run(ctx, nil, "gh", "api", "repos/devantler-tech/platform/actions/runs/"+record.OwnerRun+"/attempts/1") + var run object + if err != nil || json.Unmarshal(data, &run) != nil || text(run, "status") != "completed" || text(run, "conclusion") != "success" || + text(run, "head_sha") != record.Source || text(run, "head_branch") != "main" || text(run, "event") != "workflow_dispatch" || + value(run, "run_attempt") != float64(1) || !reviewedWorkflowPath(text(run, "path")) { + return temporaryV2Binding{}, errTemporaryActivation + } + binding.PreparationUID = text(cm, "metadata", "uid") + return binding, nil +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go b/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go new file mode 100644 index 0000000000..d166412fa7 --- /dev/null +++ b/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func temporaryPreparedFixture(t *testing.T, binding temporaryV2Binding) object { + t.Helper() + j := journal{Schema: 2, Target: binding.Name, Source: strings.Repeat("1", 40), OwnerRun: "7777", Phase: "prepared", + NodeUID: binding.NodeUID, ProviderID: binding.ProviderID, MachineUUID: binding.MachineUUID, LonghornUID: binding.LonghornUID, + EphemeralUUID: "original-ephemeral", OriginalDisks: object{}, TemporarySelection: binding.Selection, + NativeEmptyDigest: strings.Repeat("c", 64), OriginalBootID: "12345678-1234-4234-8234-000000000099", RawVolumeAbsent: true} + data, err := json.Marshal(j) + if err != nil { + t.Fatal(err) + } + cm := resource("longhorn-v2-prepare-" + binding.Selection) + cm["metadata"].(object)["namespace"] = "longhorn-system" + cm["data"] = object{"record": string(data)} + return cm +} + +func TestTemporaryActivationRequiresTheCompletedPreparationIdentity(t *testing.T) { + n, lh, raw, eph, pool := temporaryRawBindingFixture() + binding, err := projectTemporaryV2Binding("temporary-v1-1", n, lh, raw, eph, pool, text(n, "status", "nodeInfo", "bootID")) + if err != nil { + t.Fatal(err) + } + cm := temporaryPreparedFixture(t, binding) + if _, err = preparedTemporaryBinding(cm, binding); err != nil { + t.Fatal("completed matching preparation was refused:", err) + } + for name, change := range map[string]func(object){ + "foreign scope": func(cm object) { cm["metadata"].(object)["namespace"] = "other" }, + "missing identity": func(cm object) { delete(cm["metadata"].(object), "uid") }, + "pending deletion": func(cm object) { cm["metadata"].(object)["deletionTimestamp"] = "2026-11-03T09:00:00Z" }, + "unfinished preparation": func(cm object) { + cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), `"phase":"prepared"`, `"phase":"reset-requested"`, 1) + }, + "replaced worker": func(cm object) { + cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), binding.NodeUID, "other-node", 1) + }, + "unchanged boot": func(cm object) { + cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), "12345678-1234-4234-8234-000000000099", binding.BootID, 1) + }, + "unchanged ephemeral partition": func(cm object) { + cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), "original-ephemeral", binding.EphemeralUUID, 1) + }, + "ambiguous phase": func(cm object) { + cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), `"phase":"prepared"`, `"phase":"reset-requested","phase":"prepared"`, 1) + }, + } { + t.Run(name, func(t *testing.T) { + cm := temporaryPreparedFixture(t, binding) + change(cm) + if _, err := preparedTemporaryBinding(cm, binding); err == nil { + t.Fatal("incomplete or foreign preparation authorized raw storage") + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_record.go b/scripts/migrate-longhorn-v2/temporary_activation_record.go index 376068dea6..ddb18a9e46 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_record.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_record.go @@ -28,7 +28,7 @@ func decodeTemporaryActivationRecord(data []byte) (temporaryActivationRecord, er return temporaryActivationRecord{}, errTemporaryActivation } canonical := map[string][]string{"": {"version", "source", "ownerRun", "targets", "completed"}} - fields := []string{"selection", "name", "nodeUID", "providerID", "machineUUID", "bootID", "osImage", "longhornUID", "ephemeralUUID", "rawPartitionUUID"} + fields := []string{"selection", "name", "nodeUID", "providerID", "machineUUID", "bootID", "osImage", "longhornUID", "ephemeralUUID", "rawPartitionUUID", "preparationUID"} for _, selection := range temporaryV2Selections { canonical["targets."+selection] = fields } @@ -52,6 +52,7 @@ func (r temporaryActivationRecord) validate() error { return errTemporaryActivation } seen := [7]map[string]bool{} + preparations := map[string]bool{} for i := range seen { seen[i] = map[string]bool{} } @@ -64,6 +65,16 @@ func (r temporaryActivationRecord) validate() error { !inventoryBoot.MatchString(strings.ToLower(b.RawPartitionUUID)) || strings.EqualFold(b.RawPartitionUUID, b.EphemeralUUID) { return errTemporaryActivation } + if selection == "temporary-v2-1" { + if b.PreparationUID != "" { + return errTemporaryActivation + } + } else { + if b.PreparationUID == "" || preparations[b.PreparationUID] { + return errTemporaryActivation + } + preparations[b.PreparationUID] = true + } for i, identity := range []string{b.Name, b.NodeUID, b.ProviderID, strings.ToLower(b.MachineUUID), b.LonghornUID, strings.ToLower(b.EphemeralUUID), strings.ToLower(b.RawPartitionUUID)} { if seen[i][identity] { return errTemporaryActivation diff --git a/scripts/migrate-longhorn-v2/temporary_activation_record_test.go b/scripts/migrate-longhorn-v2/temporary_activation_record_test.go index ce04507c5a..b2f03a83bc 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_record_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_record_test.go @@ -22,6 +22,9 @@ func temporaryActivationRecordFixture(t *testing.T) temporaryActivationRecord { b.NodeUID, b.ProviderID, b.MachineUUID = fmt.Sprintf("node-%d", i), fmt.Sprintf("hcloud://%d", 100+i), fmt.Sprintf("hardware-%d", i) b.LonghornUID, b.EphemeralUUID = fmt.Sprintf("longhorn-%d", i), fmt.Sprintf("ephemeral-%d", i) b.RawPartitionUUID = fmt.Sprintf("12345678-1234-4234-8234-%012x", i+10) + if selection != "temporary-v2-1" { + b.PreparationUID = fmt.Sprintf("preparation-%d", i) + } record.Targets[selection] = b } return record @@ -37,6 +40,16 @@ func TestTemporaryActivationRecordRequiresThreeDistinctCompleteBindings(t *testi "unreviewed source": func(r *temporaryActivationRecord) { r.Source = "unknown" }, "missing owner": func(r *temporaryActivationRecord) { r.OwnerRun = "" }, "missing third disk": func(r *temporaryActivationRecord) { delete(r.Targets, "temporary-v2-1") }, + "missing preparation identity": func(r *temporaryActivationRecord) { + b := r.Targets["temporary-v1-1"] + b.PreparationUID = "" + r.Targets["temporary-v1-1"] = b + }, + "duplicated preparation identity": func(r *temporaryActivationRecord) { + b := r.Targets["temporary-v1-2"] + b.PreparationUID = r.Targets["temporary-v1-1"].PreparationUID + r.Targets["temporary-v1-2"] = b + }, "wrong target pool": func(r *temporaryActivationRecord) { b := r.Targets["temporary-v2-1"] b.Name = r.Targets["temporary-v1-1"].Name diff --git a/scripts/migrate-longhorn-v2/temporary_activation_run.go b/scripts/migrate-longhorn-v2/temporary_activation_run.go index d00ac5bf46..f64c92d81d 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_run.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_run.go @@ -126,6 +126,10 @@ func (a temporaryV2Activation) targets(ctx context.Context, s inventory, journal } role := text(nodes[name], "metadata", "labels", migrationPoolLabel) binding, err := a.readBinding(ctx, selection, nodes[name], storage[name], a.pools[role]) + if err != nil { + return nil, errTemporaryActivation + } + binding, err = a.prepared(ctx, binding) if err != nil || expected != nil && expected[selection] != binding { return nil, errTemporaryActivation } From a13ee4f2eba5241be7b55266592f65090711234b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 12:02:43 +0100 Subject: [PATCH 24/35] fix(storage): recognize completed raw pool transitions --- internal/longhornmigration/activation.go | 134 ++++++++++++++++++ internal/longhornmigration/pools.go | 7 +- internal/longhornmigration/pools_test.go | 8 +- .../maintain-longhorn-clients/admission.go | 2 +- scripts/maintain-longhorn-clients/main.go | 2 +- .../migration_capacity.go | 12 +- .../migration_capacity_test.go | 85 +++++++++++ scripts/migrate-longhorn-v2/README.md | 6 + .../temporary_activation.go | 18 +-- .../temporary_activation_flow_test.go | 16 +++ .../temporary_activation_record.go | 83 ++--------- .../temporary_activation_run.go | 2 +- 12 files changed, 275 insertions(+), 100 deletions(-) create mode 100644 internal/longhornmigration/activation.go diff --git a/internal/longhornmigration/activation.go b/internal/longhornmigration/activation.go new file mode 100644 index 0000000000..13966b50a9 --- /dev/null +++ b/internal/longhornmigration/activation.go @@ -0,0 +1,134 @@ +package longhornmigration + +import ( + "bytes" + "encoding/json" + "errors" + "regexp" + "strings" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +const ( + // ActivationJournal is the fixed three-worker raw registration receipt. + ActivationJournal = "longhorn-v2-temporary-activation" + // ActivationOwner binds registered storage to that receipt's Kubernetes UID. + ActivationOwner = "storage.platform.devantler.tech/v2-activation" +) + +// ActivationBinding retains the native identities verified before registration. +type ActivationBinding struct { + Selection string `json:"selection"` + Name string `json:"name"` + NodeUID string `json:"nodeUID"` + ProviderID string `json:"providerID"` + MachineUUID string `json:"machineUUID"` + BootID string `json:"bootID"` + OSImage string `json:"osImage"` + LonghornUID string `json:"longhornUID"` + EphemeralUUID string `json:"ephemeralUUID"` + RawPartitionUUID string `json:"rawPartitionUUID"` + PreparationUID string `json:"preparationUID,omitempty"` +} + +// ActivationRecord is shared by the registering writer and publication readers. +type ActivationRecord struct { + Version int `json:"version"` + Source string `json:"source"` + OwnerRun string `json:"ownerRun"` + Targets map[string]ActivationBinding `json:"targets"` + Completed bool `json:"completed"` +} + +var errActivation = errors.New("temporary raw storage identity or registration ownership is unproven") + +// ActivationSelections returns the fixed roster without exposing mutable state. +func ActivationSelections() [3]string { + return [3]string{"temporary-v1-1", "temporary-v1-2", "temporary-v2-1"} +} + +// DecodeActivationRecord rejects oversized, duplicate, aliased and unknown JSON. +func DecodeActivationRecord(data []byte) (ActivationRecord, error) { + if len(data) == 0 || len(data) > 32<<10 { + return ActivationRecord{}, errActivation + } + canonical := map[string][]string{"": {"version", "source", "ownerRun", "targets", "completed"}} + fields := []string{"selection", "name", "nodeUID", "providerID", "machineUUID", "bootID", "osImage", "longhornUID", "ephemeralUUID", "rawPartitionUUID", "preparationUID"} + for _, selection := range ActivationSelections() { + canonical["targets."+selection] = fields + } + if strictjson.Validate(data, canonical) != nil { + return ActivationRecord{}, errActivation + } + var record ActivationRecord + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if decoder.Decode(&record) != nil || record.Validate() != nil { + return ActivationRecord{}, errActivation + } + return record, nil +} + +// Validate requires the three distinct workers, partitions and preparation UIDs. +// It validates receipt structure; native callers still establish live acceptance. +func (r ActivationRecord) Validate() error { + if r.Version != 1 || !regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString(r.Source) || !regexp.MustCompile(`^[1-9][0-9]*$`).MatchString(r.OwnerRun) || len(r.Targets) != 3 { + return errActivation + } + uuid := regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + seen := [7]map[string]bool{} + preparations := map[string]bool{} + for i := range seen { + seen[i] = map[string]bool{} + } + for _, selection := range ActivationSelections() { + b, ok := r.Targets[selection] + if !ok || b.Selection != selection || !regexp.MustCompile(`^autoscale-longhorn-(v[12])-[0-9a-f]{16}$`).MatchString(b.Name) || + !strings.HasPrefix(b.Name, "autoscale-longhorn-"+strings.Split(selection, "-")[1]+"-") || + b.NodeUID == "" || !regexp.MustCompile(`^hcloud://[1-9][0-9]*$`).MatchString(b.ProviderID) || b.MachineUUID == "" || + !uuid.MatchString(b.BootID) || b.OSImage == "" || b.LonghornUID == "" || b.EphemeralUUID == "" || + !uuid.MatchString(strings.ToLower(b.RawPartitionUUID)) || strings.EqualFold(b.RawPartitionUUID, b.EphemeralUUID) { + return errActivation + } + if selection == "temporary-v2-1" { + if b.PreparationUID != "" { + return errActivation + } + } else { + if b.PreparationUID == "" || preparations[b.PreparationUID] { + return errActivation + } + preparations[b.PreparationUID] = true + } + for i, identity := range []string{b.Name, b.NodeUID, b.ProviderID, strings.ToLower(b.MachineUUID), b.LonghornUID, strings.ToLower(b.EphemeralUUID), strings.ToLower(b.RawPartitionUUID)} { + if seen[i][identity] { + return errActivation + } + seen[i][identity] = true + } + } + return nil +} + +// registeredRawTransition checks stable ownership for the read-only pool +// exception. A later normal reboot or OS upgrade does not revoke registration; +// destructive operations continue to require their separate native boot proof. +func registeredRawTransition(n, lh, cm Object) bool { + if text(cm, "metadata", "name") != ActivationJournal || text(cm, "metadata", "namespace") != "longhorn-system" || + text(cm, "metadata", "uid") == "" || value(cm, "metadata", "deletionTimestamp") != nil || + text(lh, "metadata", "annotations", ActivationOwner) != text(cm, "metadata", "uid") { + return false + } + r, err := DecodeActivationRecord([]byte(text(cm, "data", "record"))) + if err != nil || !r.Completed { + return false + } + for _, b := range r.Targets { + if b.Name == text(n, "metadata", "name") { + return b.NodeUID == text(n, "metadata", "uid") && b.ProviderID == text(n, "spec", "providerID") && + strings.EqualFold(b.MachineUUID, text(n, "status", "nodeInfo", "systemUUID")) && b.LonghornUID == text(lh, "metadata", "uid") + } + } + return false +} diff --git a/internal/longhornmigration/pools.go b/internal/longhornmigration/pools.go index eb47fb2676..0029f8e832 100644 --- a/internal/longhornmigration/pools.go +++ b/internal/longhornmigration/pools.go @@ -108,7 +108,7 @@ func VerifyNode(n Object, pool Pool) error { // Eligible grants no writes. Readers still enforce ordinary compute-node // exclusion on false, and treat every returned error as a hold. -func Eligible(data []byte, n, lh Object) (bool, error) { +func Eligible(data []byte, n, lh, activation Object) (bool, error) { enabled := text(lh, "metadata", "labels", EnabledLabel) if enabled == "" { return false, nil @@ -138,12 +138,15 @@ func Eligible(data []byte, n, lh Object) (bool, error) { return false, errors.New("migration disk scope is incomplete") } var disk Object - if role == "v1" { + if role == "v1" && disks["longhorn-v2"] == nil { disk, _ = disks["migration-v1"].(map[string]any) if text(disk, "path") != "/var/lib/longhorn" || text(disk, "diskType") != "filesystem" { return false, errors.New("migration filesystem differs from its reviewed scope") } } else { + if role == "v1" && !registeredRawTransition(n, lh, activation) { + return false, errors.New("migration raw transition lacks its completed identity-bound registration") + } disk, _ = disks["longhorn-v2"].(map[string]any) if text(disk, "path") != "/dev/mapper/luks2-r-longhorn" || text(disk, "diskType") != "block" || text(disk, "diskDriver") != "aio" { return false, errors.New("migration raw disk differs from its reviewed scope") diff --git a/internal/longhornmigration/pools_test.go b/internal/longhornmigration/pools_test.go index a0b3b24e66..8941fb43f0 100644 --- a/internal/longhornmigration/pools_test.go +++ b/internal/longhornmigration/pools_test.go @@ -28,7 +28,7 @@ func eligibilityFixture(role string) (Object, Object) { func TestOnlyReviewedPinnedMigrationPoolsAreStorageExceptions(t *testing.T) { for _, role := range []string{"v1", "v2"} { n, lh := eligibilityFixture(role) - if ok, err := Eligible(sourceFixture(), n, lh); !ok || err != nil { + if ok, err := Eligible(sourceFixture(), n, lh, nil); !ok || err != nil { t.Fatalf("verified %s pool rejected: %v", role, err) } } @@ -55,7 +55,7 @@ func TestOnlyReviewedPinnedMigrationPoolsAreStorageExceptions(t *testing.T) { t.Run(name, func(t *testing.T) { n, lh := eligibilityFixture("v1") mutate(n, lh) - if ok, err := Eligible(sourceFixture(), n, lh); ok || err == nil { + if ok, err := Eligible(sourceFixture(), n, lh, nil); ok || err == nil { t.Fatal("unverified storage exception accepted") } }) @@ -74,11 +74,11 @@ func TestFixedPoolMinimumCannotBeRemovedWhileStorageIsAuthorized(t *testing.T) { if err != nil { t.Fatal(err) } - if ok, err := Eligible(b, n, lh); ok || err == nil { + if ok, err := Eligible(b, n, lh, nil); ok || err == nil { t.Fatal("a zero-minimum pool retained its storage exception") } delete(lh["metadata"].(Object)["labels"].(Object), EnabledLabel) - if ok, err := Eligible(nil, n, lh); ok || err != nil { + if ok, err := Eligible(nil, n, lh, nil); ok || err != nil { t.Fatal("ordinary compute classification depends on temporary migration source") } } diff --git a/scripts/maintain-longhorn-clients/admission.go b/scripts/maintain-longhorn-clients/admission.go index b1dc550088..4867b19807 100644 --- a/scripts/maintain-longhorn-clients/admission.go +++ b/scripts/maintain-longhorn-clients/admission.go @@ -71,7 +71,7 @@ func (e engine) replicaExclusion(ctx context.Context) error { if !strings.HasPrefix(name, "autoscale-") { continue } - eligible, proofErr := e.migrationStorageEligible(byName[name], n) + eligible, proofErr := e.migrationStorageEligible(ctx, byName[name], n) if proofErr != nil { return proofErr } diff --git a/scripts/maintain-longhorn-clients/main.go b/scripts/maintain-longhorn-clients/main.go index f26f98d4d1..fb3131c36f 100644 --- a/scripts/maintain-longhorn-clients/main.go +++ b/scripts/maintain-longhorn-clients/main.go @@ -472,7 +472,7 @@ func (e engine) clientsReady(ctx context.Context) (bool, error) { if l == nil { return false, nil } - eligible, proofErr := e.migrationStorageEligible(n, l) + eligible, proofErr := e.migrationStorageEligible(ctx, n, l) if proofErr != nil { return false, proofErr } diff --git a/scripts/maintain-longhorn-clients/migration_capacity.go b/scripts/maintain-longhorn-clients/migration_capacity.go index c6ca9f62b3..f207dcb9e3 100644 --- a/scripts/maintain-longhorn-clients/migration_capacity.go +++ b/scripts/maintain-longhorn-clients/migration_capacity.go @@ -1,12 +1,13 @@ package main import ( + "context" "os" "github.com/devantler-tech/platform/internal/longhornmigration" ) -func (e engine) migrationStorageEligible(node, storage object) (bool, error) { +func (e engine) migrationStorageEligible(ctx context.Context, node, storage object) (bool, error) { if str(storage, "metadata", "labels", longhornmigration.EnabledLabel) == "" { return false, nil } @@ -18,5 +19,12 @@ func (e engine) migrationStorageEligible(node, storage object) (bool, error) { if err != nil { return false, err } - return longhornmigration.Eligible(data, node, storage) + var activation object + if str(node, "metadata", "labels", longhornmigration.PoolLabel) == "v1" && val(storage, "spec", "disks", "longhorn-v2") != nil { + activation, err = e.c.read(ctx, "configmap", "longhorn-system", longhornmigration.ActivationJournal) + if err != nil { + return false, err + } + } + return longhornmigration.Eligible(data, node, storage, activation) } diff --git a/scripts/maintain-longhorn-clients/migration_capacity_test.go b/scripts/maintain-longhorn-clients/migration_capacity_test.go index 54b0cea6c7..a0f98acfc4 100644 --- a/scripts/maintain-longhorn-clients/migration_capacity_test.go +++ b/scripts/maintain-longhorn-clients/migration_capacity_test.go @@ -2,6 +2,9 @@ package main import ( "context" + "encoding/json" + "fmt" + "strings" "testing" "github.com/devantler-tech/platform/internal/longhornmigration" @@ -53,6 +56,88 @@ func TestClientReadinessRecognizesOnlyProtectedMigrationStorage(t *testing.T) { } } +// rawMigrationFixture represents a completed registration, independently of +// the activation builder. A pool's original allocation role remains unchanged. +func rawMigrationFixture(t *testing.T) (*fakeAPI, engine, object, object, object) { + t.Helper() + a, e, n, lh := protectedMigrationFixture() + lh["metadata"].(object)["annotations"] = object{"storage.platform.devantler.tech/v2-activation": "activation-uid"} + lh["spec"].(object)["disks"] = object{"longhorn-v2": object{"diskType": "block", "path": "/dev/mapper/luks2-r-longhorn", "diskDriver": "aio", "allowScheduling": true, "evictionRequested": false}} + targets := object{} + for i, selection := range []string{"temporary-v1-1", "temporary-v1-2", "temporary-v2-1"} { + role := "v1" + if i == 2 { + role = "v2" + } + binding := object{"selection": selection, "name": fmt.Sprintf("autoscale-longhorn-%s-%016x", role, i+1), "nodeUID": fmt.Sprintf("node-%d", i), "providerID": fmt.Sprintf("hcloud://%d", i+1), "machineUUID": fmt.Sprintf("machine-%d", i), "bootID": "11111111-1111-1111-1111-111111111111", "osImage": "Talos (v1.13.12)", "longhornUID": fmt.Sprintf("longhorn-%d", i), "ephemeralUUID": fmt.Sprintf("ephemeral-%d", i), "rawPartitionUUID": fmt.Sprintf("22222222-2222-2222-2222-%012d", i+1)} + if i < 2 { + binding["preparationUID"] = fmt.Sprintf("preparation-%d", i) + } + if i == 0 { + binding["name"], binding["nodeUID"], binding["providerID"] = str(n, "metadata", "name"), str(n, "metadata", "uid"), "hcloud://42" + binding["machineUUID"], binding["longhornUID"] = "machine", str(lh, "metadata", "uid") + } + targets[selection] = binding + } + record, err := json.Marshal(object{"version": 1, "source": strings.Repeat("a", 40), "ownerRun": "1234", "completed": true, "targets": targets}) + if err != nil { + t.Fatal(err) + } + cm := resource("ConfigMap", "longhorn-system", "longhorn-v2-temporary-activation") + cm["metadata"].(object)["uid"] = "activation-uid" + cm["data"] = object{"record": string(record)} + a.objects[fakeKey("configmap", "longhorn-system", "longhorn-v2-temporary-activation")] = []object{cm} + return a, e, n, lh, cm +} + +func TestCompletedRawTransitionPreservesPublicationAndReplicaExclusion(t *testing.T) { + _, e, _, _, _ := rawMigrationFixture(t) + if ok, err := e.clientsReady(context.Background()); !ok || err != nil { + t.Fatalf("completed raw registration blocks production publication: %v", err) + } + if err := e.replicaExclusion(context.Background()); err != nil { + t.Fatalf("completed raw registration loses its storage exception: %v", err) + } +} + +func TestRawTransitionCannotBorrowAnotherActivationOrWorkerIdentity(t *testing.T) { + for name, mutate := range map[string]func(*fakeAPI, object, object, object){ + "missing journal": func(a *fakeAPI, _, _, _ object) { + delete(a.objects, fakeKey("configmap", "longhorn-system", "longhorn-v2-temporary-activation")) + }, + "replaced journal": func(_ *fakeAPI, _, _ object, cm object) { cm["metadata"].(object)["uid"] = "replacement" }, + "incomplete registration": func(_ *fakeAPI, _, _ object, cm object) { + cm["data"].(object)["record"] = strings.Replace(str(cm, "data", "record"), `"completed":true`, `"completed":false`, 1) + }, + "replaced Kubernetes node": func(_ *fakeAPI, n, _, _ object) { n["metadata"].(object)["uid"] = "replacement" }, + "replaced provider": func(_ *fakeAPI, n, _, _ object) { n["spec"].(object)["providerID"] = "hcloud://99" }, + "replaced hardware": func(_ *fakeAPI, n, _, _ object) { + n["status"].(object)["nodeInfo"].(object)["systemUUID"] = "replacement" + }, + "replaced Longhorn node": func(_ *fakeAPI, _, lh, _ object) { lh["metadata"].(object)["uid"] = "replacement" }, + "foreign activation owner": func(_ *fakeAPI, _, lh, _ object) { + lh["metadata"].(object)["annotations"].(object)["storage.platform.devantler.tech/v2-activation"] = "foreign" + }, + "deleting journal": func(_ *fakeAPI, _, _ object, cm object) { + cm["metadata"].(object)["deletionTimestamp"] = "2026-11-03T00:00:00Z" + }, + "ambiguous completion": func(_ *fakeAPI, _, _ object, cm object) { + cm["data"].(object)["record"] = strings.Replace(str(cm, "data", "record"), `"completed":true`, `"completed":false,"completed":true`, 1) + }, + } { + t.Run(name, func(t *testing.T) { + a, e, n, lh, cm := rawMigrationFixture(t) + mutate(a, n, lh, cm) + if ok, err := e.clientsReady(context.Background()); ok || err == nil { + t.Fatal("unbound raw registration accepted for publication") + } + if err := e.replicaExclusion(context.Background()); err == nil { + t.Fatal("unbound raw registration accepted for replica storage") + } + }) + } +} + func TestReplicaExclusionAdmitsOnlyProtectedMigrationReplicas(t *testing.T) { _, e, n, _ := protectedMigrationFixture() if err := e.replicaExclusion(context.Background()); err != nil { diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 2c3d12a28d..fae4fcff40 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -100,6 +100,12 @@ it revalidates all three native bindings and every remaining prerequisite. Completion requires three independent ready raw disks and all qualified clients; it grants no application cutover or original-disk reset authority. +The two rebuilt workers retain their original `migration-pool=v1` allocation +label. The shared publication and replica-exclusion checks recognize their raw +disks only with the completed activation journal, its matching ownership UID, +and the unchanged Kubernetes, provider, hardware and Longhorn node identities. +An ordinary later reboot or OS upgrade does not revoke this registration. + Run `canary-v2` with `verify-longhorn-v2-restore` for filesystem backup recovery, then `canary-csi-v2` with `verify-longhorn-v2-csi-restore` for CSI snapshot data movement. Each has a distinct journal and confirmation. Both first revalidate the diff --git a/scripts/migrate-longhorn-v2/temporary_activation.go b/scripts/migrate-longhorn-v2/temporary_activation.go index 2f6952d475..1a0386e142 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation.go +++ b/scripts/migrate-longhorn-v2/temporary_activation.go @@ -6,25 +6,15 @@ import ( "regexp" "strings" "time" + + "github.com/devantler-tech/platform/internal/longhornmigration" ) -type temporaryV2Binding struct { - Selection string `json:"selection"` - Name string `json:"name"` - NodeUID string `json:"nodeUID"` - ProviderID string `json:"providerID"` - MachineUUID string `json:"machineUUID"` - BootID string `json:"bootID"` - OSImage string `json:"osImage"` - LonghornUID string `json:"longhornUID"` - EphemeralUUID string `json:"ephemeralUUID"` - RawPartitionUUID string `json:"rawPartitionUUID"` - PreparationUID string `json:"preparationUID,omitempty"` -} +type temporaryV2Binding = longhornmigration.ActivationBinding var errTemporaryActivation = errors.New("temporary raw storage identity or registration ownership is unproven") -const temporaryActivationOwner = "storage.platform.devantler.tech/v2-activation" +const temporaryActivationOwner = longhornmigration.ActivationOwner // authorizeTemporaryActivation requires a distinct confirmation for registering // the fixed three-disk pool through the protected first-attempt main workflow. diff --git a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go index 53f985f107..2906c8d95b 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go @@ -10,6 +10,8 @@ import ( "strings" "testing" "time" + + "github.com/devantler-tech/platform/internal/longhornmigration" ) type activationAPI struct { @@ -252,6 +254,20 @@ func TestTemporaryActivationRegistersOnlyThreeJournalOwnedRawDisks(t *testing.T) if err = a.run(t.Context()); err != nil || f.writes != 3 { t.Fatal("completed activation repeated a registration:", err) } + for _, node := range f.s.nodes { + if text(node, "metadata", "labels", longhornmigration.PoolLabel) == "" { + continue + } + var storage object + for _, lh := range f.s.storageNodes { + if text(lh, "metadata", "name") == text(node, "metadata", "name") { + storage = lh + } + } + if eligible, err := longhornmigration.Eligible(poolConfigFixture(), node, storage, f.record); !eligible || err != nil { + t.Fatal("actual registration output blocks the shared publication gate:", err) + } + } } func TestTemporaryActivationHoldsBeforeWritesWithoutSuccessfulPreparation(t *testing.T) { diff --git a/scripts/migrate-longhorn-v2/temporary_activation_record.go b/scripts/migrate-longhorn-v2/temporary_activation_record.go index ddb18a9e46..d492617d0a 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_record.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_record.go @@ -1,86 +1,19 @@ package main -import ( - "bytes" - "encoding/json" - "regexp" - "strings" +import "github.com/devantler-tech/platform/internal/longhornmigration" - "github.com/devantler-tech/platform/internal/strictjson" -) +const temporaryActivationJournal = longhornmigration.ActivationJournal -const temporaryActivationJournal = "longhorn-v2-temporary-activation" +var temporaryV2Selections = longhornmigration.ActivationSelections() -var temporaryV2Selections = [...]string{"temporary-v1-1", "temporary-v1-2", "temporary-v2-1"} +type temporaryActivationRecord = longhornmigration.ActivationRecord -type temporaryActivationRecord struct { - Version int `json:"version"` - Source string `json:"source"` - OwnerRun string `json:"ownerRun"` - Targets map[string]temporaryV2Binding `json:"targets"` - Completed bool `json:"completed"` -} - -// decodeTemporaryActivationRecord rejects ambiguous JSON before its typed -// decode, including duplicate members and case aliases in every fixed target. +// decodeTemporaryActivationRecord shares the strict receipt parser with the +// read-only publication gate, while preserving this operation's error boundary. func decodeTemporaryActivationRecord(data []byte) (temporaryActivationRecord, error) { - if len(data) == 0 || len(data) > 32<<10 { - return temporaryActivationRecord{}, errTemporaryActivation - } - canonical := map[string][]string{"": {"version", "source", "ownerRun", "targets", "completed"}} - fields := []string{"selection", "name", "nodeUID", "providerID", "machineUUID", "bootID", "osImage", "longhornUID", "ephemeralUUID", "rawPartitionUUID", "preparationUID"} - for _, selection := range temporaryV2Selections { - canonical["targets."+selection] = fields - } - if strictjson.Validate(data, canonical) != nil { - return temporaryActivationRecord{}, errTemporaryActivation - } - var record temporaryActivationRecord - decoder := json.NewDecoder(bytes.NewReader(data)) - decoder.DisallowUnknownFields() - if decoder.Decode(&record) != nil || record.validate() != nil { + record, err := longhornmigration.DecodeActivationRecord(data) + if err != nil { return temporaryActivationRecord{}, errTemporaryActivation } return record, nil } - -// validate requires three independent fixed pool members and complete native -// partition identities. An existing journal cannot substitute another worker or -// reuse one physical disk under multiple selections when a run resumes. -func (r temporaryActivationRecord) validate() error { - if r.Version != 1 || !regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString(r.Source) || !regexp.MustCompile(`^[1-9][0-9]*$`).MatchString(r.OwnerRun) || len(r.Targets) != len(temporaryV2Selections) { - return errTemporaryActivation - } - seen := [7]map[string]bool{} - preparations := map[string]bool{} - for i := range seen { - seen[i] = map[string]bool{} - } - for _, selection := range temporaryV2Selections { - b, ok := r.Targets[selection] - if !ok || b.Selection != selection || !inventoryTemporaryNode.MatchString(b.Name) || - !strings.HasPrefix(b.Name, "autoscale-longhorn-"+strings.Split(selection, "-")[1]+"-") || - b.NodeUID == "" || !regexp.MustCompile(`^hcloud://[1-9][0-9]*$`).MatchString(b.ProviderID) || b.MachineUUID == "" || - !inventoryBoot.MatchString(b.BootID) || b.OSImage == "" || b.LonghornUID == "" || b.EphemeralUUID == "" || - !inventoryBoot.MatchString(strings.ToLower(b.RawPartitionUUID)) || strings.EqualFold(b.RawPartitionUUID, b.EphemeralUUID) { - return errTemporaryActivation - } - if selection == "temporary-v2-1" { - if b.PreparationUID != "" { - return errTemporaryActivation - } - } else { - if b.PreparationUID == "" || preparations[b.PreparationUID] { - return errTemporaryActivation - } - preparations[b.PreparationUID] = true - } - for i, identity := range []string{b.Name, b.NodeUID, b.ProviderID, strings.ToLower(b.MachineUUID), b.LonghornUID, strings.ToLower(b.EphemeralUUID), strings.ToLower(b.RawPartitionUUID)} { - if seen[i][identity] { - return errTemporaryActivation - } - seen[i][identity] = true - } - } - return nil -} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_run.go b/scripts/migrate-longhorn-v2/temporary_activation_run.go index f64c92d81d..fbb34877a8 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_run.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_run.go @@ -26,7 +26,7 @@ func (a temporaryV2Activation) read(ctx context.Context) (object, temporaryActiv // persist conditionally writes the plan on current main, then verifies the // same journal identity and complete record through a fresh native read. func (a temporaryV2Activation) persist(ctx context.Context, cm object, record temporaryActivationRecord) (object, error) { - if record.validate() != nil || record.Source != a.source || record.OwnerRun != a.owner { + if record.Validate() != nil || record.Source != a.source || record.OwnerRun != a.owner { return nil, errTemporaryActivation } data, err := json.Marshal(record) From 0723c2b1130f40ff4480b2c85c6b9c52c5c5e136 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 14:05:46 +0100 Subject: [PATCH 25/35] test(storage): keep activation OS metadata synthetic --- scripts/maintain-longhorn-clients/migration_capacity_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/maintain-longhorn-clients/migration_capacity_test.go b/scripts/maintain-longhorn-clients/migration_capacity_test.go index a0f98acfc4..d255e7c7e8 100644 --- a/scripts/maintain-longhorn-clients/migration_capacity_test.go +++ b/scripts/maintain-longhorn-clients/migration_capacity_test.go @@ -69,7 +69,7 @@ func rawMigrationFixture(t *testing.T) (*fakeAPI, engine, object, object, object if i == 2 { role = "v2" } - binding := object{"selection": selection, "name": fmt.Sprintf("autoscale-longhorn-%s-%016x", role, i+1), "nodeUID": fmt.Sprintf("node-%d", i), "providerID": fmt.Sprintf("hcloud://%d", i+1), "machineUUID": fmt.Sprintf("machine-%d", i), "bootID": "11111111-1111-1111-1111-111111111111", "osImage": "Talos (v1.13.12)", "longhornUID": fmt.Sprintf("longhorn-%d", i), "ephemeralUUID": fmt.Sprintf("ephemeral-%d", i), "rawPartitionUUID": fmt.Sprintf("22222222-2222-2222-2222-%012d", i+1)} + binding := object{"selection": selection, "name": fmt.Sprintf("autoscale-longhorn-%s-%016x", role, i+1), "nodeUID": fmt.Sprintf("node-%d", i), "providerID": fmt.Sprintf("hcloud://%d", i+1), "machineUUID": fmt.Sprintf("machine-%d", i), "bootID": "11111111-1111-1111-1111-111111111111", "osImage": "Talos (fixture)", "longhornUID": fmt.Sprintf("longhorn-%d", i), "ephemeralUUID": fmt.Sprintf("ephemeral-%d", i), "rawPartitionUUID": fmt.Sprintf("22222222-2222-2222-2222-%012d", i+1)} if i < 2 { binding["preparationUID"] = fmt.Sprintf("preparation-%d", i) } From 1cb334e8d531499e762b8c962eba81c6ecd16397 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 15:16:28 +0100 Subject: [PATCH 26/35] fix(storage): restore Kopia canaries onto independent volumes --- scripts/migrate-longhorn-v2/README.md | 6 +- scripts/migrate-longhorn-v2/canary.go | 7 ++- scripts/migrate-longhorn-v2/canary_assets.go | 2 + .../migrate-longhorn-v2/canary_assets_test.go | 31 ++++++++++ .../migrate-longhorn-v2/canary_repository.go | 39 ++++++++++++ .../canary_repository_test.go | 60 +++++++++++++++++++ .../migrate-longhorn-v2/canary_transfer.go | 2 +- .../canary_transfer_test.go | 6 +- 8 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 scripts/migrate-longhorn-v2/canary_repository.go create mode 100644 scripts/migrate-longhorn-v2/canary_repository_test.go diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 5dd283d99a..693b8965b6 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -119,7 +119,11 @@ three-replica V2 backend, compare the original bytes and sentinel, then reattach the same V2 backend from a second permanent worker and compare again. Backup metadata alone cannot pass either probe. Filesystem recovery binds the -native PodVolumeBackup and PodVolumeRestore; CSI recovery binds the DataUpload, +native PodVolumeBackup and PodVolumeRestore to the same Ready Kopia repository +UID and specification, selected by Velero's source-namespace, storage-location +and repository-type key. Kopia does not use the Restic-only repository identifier. +The restored claim's source PV binding is removed before independent V2 +provisioning. CSI recovery binds the DataUpload, DataDownload, claim, PV and Longhorn backend, including short-lived scratch identities captured by bounded native list/watch streams. Successful cleanup is verified before the journal records completion. Any failed or interrupted probe diff --git a/scripts/migrate-longhorn-v2/canary.go b/scripts/migrate-longhorn-v2/canary.go index 7580d62ff5..469aaa58b3 100644 --- a/scripts/migrate-longhorn-v2/canary.go +++ b/scripts/migrate-longhorn-v2/canary.go @@ -34,6 +34,7 @@ type canaryRun struct { journalUID string csi *csiCapture sourceClaim, sourceVolume, snapshotClass object + kopiaRepository object } // canaryWorkload binds recovery evidence to both reviewed payload commands. @@ -292,13 +293,17 @@ func (p *canaryRun) transfer(ctx context.Context, desired, pod object, expectedS if err != nil { return err } + repositories, err := p.c.list(ctx, "backuprepositories.velero.io", "velero") + if err != nil { + return err + } if kind == "Restore" { pod, err = p.c.get(ctx, "pod", "longhorn-canary-v2-"+p.run, "fio", false) if err != nil { return err } } - snapshot, err = verifyCanaryTransfer(transfers, parent, pod, expectedSnapshot, sourceNS) + snapshot, err = p.verifyFilesystemTransfer(repositories, transfers, parent, pod, expectedSnapshot, sourceNS) return err }) return parent, snapshot, err diff --git a/scripts/migrate-longhorn-v2/canary_assets.go b/scripts/migrate-longhorn-v2/canary_assets.go index f3ca394264..c2631fcedd 100644 --- a/scripts/migrate-longhorn-v2/canary_assets.go +++ b/scripts/migrate-longhorn-v2/canary_assets.go @@ -89,6 +89,8 @@ resourceModifierRules: - operation: replace path: /spec/storageClassName value: "%s" + - operation: remove + path: /spec/volumeName - conditions: groupResource: pods resourceNameRegex: "^fio$" diff --git a/scripts/migrate-longhorn-v2/canary_assets_test.go b/scripts/migrate-longhorn-v2/canary_assets_test.go index b5d8830bcf..d249a326e1 100644 --- a/scripts/migrate-longhorn-v2/canary_assets_test.go +++ b/scripts/migrate-longhorn-v2/canary_assets_test.go @@ -1,9 +1,11 @@ package main import ( + "encoding/json" "strings" "testing" + jsonpatch "gopkg.in/evanphx/json-patch.v4" "sigs.k8s.io/yaml" ) @@ -54,6 +56,35 @@ func TestCanaryRestoresOnlyItsOwnFilesystemDataOntoV2(t *testing.T) { if err := yaml.Unmarshal([]byte(rules), &native); err != nil { t.Fatal("modifier cannot decode Velero's native string-valued JSON patches:", err) } + // The backup excludes PVs, so the modifier receives a claim still bound to + // its V1 source PV. Apply the emitted RFC 6902 operations to that real shape. + var operations []object + for _, patch := range native.ResourceModifierRules[0].Patches { + op := object{"op": patch.Operation, "path": patch.Path} + if patch.Value != "" { + op["value"] = patch.Value + } + operations = append(operations, op) + } + encoded, err := json.Marshal(operations) + if err != nil { + t.Fatal(err) + } + patch, err := jsonpatch.DecodePatch(encoded) + if err != nil { + t.Fatal(err) + } + restored, err := patch.Apply([]byte(`{"spec":{"storageClassName":"lh-canary-v1-1234","volumeName":"source-v1-pv"}}`)) + if err != nil { + t.Fatal(err) + } + var claim object + if err := json.Unmarshal(restored, &claim); err != nil { + t.Fatal(err) + } + if value(claim, "spec", "volumeName") != nil || text(claim, "spec", "storageClassName") != "lh-canary-v2-1234" { + t.Fatal("restored claim retains its source PV binding instead of provisioning V2 storage") + } for _, required := range []string{"longhorn-canary-v1-1234", "^fio-data$", "^fio$", "/spec/storageClassName", "lh-canary-v2-1234", "operation: test"} { if !strings.Contains(rules, required) { t.Fatal("restore modifier lacks exact source or destination binding") diff --git a/scripts/migrate-longhorn-v2/canary_repository.go b/scripts/migrate-longhorn-v2/canary_repository.go new file mode 100644 index 0000000000..2cf4391905 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_repository.go @@ -0,0 +1,39 @@ +package main + +import ( + "errors" + "reflect" +) + +// Velero v1.18.4 selects Kopia repositories by this label tuple. Its Restic +// identifier is empty for Kopia. Bind the supported key, ready native object +// and spec, retaining that same repository across backup and restore. +func (p *canaryRun) verifyFilesystemTransfer(repositories, transfers []object, parent, pod object, expectedSnapshot, sourceNS string) (string, error) { + if sourceNS == "" || text(parent, "kind") == "Backup" && text(pod, "metadata", "namespace") != sourceNS || text(parent, "kind") == "Restore" && p.kopiaRepository == nil { + return "", errors.New("canary Kopia source repository is unbound") + } + var selected object + for _, repository := range repositories { + if text(repository, "spec", "volumeNamespace") != sourceNS && text(repository, "metadata", "labels", "velero.io/volume-namespace") != sourceNS { + continue + } + if selected != nil || text(repository, "apiVersion") != "velero.io/v1" || text(repository, "kind") != "BackupRepository" || + text(repository, "metadata", "namespace") != "velero" || text(repository, "metadata", "name") == "" || text(repository, "metadata", "uid") == "" || text(repository, "metadata", "resourceVersion") == "" || value(repository, "metadata", "deletionTimestamp") != nil || + text(repository, "metadata", "labels", "velero.io/volume-namespace") != sourceNS || text(repository, "metadata", "labels", "velero.io/storage-location") != "default" || text(repository, "metadata", "labels", "velero.io/repository-type") != "kopia" || + text(repository, "spec", "volumeNamespace") != sourceNS || text(repository, "spec", "backupStorageLocation") != "default" || text(repository, "spec", "repositoryType") != "kopia" || text(repository, "spec", "resticIdentifier") != "" || text(repository, "status", "phase") != "Ready" { + return "", errors.New("canary Kopia repository key, identity or readiness is unproven") + } + selected = repository + } + if selected == nil || p.kopiaRepository != nil && (benchmarkIdentity(selected) != benchmarkIdentity(p.kopiaRepository) || !reflect.DeepEqual(value(selected, "spec"), value(p.kopiaRepository, "spec"))) { + return "", errors.New("canary Kopia repository is missing or changed") + } + snapshot, err := verifyCanaryTransfer(transfers, parent, pod, expectedSnapshot, sourceNS) + if err != nil { + return "", err + } + if text(parent, "kind") == "Backup" { + p.kopiaRepository = selected + } + return snapshot, nil +} diff --git a/scripts/migrate-longhorn-v2/canary_repository_test.go b/scripts/migrate-longhorn-v2/canary_repository_test.go new file mode 100644 index 0000000000..3a91a21018 --- /dev/null +++ b/scripts/migrate-longhorn-v2/canary_repository_test.go @@ -0,0 +1,60 @@ +package main + +import "testing" + +func canaryRepositoryFixture() object { + repository := resource("native-kopia-repository") + repository["apiVersion"], repository["kind"] = "velero.io/v1", "BackupRepository" + repository["metadata"].(object)["namespace"] = "velero" + repository["metadata"].(object)["labels"] = object{"velero.io/volume-namespace": "longhorn-canary-v1-1234", "velero.io/storage-location": "default", "velero.io/repository-type": "kopia"} + repository["spec"] = object{"volumeNamespace": "longhorn-canary-v1-1234", "backupStorageLocation": "default", "repositoryType": "kopia"} + repository["status"] = object{"phase": "Ready"} + return repository +} + +func TestFilesystemRecoveryUsesTheSameReadyKopiaRepository(t *testing.T) { + var p canaryRun + repository := canaryRepositoryFixture() + parent, pod, transfer := transferFixture("Backup") + if _, err := p.verifyFilesystemTransfer([]object{repository}, []object{transfer}, parent, pod, "", "longhorn-canary-v1-1234"); err != nil { + t.Fatal("native Kopia backup without a Restic identifier was refused:", err) + } + parent, pod, transfer = transferFixture("Restore") + pod["metadata"].(object)["namespace"] = "longhorn-canary-v2-1234" + transfer["spec"].(object)["pod"].(object)["namespace"] = "longhorn-canary-v2-1234" + current := canaryRepositoryFixture() + current["metadata"].(object)["resourceVersion"] = "2" + current["status"].(object)["lastMaintenanceTime"] = "2026-11-03T00:00:00Z" + if _, err := p.verifyFilesystemTransfer([]object{current}, []object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err != nil { + t.Fatal("ordinary repository status updates broke restoration:", err) + } + for name, mutate := range map[string]func(object){ + "replaced repository": func(o object) { o["metadata"].(object)["uid"] = "replacement" }, + "changed repository config": func(o object) { o["spec"].(object)["repositoryConfig"] = object{"changed": "true"} }, + "wrong location": func(o object) { o["spec"].(object)["backupStorageLocation"] = "other" }, + "wrong labels": func(o object) { o["metadata"].(object)["labels"].(object)["velero.io/volume-namespace"] = "other" }, + "wrong source": func(o object) { o["spec"].(object)["volumeNamespace"] = "other" }, + "wrong type": func(o object) { o["spec"].(object)["repositoryType"] = "restic" }, + "not ready": func(o object) { o["status"].(object)["phase"] = "NotReady" }, + "deleting": func(o object) { o["metadata"].(object)["deletionTimestamp"] = "2026-11-03T00:00:00Z" }, + } { + t.Run(name, func(t *testing.T) { + candidate := canaryRepositoryFixture() + mutate(candidate) + if _, err := p.verifyFilesystemTransfer([]object{candidate}, []object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { + t.Fatal("repository replacement or contradictory native evidence was accepted") + } + }) + } + for name, repositories := range map[string][]object{"missing": nil, "ambiguous": {repository, repository}} { + t.Run(name, func(t *testing.T) { + if _, err := p.verifyFilesystemTransfer(repositories, []object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { + t.Fatal("incomplete repository evidence accepted") + } + }) + } + var unbound canaryRun + if _, err := unbound.verifyFilesystemTransfer([]object{repository}, []object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { + t.Fatal("restore accepted without the verified backup repository") + } +} diff --git a/scripts/migrate-longhorn-v2/canary_transfer.go b/scripts/migrate-longhorn-v2/canary_transfer.go index 861fdb4d7e..6c1b910872 100644 --- a/scripts/migrate-longhorn-v2/canary_transfer.go +++ b/scripts/migrate-longhorn-v2/canary_transfer.go @@ -30,7 +30,7 @@ func verifyCanaryTransfer(transfers []object, parent, pod object, expectedSnapsh matched++ if !owned || text(transfer, "metadata", "namespace") != "velero" || text(transfer, "spec", "pod", "uid") != text(pod, "metadata", "uid") || text(transfer, "spec", "pod", "namespace") != text(pod, "metadata", "namespace") || text(transfer, "spec", "pod", "name") != "fio" || - text(transfer, "spec", "volume") != "data" || text(transfer, "spec", "backupStorageLocation") != "default" || text(transfer, "spec", "repoIdentifier") == "" || text(transfer, "spec", "uploaderType") != "kopia" { + text(transfer, "spec", "volume") != "data" || text(transfer, "spec", "backupStorageLocation") != "default" || text(transfer, "spec", "repoIdentifier") != "" || text(transfer, "spec", "uploaderType") != "kopia" { return "", errors.New("canary data transfer belongs to another pod, volume, repository or operation") } if err := cleanVeleroResult(transfer); err != nil { diff --git a/scripts/migrate-longhorn-v2/canary_transfer_test.go b/scripts/migrate-longhorn-v2/canary_transfer_test.go index 42b986dd37..2c1edb0568 100644 --- a/scripts/migrate-longhorn-v2/canary_transfer_test.go +++ b/scripts/migrate-longhorn-v2/canary_transfer_test.go @@ -8,7 +8,8 @@ func transferFixture(kind string) (object, object, object) { pod["metadata"].(object)["namespace"] = "longhorn-canary-v1-1234" transfer["metadata"].(object)["namespace"] = "velero" transfer["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "velero.io/v1", "kind": kind, "name": text(parent, "metadata", "name"), "uid": text(parent, "metadata", "uid"), "controller": true}} - transfer["spec"] = object{"pod": object{"name": "fio", "namespace": text(pod, "metadata", "namespace"), "uid": text(pod, "metadata", "uid")}, "volume": "data", "backupStorageLocation": "default", "repoIdentifier": "private-repository", "uploaderType": "kopia", "snapshotID": "snapshot", "sourceNamespace": "longhorn-canary-v1-1234"} + // Velero leaves the Restic-only repoIdentifier field absent for Kopia. + transfer["spec"] = object{"pod": object{"name": "fio", "namespace": text(pod, "metadata", "namespace"), "uid": text(pod, "metadata", "uid")}, "volume": "data", "backupStorageLocation": "default", "uploaderType": "kopia", "snapshotID": "snapshot", "sourceNamespace": "longhorn-canary-v1-1234"} transfer["status"] = object{"phase": "Completed", "completionTimestamp": "2026-10-29T19:00:00Z", "snapshotID": "snapshot", "progress": object{"bytesDone": float64(134217800), "totalBytes": float64(134217800)}} return parent, pod, transfer } @@ -23,11 +24,12 @@ func TestCanaryRequiresTheExactCompletedDataTransfer(t *testing.T) { "unrelated parent": func(o object) { o["metadata"].(object)["ownerReferences"] = []any{} }, "wrong pod uid": func(o object) { o["spec"].(object)["pod"].(object)["uid"] = "replacement" }, "wrong volume": func(o object) { o["spec"].(object)["volume"] = "scratch" }, - "no repository": func(o object) { o["spec"].(object)["repoIdentifier"] = "" }, + "wrong backup location": func(o object) { o["spec"].(object)["backupStorageLocation"] = "other" }, "unproven progress": func(o object) { o["status"].(object)["progress"] = object{} }, "incomplete transfer": func(o object) { o["status"].(object)["progress"].(object)["bytesDone"] = float64(32) }, "failed transfer": func(o object) { o["status"].(object)["phase"] = "Failed" }, "restic instead of kopia": func(o object) { o["spec"].(object)["uploaderType"] = "restic" }, + "restic repository id": func(o object) { o["spec"].(object)["repoIdentifier"] = "restic-only" }, } { t.Run(kind+"/"+name, func(t *testing.T) { p, pod, transfer := transferFixture(kind) From 591b22696f885cf28c6106fd4906f667e3fd4692 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 3 Nov 2026 16:49:00 +0100 Subject: [PATCH 27/35] refactor(storage): separate V2 activation from recovery probes --- .../workflows/prepare-longhorn-v2-node.yaml | 23 +- internal/longhornmigration/activation.go | 9 + internal/longhornmigration/pools_test.go | 5 + .../controllers/longhorn/limit-range.yaml | 2 +- scripts/maintain-longhorn-clients/main.go | 2 + .../migration_capacity.go | 2 + .../migration_capacity_test.go | 4 + scripts/migrate-longhorn-v2/README.md | 44 +- scripts/migrate-longhorn-v2/authorization.go | 13 - scripts/migrate-longhorn-v2/canary.go | 524 ------------------ scripts/migrate-longhorn-v2/canary_assets.go | 134 ----- .../migrate-longhorn-v2/canary_assets_test.go | 127 ----- scripts/migrate-longhorn-v2/canary_cleanup.go | 127 ----- .../canary_cleanup_test.go | 35 -- .../migrate-longhorn-v2/canary_cli_test.go | 47 -- .../migrate-longhorn-v2/canary_csi_assets.go | 21 - .../canary_csi_assets_test.go | 32 -- .../migrate-longhorn-v2/canary_csi_binding.go | 53 -- .../canary_csi_binding_test.go | 53 -- .../migrate-longhorn-v2/canary_csi_capture.go | 220 -------- .../canary_csi_capture_test.go | 78 --- .../canary_csi_native_test.go | 87 --- .../migrate-longhorn-v2/canary_csi_runtime.go | 183 ------ .../canary_csi_runtime_test.go | 48 -- .../canary_csi_transfer.go | 65 --- .../canary_csi_transfer_test.go | 84 --- .../canary_identity_test.go | 68 --- .../migrate-longhorn-v2/canary_repository.go | 39 -- .../canary_repository_test.go | 60 -- .../migrate-longhorn-v2/canary_transfer.go | 59 -- .../canary_transfer_test.go | 51 -- scripts/migrate-longhorn-v2/main.go | 33 +- ...temporary_activation_authorization_test.go | 12 +- .../temporary_activation_flow_test.go | 51 +- .../temporary_activation_native_test.go | 3 + .../temporary_activation_prepared_test.go | 4 + .../temporary_activation_record_test.go | 4 + .../temporary_activation_run.go | 10 +- .../temporary_activation_test.go | 6 + scripts/migrate-longhorn-v2/v2_environment.go | 58 ++ .../migrate-longhorn-v2/v2_readiness_test.go | 9 +- 41 files changed, 187 insertions(+), 2302 deletions(-) delete mode 100644 scripts/migrate-longhorn-v2/canary.go delete mode 100644 scripts/migrate-longhorn-v2/canary_assets.go delete mode 100644 scripts/migrate-longhorn-v2/canary_assets_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_cleanup.go delete mode 100644 scripts/migrate-longhorn-v2/canary_cleanup_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_cli_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_assets.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_assets_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_binding.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_binding_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_capture.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_capture_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_native_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_runtime.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_runtime_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_transfer.go delete mode 100644 scripts/migrate-longhorn-v2/canary_csi_transfer_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_identity_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_repository.go delete mode 100644 scripts/migrate-longhorn-v2/canary_repository_test.go delete mode 100644 scripts/migrate-longhorn-v2/canary_transfer.go delete mode 100644 scripts/migrate-longhorn-v2/canary_transfer_test.go create mode 100644 scripts/migrate-longhorn-v2/v2_environment.go diff --git a/.github/workflows/prepare-longhorn-v2-node.yaml b/.github/workflows/prepare-longhorn-v2-node.yaml index 5dd5dda195..c577dfe137 100644 --- a/.github/workflows/prepare-longhorn-v2-node.yaml +++ b/.github/workflows/prepare-longhorn-v2-node.yaml @@ -17,7 +17,7 @@ on: description: The guarded migration step to run required: true type: choice - options: [prepare-worker, prepare-unused-temporary-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas, inventory-storage-root, inventory-storage-layout, inspect-host-memory, reboot-host-for-hugepages, canary-v2, canary-csi-v2, register-temporary-v2-storage] + options: [prepare-worker, prepare-unused-temporary-worker, refresh-clients, benchmark-v1, benchmark-v2, inventory-replicas, inventory-storage-root, inventory-storage-layout, inspect-host-memory, reboot-host-for-hugepages, register-temporary-v2-storage] node: description: Permanent worker or fixed temporary-worker selection for the chosen operation required: false @@ -47,22 +47,7 @@ jobs: with: go-version-file: go.mod cache: false - - name: Install the approved client for hermetic native watch tests - shell: bash - env: - KUBECTL_VERSION: v1.36.2 - KUBECTL_SHA256: 1e9045ec32bea85da43de85f0065358529ea7c7a152eca78154fba5b58c27d82 - run: | - set -euo pipefail - kubectl_path="$RUNNER_TEMP/longhorn-v2-test-kubectl" - curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" -o "$kubectl_path" - printf '%s %s\n' "$KUBECTL_SHA256" "$kubectl_path" | sha256sum --check - mkdir -p "$RUNNER_TEMP/longhorn-v2-test-tools" - install -m 0755 "$kubectl_path" "$RUNNER_TEMP/longhorn-v2-test-tools/kubectl" - printf '%s\n' "$RUNNER_TEMP/longhorn-v2-test-tools" >> "$GITHUB_PATH" - name: Test complete observations and interrupted recovery - env: - LONGHORN_V2_NATIVE_TEST: '1' run: go test -race -count=1 -timeout=5m ./scripts/migrate-longhorn-v2 ./internal/longhornmigration ./internal/longhorninventory prepare: @@ -119,12 +104,6 @@ jobs: benchmark-v1|benchmark-v2) [[ "${LONGHORN_V2_CONFIRM}" == benchmark-longhorn-storage ]] ;; - canary-v2) - [[ "${LONGHORN_V2_CONFIRM}" == verify-longhorn-v2-restore ]] - ;; - canary-csi-v2) - [[ "${LONGHORN_V2_CONFIRM}" == verify-longhorn-v2-csi-restore ]] - ;; register-temporary-v2-storage) [[ "${LONGHORN_V2_CONFIRM}" == register-longhorn-v2-temporary-storage ]] ;; diff --git a/internal/longhornmigration/activation.go b/internal/longhornmigration/activation.go index 13966b50a9..18d8900b5c 100644 --- a/internal/longhornmigration/activation.go +++ b/internal/longhornmigration/activation.go @@ -32,6 +32,15 @@ type ActivationBinding struct { PreparationUID string `json:"preparationUID,omitempty"` } +// SameStorageIdentity compares the durable worker, partition and preparation +// identities. Boot and OS observations describe the original read bracket; a +// later caller must freshly prove its own bracket rather than reuse those values. +func (b ActivationBinding) SameStorageIdentity(other ActivationBinding) bool { + b.BootID, b.OSImage = "", "" + other.BootID, other.OSImage = "", "" + return b == other +} + // ActivationRecord is shared by the registering writer and publication readers. type ActivationRecord struct { Version int `json:"version"` diff --git a/internal/longhornmigration/pools_test.go b/internal/longhornmigration/pools_test.go index 8941fb43f0..0c2a00b5c5 100644 --- a/internal/longhornmigration/pools_test.go +++ b/internal/longhornmigration/pools_test.go @@ -10,6 +10,8 @@ func sourceFixture() []byte { return []byte(`{"spec":{"cluster":{"controlPlanes":3,"workers":3,"autoscaler":{"node":{"maxNodesTotal":9,"pools":[{"name":"autoscale-longhorn-v1","serverType":"cpx42","location":"fsn1","min":2,"max":2,"labels":{"storage.platform.devantler.tech/migration-pool":"v1"}},{"name":"autoscale-longhorn-v2","serverType":"cpx42","location":"fsn1","min":1,"max":1,"labels":{"storage.platform.devantler.tech/migration-pool":"v2"}}]}}},"provider":{"hetzner":{"serverLimit":10}}}}`) } +// eligibilityFixture binds one fixed pool member to owned scale-down protection +// and its role-specific storage, independently of the activation writer. func eligibilityFixture(role string) (Object, Object) { name := "autoscale-longhorn-" + role + "-0123456789abcdef" n := Object{"metadata": Object{"name": name, "uid": "kube-uid", "resourceVersion": "1", "labels": Object{PoolLabel: role, "ksail.io/autoscaled": "true", "node.kubernetes.io/instance-type": "cpx42"}, "annotations": Object{ScaleDownDisabled: "true", OwnerAnnotation: OwnerValue}}, "spec": Object{"providerID": "hcloud://42"}, "status": Object{"nodeInfo": Object{"systemUUID": "machine"}, "conditions": []any{Object{"type": "Ready", "status": "True"}}}} @@ -25,6 +27,8 @@ func eligibilityFixture(role string) (Object, Object) { return n, lh } +// Only fixed reviewed pool members with intact ownership and storage can escape +// ordinary compute-node exclusion; missing or altered authorization holds. func TestOnlyReviewedPinnedMigrationPoolsAreStorageExceptions(t *testing.T) { for _, role := range []string{"v1", "v2"} { n, lh := eligibilityFixture(role) @@ -62,6 +66,7 @@ func TestOnlyReviewedPinnedMigrationPoolsAreStorageExceptions(t *testing.T) { } } +// Fixed pool capacity cannot be reduced while its storage exception is active. func TestFixedPoolMinimumCannotBeRemovedWhileStorageIsAuthorized(t *testing.T) { n, lh := eligibilityFixture("v1") var cfg Object diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/limit-range.yaml b/k8s/providers/hetzner/infrastructure/controllers/longhorn/limit-range.yaml index d455824a3c..c39507bc34 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/longhorn/limit-range.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/limit-range.yaml @@ -23,7 +23,7 @@ # must stay unset (instance-managers must be free to burst memory during a # rebuild). # * defaultRequest is a low floor so a container that omits requests is not -# handed the 2-core limit as its request. +# handed the 4-core limit as its request. apiVersion: v1 kind: LimitRange metadata: diff --git a/scripts/maintain-longhorn-clients/main.go b/scripts/maintain-longhorn-clients/main.go index fb3131c36f..7acc18735f 100644 --- a/scripts/maintain-longhorn-clients/main.go +++ b/scripts/maintain-longhorn-clients/main.go @@ -389,6 +389,8 @@ func (e engine) detached(ctx context.Context, j journal) (bool, error) { return true, nil } +// clientsReady verifies effective Longhorn and CSI workloads and worker +// registration, including only journal-qualified temporary storage exceptions. func (e engine) clientsReady(ctx context.Context) (bool, error) { for _, setting := range []struct{ name, value string }{{"system-managed-components-node-selector", ""}, {"create-default-disk-labeled-nodes", "true"}} { o, err := e.c.read(ctx, "settings.longhorn.io", "longhorn-system", setting.name) diff --git a/scripts/maintain-longhorn-clients/migration_capacity.go b/scripts/maintain-longhorn-clients/migration_capacity.go index f207dcb9e3..2e0be86145 100644 --- a/scripts/maintain-longhorn-clients/migration_capacity.go +++ b/scripts/maintain-longhorn-clients/migration_capacity.go @@ -7,6 +7,8 @@ import ( "github.com/devantler-tech/platform/internal/longhornmigration" ) +// migrationStorageEligible reads the shared activation receipt only for an +// enabled migration pool, retaining ordinary-node exclusion and read failures. func (e engine) migrationStorageEligible(ctx context.Context, node, storage object) (bool, error) { if str(storage, "metadata", "labels", longhornmigration.EnabledLabel) == "" { return false, nil diff --git a/scripts/maintain-longhorn-clients/migration_capacity_test.go b/scripts/maintain-longhorn-clients/migration_capacity_test.go index d255e7c7e8..a40a8c8882 100644 --- a/scripts/maintain-longhorn-clients/migration_capacity_test.go +++ b/scripts/maintain-longhorn-clients/migration_capacity_test.go @@ -90,6 +90,8 @@ func rawMigrationFixture(t *testing.T) (*fakeAPI, engine, object, object, object return a, e, n, lh, cm } +// A completed raw registration must satisfy both publication and replica +// exclusion while the pool retains its original allocation label. func TestCompletedRawTransitionPreservesPublicationAndReplicaExclusion(t *testing.T) { _, e, _, _, _ := rawMigrationFixture(t) if ok, err := e.clientsReady(context.Background()); !ok || err != nil { @@ -100,6 +102,8 @@ func TestCompletedRawTransitionPreservesPublicationAndReplicaExclusion(t *testin } } +// An incomplete, replaced or foreign activation must fail both independent +// readers rather than turn a matching raw path into storage authorization. func TestRawTransitionCannotBorrowAnotherActivationOrWorkerIdentity(t *testing.T) { for name, mutate := range map[string]func(*fakeAPI, object, object, object){ "missing journal": func(a *fakeAPI, _, _, _ object) { diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 693b8965b6..c0ef26ca01 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -59,8 +59,8 @@ may adopt it only after the previous run has ended. This operation enables no V2 storage and removes no iSCSI support. The `Prepare Longhorn V2 Migration` workflow runs guarded steps towards -platform#5582. Its recovery probes use disposable claims; application cutover, -V1 retirement and iSCSI removal have separate acceptance gates. +platform#5582. Application cutover, recovery probes, V1 retirement and iSCSI +removal have separate acceptance gates. This tool lands in stages (platform#5655). It includes `refresh-clients`, `prepare-worker`, `prepare-unused-temporary-worker`, read-only replica inventories and the benchmarks. @@ -69,7 +69,7 @@ huge-page settings are loaded. Host recovery and a cleaned V1 baseline precede temporary-disk preparation. The V2 activation change must follow those completed operations, rather than their source PRs alone. -## Temporary V2 activation and disposable recovery proof +## Temporary V2 activation The activation manifests retain V1 and the existing default StorageClass, enable V2, and add the opt-in `longhorn-v2` Retain class. Eight-CPU workers reserve two @@ -109,34 +109,16 @@ disks only with the completed activation journal, its matching ownership UID, and the unchanged Kubernetes, provider, hardware and Longhorn node identities. An ordinary later reboot or OS upgrade does not revoke this registration. -Run `canary-v2` with `verify-longhorn-v2-restore` for filesystem backup recovery, -then `canary-csi-v2` with `verify-longhorn-v2-csi-restore` for CSI snapshot data -movement. Each has a distinct journal and confirmation. Both first revalidate the -completed activation against its current native identities and require its -owning workflow to be terminal. The probes create their own restricted namespaces, -classes and claims, write and checksum 128 MiB on V1, restore to an independent -three-replica V2 backend, compare the original bytes and sentinel, then reattach -the same V2 backend from a second permanent worker and compare again. - -Backup metadata alone cannot pass either probe. Filesystem recovery binds the -native PodVolumeBackup and PodVolumeRestore to the same Ready Kopia repository -UID and specification, selected by Velero's source-namespace, storage-location -and repository-type key. Kopia does not use the Restic-only repository identifier. -The restored claim's source PV binding is removed before independent V2 -provisioning. CSI recovery binds the DataUpload, -DataDownload, claim, PV and Longhorn backend, including short-lived scratch -identities captured by bounded native list/watch streams. Successful cleanup is -verified before the journal records completion. Any failed or interrupted probe -retains its journal and scratch for diagnosis; it is not rerun or adopted by name. -All original application volumes and historical recovery states remain protected. - -`go run ./scripts/migrate-longhorn-v2 --render-canary 1234` and -`go run ./scripts/migrate-longhorn-v2 --render-csi-canary 1234` render example -disposable assets without production credentials -or mutations. The native watch transport test uses an isolated local HTTP API -fixture and kubeconfig; CI installs the checksum-pinned approved kubectl and runs -it with `LONGHORN_V2_NATIVE_TEST=1`. That protocol test is not a production restore -receipt. Real recovery results come only from the confirmed main dispatches. +The journal retains the original boot and OS observations for audit. Later +registration recovery and acceptance compare its durable worker, partition and +preparation identities, while each native read still requires an unchanged live +boot bracket. A normal reboot does not adopt replacement hardware or storage. + +Filesystem and CSI recovery probes follow separately under platform#5809. +They must verify restored bytes, persistence after reattachment and owned scratch +cleanup before any application cutover. This activation operation has no data +movement or probe entrypoint. All original application volumes and historical +recovery states remain protected. For the unused temporary-worker route (platform#5787), finish the client refresh and the cleaned V1 baseline first. Select `prepare-unused-temporary-worker`, `temporary-v1-1` or diff --git a/scripts/migrate-longhorn-v2/authorization.go b/scripts/migrate-longhorn-v2/authorization.go index 60a6cf174a..248f9ede82 100644 --- a/scripts/migrate-longhorn-v2/authorization.go +++ b/scripts/migrate-longhorn-v2/authorization.go @@ -45,16 +45,3 @@ func authorizeBenchmark(getenv func(string) string, phase string) error { } return authorizeWorkflow(getenv, "benchmark-longhorn-storage") } - -// authorizeCanary keeps filesystem and CSI data movement behind separate -// confirmations in the same first-attempt protected main workflow. -func authorizeCanary(getenv func(string) string) error { - switch getenv("LONGHORN_V2_OPERATION") { - case "canary-v2": - return authorizeWorkflow(getenv, "verify-longhorn-v2-restore") - case "canary-csi-v2": - return authorizeWorkflow(getenv, "verify-longhorn-v2-csi-restore") - default: - return errors.New("the explicit V2 recovery canary operation is required") - } -} diff --git a/scripts/migrate-longhorn-v2/canary.go b/scripts/migrate-longhorn-v2/canary.go deleted file mode 100644 index 469aaa58b3..0000000000 --- a/scripts/migrate-longhorn-v2/canary.go +++ /dev/null @@ -1,524 +0,0 @@ -package main - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "reflect" - "regexp" - "strings" - "time" -) - -type canaryRecord struct { - Version int - Source, Run, Phase, Image, Workload, PayloadSHA256, Snapshot, CompletedAt string - Resources map[string]benchmarkResource - Transport string - CSIScratch map[string]csiScratchEvidence - RestoreVerified, ReattachVerified, Cleaned bool -} - -// The canary is qualified for the instance-manager image shipped by the reviewed -// Longhorn 1.13.0 chart. A chart upgrade requires a new qualification, rather -// than accepting matching but unknown live declarations as image provenance. -const canaryManagerImage = "longhornio/longhorn-instance-manager:v1.13.0" - -type canaryRun struct { - c client - source, run string - record canaryRecord - journalUID string - csi *csiCapture - sourceClaim, sourceVolume, snapshotClass object - kopiaRepository object -} - -// canaryWorkload binds recovery evidence to both reviewed payload commands. -func canaryWorkload() string { - digest := sha256.Sum256([]byte(canaryScript + canaryVerify)) - return hex.EncodeToString(digest[:]) -} - -// save conditionally updates the owned journal without changing previously -// recorded scratch identities, then requires an exact native readback. -func (p *canaryRun) save(ctx context.Context) error { - name, err := canaryJournalName(p.record.Transport) - if err != nil { - return err - } - cm, err := p.c.get(ctx, "configmap", "longhorn-system", name, false) - if err != nil { - return err - } - var old canaryRecord - if p.journalUID == "" || text(cm, "metadata", "uid") != p.journalUID || json.Unmarshal([]byte(text(cm, "data", "record")), &old) != nil || old.Source != p.source || old.Run != p.run || old.Transport != p.record.Transport || old.Cleaned { - return errors.New("canary journal ownership changed") - } - for key, identity := range old.Resources { - if p.record.Resources[key] != identity { - return errors.New("canary scratch identity changed") - } - } - body, err := json.Marshal(p.record) - if err != nil { - return err - } - if err = sourceCurrent(ctx, p.c.run, p.source); err != nil { - return err - } - if err = p.c.patch(ctx, "configmap", "longhorn-system", cm, object{"op": "replace", "path": "/data/record", "value": string(body)}); err != nil { - return err - } - cm, err = p.c.get(ctx, "configmap", "longhorn-system", name, false) - if err != nil || text(cm, "metadata", "uid") != p.journalUID || text(cm, "data", "record") != string(body) { - return errors.New("canary journal update could not be read back") - } - return nil -} - -// remember persists a newly created resource identity before later operations. -func (p *canaryRun) remember(ctx context.Context, key string, o object) error { - if text(o, "metadata", "name") == "" || text(o, "metadata", "uid") == "" { - return errors.New("canary resource identity is incomplete") - } - p.record.Resources[key] = benchmarkIdentity(o) - return p.save(ctx) -} - -// create records the native identity returned by an exclusive scratch creation. -func (p *canaryRun) create(ctx context.Context, key string, desired object) (object, error) { - o, err := p.c.create(ctx, p.source, desired) - if err != nil { - return nil, err - } - if err = p.remember(ctx, key, o); err != nil { - return nil, err - } - return o, nil -} - -// verifyCanaryPod requires the original restricted consumer and executable on -// the selected client before any command reads or writes the payload. -func verifyCanaryPod(p object, uid, node string) error { - status, _ := value(p, "status").(object) - if text(p, "metadata", "uid") != uid || value(p, "metadata", "deletionTimestamp") != nil || text(p, "spec", "nodeName") != node || - value(p, "spec", "automountServiceAccountToken") != false || value(p, "spec", "hostUsers") != false || !ready(status) { - return errors.New("canary consumer identity or placement changed") - } - containers, _ := value(p, "spec", "containers").([]any) - statuses, _ := value(p, "status", "containerStatuses").([]any) - if len(containers) != 1 || len(statuses) != 1 { - return errors.New("canary consumer executable is ambiguous") - } - c, _ := containers[0].(object) - containerStatus, _ := statuses[0].(object) - if text(c, "name") != "fio" || text(c, "image") != fioImage || !reflect.DeepEqual(value(c, "command"), []any{"/bin/sh", "-ec"}) || !reflect.DeepEqual(value(c, "args"), []any{canaryScript}) || - text(containerStatus, "name") != "fio" || !strings.HasSuffix(text(containerStatus, "imageID"), strings.Split(fioImage, "@")[1]) { - return errors.New("canary executable differs from the reviewed probe") - } - return nil -} - -// verifyCanarySettings binds both engine enablement and the effective SPDK -// configuration to the reviewed migration profile before disk registration or IO. -func verifyCanarySettings(ctx context.Context, c client) error { - for name, expected := range map[string]string{ - "v1-data-engine": "true", "v2-data-engine": "true", "guaranteed-instance-manager-cpu": `{"v1":"6","v2":"25"}`, - "data-engine-cpu-mask": `{"v2":"0x3"}`, "data-engine-number-of-cpu-cores": `{"v2":"0"}`, "data-engine-cpu-isolation-enabled": `{"v2":"true"}`, - "data-engine-interrupt-mode-enabled": `{"v2":"false"}`, "data-engine-hugepage-enabled": `{"v2":"true"}`, "data-engine-memory-size": `{"v2":"2048"}`, - } { - o, err := c.get(ctx, "settings.longhorn.io", "longhorn-system", name, false) - if err != nil { - return err - } - actual := text(o, "value") - if strings.HasPrefix(expected, "{") { - var want, got map[string]string - if json.Unmarshal([]byte(expected), &want) != nil || json.Unmarshal([]byte(actual), &got) != nil || !reflect.DeepEqual(want, got) { - return errors.New("live V2 settings have not adopted the reviewed configuration") - } - } else if actual != expected { - return errors.New("both engines must remain enabled during the recovery canary") - } - } - return nil -} - -// canaryEnvironment proves both the actual worker clients and the three ready -// raw disks; enabling V2 alone cannot authorize a recovery probe. -func canaryEnvironment(ctx context.Context, c client, pools map[string]migrationPool) (inventory, error) { - if err := verifyCanarySettings(ctx, c); err != nil { - return inventory{}, err - } - s, err := c.snapshot(ctx) - if err != nil { - return inventory{}, err - } - managers, err := c.list(ctx, "instancemanagers.longhorn.io", "longhorn-system") - if err != nil { - return inventory{}, err - } - if err = verifyV2Runtime(s, managers, pools, canaryManagerImage); err != nil { - return inventory{}, err - } - return s, nil -} - -// verifyCanaryBindingIdentity rejects replacement source objects and target -// claims outside this run's exact native restore. -func verifyCanaryBindingIdentity(phase string, record canaryRecord, class, claim object) error { - group := "source" - if phase == "v2" { - group = "target" - } else if phase != "v1" { - return errors.New("canary engine is outside this procedure") - } - expected := record.Resources[group+"/StorageClass"] - if expected.UID == "" || expected.UID != text(class, "metadata", "uid") || expected.Name != text(class, "metadata", "name") { - return errors.New("canary storage class was replaced") - } - if phase == "v1" { - expected = record.Resources["source/PersistentVolumeClaim"] - if expected.UID == "" || expected.UID != text(claim, "metadata", "uid") || expected.Name != text(claim, "metadata", "name") { - return errors.New("canary source claim was replaced") - } - } else if text(claim, "metadata", "labels", "velero.io/restore-name") != "longhorn-v2-canary-"+record.Run || text(claim, "metadata", "labels", "velero.io/backup-name") != "longhorn-v2-canary-"+record.Run { - return errors.New("V2 claim was not produced by the exact canary restore") - } - return nil -} - -// binding waits for the owned consumer's complete claim/PV/backend graph and -// healthy replicas, preserving each identity for comparison and cleanup. -func (p *canaryRun) binding(ctx context.Context, namespace, class, phase, node, podUID string) (map[string]object, object, object, error) { - var result map[string]object - var pv, volume object - err := waitReady(ctx, func(ctx context.Context) error { - pod, err := p.c.get(ctx, "pod", namespace, "fio", false) - if err != nil || text(pod, "status", "phase") != "Running" { - return errors.New("canary consumer is not running") - } - if err = verifyCanaryPod(pod, podUID, node); err != nil { - return err - } - claim, err := p.c.get(ctx, "persistentvolumeclaim", namespace, "fio-data", false) - if err != nil || text(claim, "status", "phase") != "Bound" || text(claim, "spec", "storageClassName") != class { - return errors.New("canary claim has not bound to its reviewed class") - } - pv, err = p.c.get(ctx, "persistentvolume", "", text(claim, "spec", "volumeName"), false) - if err != nil { - return err - } - volume, err = p.c.get(ctx, "volumes.longhorn.io", "longhorn-system", text(pv, "spec", "csi", "volumeHandle"), false) - if err != nil || text(volume, "spec", "dataEngine") != phase || text(volume, "status", "state") != "attached" || text(volume, "status", "robustness") != "healthy" { - return errors.New("canary backend has not become healthy on the selected engine") - } - classObject, err := p.c.get(ctx, "storageclass", "", class, false) - if err != nil { - return err - } - if err = verifyCanaryBindingIdentity(phase, p.record, classObject, claim); err != nil { - return err - } - if err = verifyScratchBinding(pv, volume, claim, classObject); err != nil { - return err - } - s, err := p.c.snapshot(ctx) - if err != nil { - return err - } - if err = verifyRedundancy(s); err != nil { - return err - } - result = map[string]object{"Pod": pod, "PersistentVolumeClaim": claim, "StorageClass": classObject} - return nil - }) - return result, pv, volume, err -} - -// verifyBytes rebinds the live executable before checking the sentinel, length, -// stored checksum and optional original source digest. -func (p *canaryRun) verifyBytes(ctx context.Context, pod object, node, expected string) (string, error) { - namespace := text(pod, "metadata", "namespace") - fresh, err := p.c.get(ctx, "pod", namespace, "fio", false) - if err != nil { - return "", err - } - if err = verifyCanaryPod(fresh, text(pod, "metadata", "uid"), node); err != nil { - return "", err - } - if _, err = p.c.kubectl(ctx, nil, "exec", "--namespace="+namespace, "fio", "--container=fio", "--", "/bin/sh", "-ec", canaryVerify); err != nil { - return "", errors.New("canary restored bytes failed their checksum or sentinel check") - } - data, err := p.c.kubectl(ctx, nil, "exec", "--namespace="+namespace, "fio", "--container=fio", "--", "sha256sum", "/data/payload.bin") - if err != nil { - return "", err - } - fields := strings.Fields(string(data)) - if len(fields) != 2 || fields[1] != "/data/payload.bin" || !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(fields[0]) || expected != "" && expected != fields[0] { - return "", errors.New("canary payload differs from the original source") - } - return fields[0], nil -} - -// transfer requires a completed owned native transfer; CSI uses its separate -// data-mover proof rather than filesystem-backup fields. -func (p *canaryRun) transfer(ctx context.Context, desired, pod object, expectedSnapshot, sourceNS string) (object, string, error) { - if p.csi != nil { - return p.transferCSI(ctx, desired, expectedSnapshot) - } - kind, query := text(desired, "kind"), "podvolumebackups.velero.io" - if kind == "Restore" { - query = "podvolumerestores.velero.io" - } - created, err := p.create(ctx, kind, desired) - if err != nil { - return nil, "", err - } - var parent object - var snapshot string - err = waitReady(ctx, func(ctx context.Context) error { - parent, err = p.c.get(ctx, strings.ToLower(kind)+"s.velero.io", "velero", text(created, "metadata", "name"), false) - if err != nil || text(parent, "metadata", "uid") != text(created, "metadata", "uid") { - return errors.New("canary backup or restore identity changed") - } - if err = cleanVeleroResult(parent); err != nil { - return err - } - transfers, err := p.c.list(ctx, query, "velero") - if err != nil { - return err - } - repositories, err := p.c.list(ctx, "backuprepositories.velero.io", "velero") - if err != nil { - return err - } - if kind == "Restore" { - pod, err = p.c.get(ctx, "pod", "longhorn-canary-v2-"+p.run, "fio", false) - if err != nil { - return err - } - } - snapshot, err = p.verifyFilesystemTransfer(repositories, transfers, parent, pod, expectedSnapshot, sourceNS) - return err - }) - return parent, snapshot, err -} - -// runCanaryWithTransport owns a single disposable V1-to-V2 recovery trial and -// records acceptance only after byte comparison, remote reattachment and cleanup. -func runCanaryWithTransport(ctx context.Context, c client, source, run string, pools map[string]migrationPool, transport string) error { - journalName, err := canaryJournalName(transport) - if err != nil { - return err - } - assets, err := canaryResources(run) - if transport == "csi" { - assets, err = csiCanaryResources(run) - } - if err != nil { - return err - } - if err = sourceCurrent(ctx, c.run, source); err != nil { - return err - } - if err = (temporaryV2Activation{c: c, pools: pools}).accepted(ctx); err != nil { - return err - } - s, err := canaryEnvironment(ctx, c, pools) - if err != nil { - return err - } - if err = requireBenchmarkBaseline(ctx, c, s.nodes); err != nil { - return err - } - for _, volume := range s.volumes { - if text(volume, "spec", "dataEngine") != "v1" { - return errors.New("the recovery canary must precede application migration") - } - } - bsl, err := c.get(ctx, "backupstoragelocations.velero.io", "velero", "default", false) - if err != nil || text(bsl, "status", "phase") != "Available" || text(bsl, "spec", "accessMode") == "ReadOnly" { - return errors.New("the existing backup destination is unavailable for the recovery canary") - } - old, err := c.get(ctx, "configmap", "longhorn-system", journalName, true) - if err != nil { - return err - } - if old != nil { - return errors.New("a prior canary journal exists; preserve its resources and diagnose before explicit recovery") - } - p := canaryRun{c: c, source: source, run: run, record: canaryRecord{Version: 2, Source: source, Run: run, Transport: transport, Phase: "created", Image: fioImage, Workload: canaryWorkload(), Resources: map[string]benchmarkResource{}, CSIScratch: map[string]csiScratchEvidence{}}} - body, _ := json.Marshal(p.record) - journal, err := c.create(ctx, source, object{"apiVersion": "v1", "kind": "ConfigMap", "metadata": object{"name": journalName, "namespace": "longhorn-system"}, "data": object{"record": string(body)}}) - if err != nil { - return err - } - p.journalUID = text(journal, "metadata", "uid") - created := map[string]map[string]object{"source": {}, "target": {}} - for i, desired := range assets[:8] { - group := "source" - if i >= 5 { - group = "target" - } - kind := text(desired, "kind") - o, err := p.create(ctx, group+"/"+kind, desired) - if err != nil { - return err - } - created[group][kind] = o - } - sourceNS, targetNS := "longhorn-canary-v1-"+run, "longhorn-canary-v2-"+run - sourceBound, sourcePV, sourceVolume, err := p.binding(ctx, sourceNS, "lh-canary-v1-"+run, "v1", "prod-worker-3", text(created["source"]["Pod"], "metadata", "uid")) - if err != nil { - return err - } - for kind, o := range map[string]object{"PersistentVolume": sourcePV, "LonghornVolume": sourceVolume} { - if err = p.remember(ctx, "source/"+kind, o); err != nil { - return err - } - } - p.record.Phase = "source-write-requested" - if err = p.save(ctx); err != nil { - return err - } - if err = sourceCurrent(ctx, c.run, source); err != nil { - return err - } - freshSource, err := c.get(ctx, "pod", sourceNS, "fio", false) - if err != nil { - return err - } - if err = verifyCanaryPod(freshSource, text(sourceBound["Pod"], "metadata", "uid"), "prod-worker-3"); err != nil { - return err - } - fmt.Println("Canary storage provisioned with three healthy copies; writing the disposable source payload.") - if _, err = c.kubectl(ctx, nil, "exec", "--namespace="+sourceNS, "fio", "--container=fio", "--", "/bin/sh", "-ec", "touch /tmp/canary-start"); err != nil { - return err - } - if err = waitReady(ctx, func(ctx context.Context) error { - logs, err := c.kubectl(ctx, nil, "logs", "--namespace="+sourceNS, "fio", "--container=fio") - if err != nil || strings.TrimSpace(string(logs)) != "CANARY_READY" { - return errors.New("source canary has not completed its verified write") - } - return nil - }); err != nil { - return err - } - p.record.PayloadSHA256, err = p.verifyBytes(ctx, sourceBound["Pod"], "prod-worker-3", "") - if err != nil { - return err - } - if _, err = p.create(ctx, "Modifier", assets[8]); err != nil { - return err - } - if transport == "csi" { - p.sourceClaim, p.sourceVolume = sourceBound["PersistentVolumeClaim"], sourceVolume - p.snapshotClass, err = p.create(ctx, "SnapshotClass", assets[11]) - if err != nil { - return err - } - p.csi = newCSICapture(run) - if err = p.csi.start(ctx, c); err != nil { - return err - } - defer p.csi.stop() - } - backup, snapshot, err := p.transfer(ctx, assets[9], sourceBound["Pod"], "", sourceNS) - if err != nil { - return err - } - p.record.Snapshot, p.record.Phase = snapshot, "backup-verified" - if err = p.save(ctx); err != nil { - return err - } - if p.csi != nil { - if err = p.waitCSIUploadGone(ctx); err != nil { - return err - } - } - fmt.Println("Canary backup data transfer verified; restoring onto an independent V2 volume.") - restore, _, err := p.transfer(ctx, assets[10], nil, snapshot, sourceNS) - if err != nil { - return err - } - restoredPod, err := c.get(ctx, "pod", targetNS, "fio", false) - if err != nil { - return err - } - targetBound, targetPV, targetVolume, err := p.binding(ctx, targetNS, "lh-canary-v2-"+run, "v2", "prod-worker-3", text(restoredPod, "metadata", "uid")) - if err != nil { - return err - } - if text(targetPV, "metadata", "uid") == text(sourcePV, "metadata", "uid") || text(targetVolume, "metadata", "uid") == text(sourceVolume, "metadata", "uid") { - return errors.New("restore reused source storage instead of creating an independent V2 backend") - } - if p.csi != nil { - if err = verifyCSIRestoredBinding(p.record.CSIScratch["DataDownload"], targetBound["PersistentVolumeClaim"], targetPV, targetVolume); err != nil { - return err - } - } - for kind, o := range targetBound { - created["target"][kind] = o - if err = p.remember(ctx, "restored/"+kind, o); err != nil { - return err - } - } - for kind, o := range map[string]object{"PersistentVolume": targetPV, "LonghornVolume": targetVolume} { - if err = p.remember(ctx, "target/"+kind, o); err != nil { - return err - } - } - if _, err = p.verifyBytes(ctx, targetBound["Pod"], "prod-worker-3", p.record.PayloadSHA256); err != nil { - return err - } - p.record.RestoreVerified, p.record.Phase = true, "restore-verified" - if err = p.save(ctx); err != nil { - return err - } - fmt.Println("Restored bytes match the source; checking persistence on another worker.") - if err = p.reattach(ctx, created["target"], targetPV, targetVolume); err != nil { - return err - } - p.record.ReattachVerified, p.record.Phase = true, "reattach-verified" - if err = p.save(ctx); err != nil { - return err - } - if p.csi != nil { - if err = p.removeBackup(ctx, backup, restore, assets[8]); err != nil { - return err - } - if err = p.waitCSIArtifactsGone(ctx); err != nil { - return err - } - } - if err = cleanupBenchmark(ctx, c, source, targetNS, created["target"], targetPV, targetVolume); err != nil { - return err - } - if err = cleanupBenchmark(ctx, c, source, sourceNS, created["source"], sourcePV, sourceVolume); err != nil { - return err - } - if p.csi == nil { - if err = p.removeBackup(ctx, backup, restore, assets[8]); err != nil { - return err - } - } else { - if err = p.removeCSISnapshotClass(ctx); err != nil { - return err - } - p.csi.stop() - if err = p.csi.health(); err != nil { - return err - } - } - p.record.Cleaned, p.record.Phase, p.record.CompletedAt = true, "completed", time.Now().UTC().Format(time.RFC3339) - if err = p.save(ctx); err != nil { - return err - } - fmt.Println("V2 recovery canary passed: independent three-replica restore, original-byte comparison, remote-client reattachment and scratch cleanup verified.") - return nil -} diff --git a/scripts/migrate-longhorn-v2/canary_assets.go b/scripts/migrate-longhorn-v2/canary_assets.go deleted file mode 100644 index c2631fcedd..0000000000 --- a/scripts/migrate-longhorn-v2/canary_assets.go +++ /dev/null @@ -1,134 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "regexp" - "time" -) - -const canaryScript = `set -eu -while [ ! -f /tmp/canary-start ]; do sleep 2; done -fio --name=longhorn-v2-canary --filename=/data/payload.bin --size=128M --rw=write --bs=4k --ioengine=libaio --direct=1 --verify=sha256 --do_verify=1 --verify_fatal=1 --verify_state_save=0 --end_fsync=1 --output=/tmp/probe.json --output-format=json -cd /data -sha256sum payload.bin > payload.sha256 -printf 'longhorn-v2-5582\n' > sentinel -sync -printf 'CANARY_READY\n' -while :; do sleep 10; done -` - -const canaryVerify = `set -eu -cd /data -test "$(cat sentinel)" = longhorn-v2-5582 -test "$(wc -c < payload.bin)" -eq 134217728 -sha256sum --check payload.sha256 -` - -// canaryProbeResources reuses the reviewed restricted benchmark workload while -// giving this recovery proof its own names, script, and backup-visible labels. -func canaryProbeResources(namespace, class, runID, phase string) ([]object, error) { - assets, err := benchmarkResources(phase, runID, "longhorn-benchmark-"+phase+"-"+runID) - if err != nil { - return nil, err - } - for _, asset := range assets { - metadata := asset["metadata"].(object) - delete(metadata["labels"].(object), "velero.io/exclude-from-backup") - metadata["labels"].(object)["storage.platform.devantler.tech/canary-run"] = runID - if text(asset, "kind") == "Namespace" { - metadata["name"] = namespace - } else if text(asset, "kind") == "StorageClass" { - metadata["name"] = class - asset["parameters"].(object)["nodeSelector"] = "" - if phase == "v2" { - asset["parameters"].(object)["unmapMarkSnapChainRemoved"] = "disabled" - } - } else { - metadata["namespace"] = namespace - } - } - assets[3]["spec"].(object)["storageClassName"] = class - value(assets[4], "spec", "containers").([]any)[0].(object)["args"] = []any{canaryScript} - return assets, nil -} - -// canaryResources limits backup and restore to this run's disposable workload. -func canaryResources(runID string) ([]object, error) { - if !regexp.MustCompile(`^[1-9][0-9]*$`).MatchString(runID) { - return nil, errors.New("immutable canary run identity is required") - } - sourceNS, targetNS := "longhorn-canary-v1-"+runID, "longhorn-canary-v2-"+runID - sourceClass, targetClass := "lh-canary-v1-"+runID, "lh-canary-v2-"+runID - assets, err := canaryProbeResources(sourceNS, sourceClass, runID, "v1") - if err != nil { - return nil, err - } - pod := assets[4] - pod["metadata"].(object)["annotations"] = object{"backup.velero.io/backup-volumes": "data"} - pod["spec"].(object)["activeDeadlineSeconds"] = 5400 - target, err := canaryProbeResources(targetNS, targetClass, runID, "v2") - if err != nil { - return nil, err - } - assets = append(assets, target[:3]...) - name := "longhorn-v2-canary-" + runID - meta := func() object { - return object{"name": name, "namespace": "velero", "labels": object{"storage.platform.devantler.tech/canary-run": runID}} - } - rules := fmt.Sprintf(`version: v1 -resourceModifierRules: -- conditions: - groupResource: persistentvolumeclaims - resourceNameRegex: "^fio-data$" - namespaces: ["%s"] - patches: - - operation: test - path: /spec/storageClassName - value: "%s" - - operation: replace - path: /spec/storageClassName - value: "%s" - - operation: remove - path: /spec/volumeName -- conditions: - groupResource: pods - resourceNameRegex: "^fio$" - namespaces: ["%s"] - patches: - - operation: test - path: /spec/containers/0/image - value: "%s" - - operation: replace - path: /spec/nodeSelector - value: '{"kubernetes.io/hostname":"prod-worker-3"}' -`, sourceNS, sourceClass, targetClass, sourceNS, fioImage) - assets = append(assets, - object{"apiVersion": "v1", "kind": "ConfigMap", "metadata": meta(), "data": object{"rules.yaml": rules}}, - object{"apiVersion": "velero.io/v1", "kind": "Backup", "metadata": meta(), "spec": object{ - "includedNamespaces": []any{sourceNS}, "includedResources": []any{"pods", "persistentvolumeclaims"}, "includeClusterResources": false, - "storageLocation": "default", "snapshotVolumes": false, "defaultVolumesToFsBackup": false, "ttl": "24h0m0s", "csiSnapshotTimeout": "10m0s", "itemOperationTimeout": "30m0s"}}, - object{"apiVersion": "velero.io/v1", "kind": "Restore", "metadata": meta(), "spec": object{ - "backupName": name, "includedNamespaces": []any{sourceNS}, "includedResources": []any{"pods", "persistentvolumeclaims"}, "includeClusterResources": false, - "namespaceMapping": object{sourceNS: targetNS}, "resourceModifier": object{"kind": "ConfigMap", "name": name}, "itemOperationTimeout": "30m0s"}}, - ) - return assets, nil -} - -// cleanVeleroResult requires native completion without errors or warnings; -// Velero's omitted zero counters remain valid. -func cleanVeleroResult(o object) error { - if text(o, "metadata", "uid") == "" || value(o, "metadata", "deletionTimestamp") != nil || text(o, "status", "phase") != "Completed" { - return errors.New("native backup or restore has not completed") - } - if _, err := time.Parse(time.RFC3339, text(o, "status", "completionTimestamp")); err != nil { - return errors.New("native backup or restore completion is missing") - } - // Velero v1.18 serializes its zero error/warning counters with omitempty. - for _, field := range []string{"errors", "warnings"} { - if count := value(o, "status", field); count != nil && count != float64(0) { - return errors.New("native backup or restore reported warnings or errors") - } - } - return nil -} diff --git a/scripts/migrate-longhorn-v2/canary_assets_test.go b/scripts/migrate-longhorn-v2/canary_assets_test.go deleted file mode 100644 index d249a326e1..0000000000 --- a/scripts/migrate-longhorn-v2/canary_assets_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - - jsonpatch "gopkg.in/evanphx/json-patch.v4" - "sigs.k8s.io/yaml" -) - -func TestCanaryRestoresOnlyItsOwnFilesystemDataOntoV2(t *testing.T) { - assets, err := canaryResources("1234") - if err != nil { - t.Fatal(err) - } - if len(assets) != 11 { - t.Fatalf("incomplete isolated canary assets: %d", len(assets)) - } - var backup, restore, modifier object - classes := map[string]object{} - for _, o := range assets { - switch text(o, "kind") { - case "Backup": - backup = o - case "Restore": - restore = o - case "ConfigMap": - modifier = o - case "StorageClass": - classes[text(o, "parameters", "dataEngine")] = o - case "Pod": - if text(o, "spec", "nodeSelector", "kubernetes.io/hostname") != "prod-worker-3" || value(o, "spec", "automountServiceAccountToken") != false || text(o, "metadata", "annotations", "backup.velero.io/backup-volumes") != "data" { - t.Fatal("probe must exercise the remote V2 client and back up only persistent data") - } - } - } - if len(classes) != 2 || text(classes["v2"], "parameters", "numberOfReplicas") != "3" || text(classes["v2"], "reclaimPolicy") != "Retain" { - t.Fatal("canary storage is not independent retained three-copy storage") - } - if text(classes["v2"], "parameters", "unmapMarkSnapChainRemoved") != "disabled" { - t.Fatal("V2 probe must not inherit unsupported trim behavior") - } - if value(backup, "spec", "snapshotVolumes") != false || value(backup, "spec", "defaultVolumesToFsBackup") != false || value(backup, "spec", "includeClusterResources") != false || value(restore, "spec", "includeClusterResources") != false { - t.Fatal("probe might clone a V1 snapshot or restore cluster resources") - } - if text(restore, "spec", "backupName") != text(backup, "metadata", "name") || text(restore, "spec", "resourceModifier", "name") != text(modifier, "metadata", "name") || text(restore, "spec", "namespaceMapping", "longhorn-canary-v1-1234") != "longhorn-canary-v2-1234" { - t.Fatal("restore escaped the dedicated source and destination") - } - rules := text(modifier, "data", "rules.yaml") - var native struct { - ResourceModifierRules []struct { - Patches []struct{ Operation, Path, Value string } - } - } - if err := yaml.Unmarshal([]byte(rules), &native); err != nil { - t.Fatal("modifier cannot decode Velero's native string-valued JSON patches:", err) - } - // The backup excludes PVs, so the modifier receives a claim still bound to - // its V1 source PV. Apply the emitted RFC 6902 operations to that real shape. - var operations []object - for _, patch := range native.ResourceModifierRules[0].Patches { - op := object{"op": patch.Operation, "path": patch.Path} - if patch.Value != "" { - op["value"] = patch.Value - } - operations = append(operations, op) - } - encoded, err := json.Marshal(operations) - if err != nil { - t.Fatal(err) - } - patch, err := jsonpatch.DecodePatch(encoded) - if err != nil { - t.Fatal(err) - } - restored, err := patch.Apply([]byte(`{"spec":{"storageClassName":"lh-canary-v1-1234","volumeName":"source-v1-pv"}}`)) - if err != nil { - t.Fatal(err) - } - var claim object - if err := json.Unmarshal(restored, &claim); err != nil { - t.Fatal(err) - } - if value(claim, "spec", "volumeName") != nil || text(claim, "spec", "storageClassName") != "lh-canary-v2-1234" { - t.Fatal("restored claim retains its source PV binding instead of provisioning V2 storage") - } - for _, required := range []string{"longhorn-canary-v1-1234", "^fio-data$", "^fio$", "/spec/storageClassName", "lh-canary-v2-1234", "operation: test"} { - if !strings.Contains(rules, required) { - t.Fatal("restore modifier lacks exact source or destination binding") - } - } - if strings.Contains(canaryScript, "curl ") || !strings.Contains(canaryScript, "while [ ! -f /tmp/canary-start ]") || !strings.Contains(canaryScript, "--verify=sha256") { - t.Fatal("probe does not wait for verified provisioning or verify persisted bytes") - } - if _, err = canaryResources("../other"); err == nil { - t.Fatal("unbounded canary identity accepted") - } -} - -func TestCompletedBackupCannotStandInForRestoredData(t *testing.T) { - b := resource("canary") - b["status"] = object{"phase": "Completed", "completionTimestamp": "2026-10-29T19:00:00Z", "errors": float64(0), "warnings": float64(0)} - if cleanVeleroResult(b) != nil { - t.Fatal("clean native completion rejected") - } - for _, field := range []string{"errors", "warnings"} { - b["status"].(object)[field] = float64(1) - if cleanVeleroResult(b) == nil { - t.Fatal("partial or warned backup accepted") - } - b["status"].(object)[field] = float64(0) - } - b["status"].(object)["phase"] = "PartiallyFailed" - if cleanVeleroResult(b) == nil { - t.Fatal("partial native completion accepted") - } - if cleanVeleroResult(object{}) == nil { - t.Fatal("missing evidence accepted") - } - b["status"].(object)["phase"] = "Completed" - delete(b["status"].(object), "warnings") - delete(b["status"].(object), "errors") - if cleanVeleroResult(b) != nil { - t.Fatal("native omitempty zero counters rejected") - } -} diff --git a/scripts/migrate-longhorn-v2/canary_cleanup.go b/scripts/migrate-longhorn-v2/canary_cleanup.go deleted file mode 100644 index c8937eda23..0000000000 --- a/scripts/migrate-longhorn-v2/canary_cleanup.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "context" - "errors" -) - -// requireNativeRestoreGone observes Velero's own restore retirement and never -// deletes a replacement that reused the name. -func (p *canaryRun) requireNativeRestoreGone(ctx context.Context, restore object) error { - if text(restore, "metadata", "uid") == "" { - return errors.New("native restore retirement lacks its original identity") - } - // Velero 1.18.4's backup deletion controller retires its Restores too. - // Verify that native result; an unconditional second delete fails after - // successful cleanup and must never target a replacement with the same name. - current, err := p.c.get(ctx, "restores.velero.io", "velero", text(restore, "metadata", "name"), true) - if err != nil || current == nil { - return err - } - if text(current, "metadata", "uid") != text(restore, "metadata", "uid") { - return errors.New("canary restore was replaced; preserve the replacement") - } - return errors.New("native canary restore retirement has not finished") -} - -// reattach moves only this run's restored consumer to a second client while -// requiring the same claim, PV, backend and original payload bytes. -func (p *canaryRun) reattach(ctx context.Context, created map[string]object, originalPV, originalVolume object) error { - namespace := "longhorn-canary-v2-" + p.run - originalClaim := created["PersistentVolumeClaim"] - if err := removeScratch(ctx, p.c, p.source, "pod", namespace, "/api/v1/namespaces/"+namespace+"/pods/fio", created["Pod"]); err != nil { - return err - } - if err := waitReady(ctx, func(ctx context.Context) error { - volume, err := p.c.get(ctx, "volumes.longhorn.io", "longhorn-system", text(originalVolume, "metadata", "name"), false) - if err != nil || text(volume, "metadata", "uid") != text(originalVolume, "metadata", "uid") || text(volume, "status", "state") != "detached" { - return errors.New("canary volume has not detached before changing clients") - } - return nil - }); err != nil { - return err - } - assets, err := canaryProbeResources(namespace, "lh-canary-v2-"+p.run, p.run, "v2") - if err != nil { - return err - } - desired := assets[4] - desired["spec"].(object)["nodeSelector"] = object{"kubernetes.io/hostname": "prod-worker-1"} - desired["spec"].(object)["activeDeadlineSeconds"] = 1800 - pod, err := p.create(ctx, "reattached/Pod", desired) - if err != nil { - return err - } - bound, pv, volume, err := p.binding(ctx, namespace, "lh-canary-v2-"+p.run, "v2", "prod-worker-1", text(pod, "metadata", "uid")) - if err != nil { - return err - } - if text(pv, "metadata", "uid") != text(originalPV, "metadata", "uid") || text(volume, "metadata", "uid") != text(originalVolume, "metadata", "uid") || text(bound["PersistentVolumeClaim"], "metadata", "uid") != text(originalClaim, "metadata", "uid") { - return errors.New("canary reattachment replaced its original restored storage") - } - if _, err = p.verifyBytes(ctx, bound["Pod"], "prod-worker-1", p.record.PayloadSHA256); err != nil { - return err - } - created["Pod"] = bound["Pod"] - return nil -} - -// removeBackup delegates data retirement to Velero's UID-bound deletion request -// and verifies native absence before removing the owned modifier. -func (p *canaryRun) removeBackup(ctx context.Context, original, restore, modifier object) error { - name := "longhorn-v2-canary-" + p.run - backup, err := p.c.get(ctx, "backups.velero.io", "velero", name, false) - if err != nil || text(backup, "metadata", "uid") != text(original, "metadata", "uid") { - return errors.New("canary backup was replaced; refusing retirement") - } - if err = cleanVeleroResult(backup); err != nil { - return err - } - // Velero's native deletion request removes Kopia snapshots and backup - // objects together. Never delete only the Backup CR and strand its data. - request, err := p.create(ctx, "DeleteBackupRequest", object{"apiVersion": "velero.io/v1", "kind": "DeleteBackupRequest", - "metadata": object{"name": name + "-delete", "namespace": "velero", "labels": object{"velero.io/backup-name": name, "velero.io/backup-uid": text(backup, "metadata", "uid")}}, - "spec": object{"backupName": name}}) - if err != nil { - return err - } - if err = waitReady(ctx, func(ctx context.Context) error { - current, err := p.c.get(ctx, "backups.velero.io", "velero", name, true) - if err != nil { - return err - } - if current != nil && text(current, "metadata", "uid") != text(original, "metadata", "uid") { - return errors.New("backup identity changed during retirement") - } - dbr, err := p.c.get(ctx, "deletebackuprequests.velero.io", "velero", text(request, "metadata", "name"), true) - if err != nil { - return err - } - if dbr != nil { - if text(dbr, "metadata", "uid") != text(request, "metadata", "uid") { - return errors.New("backup deletion request was replaced") - } - if errs := value(dbr, "status", "errors"); errs != nil { - if list, ok := errs.([]any); !ok || len(list) != 0 { - return errors.New("native canary backup deletion reported errors") - } - } - } - // Successful native deletion removes its own request after deleting the - // backup and all volume-data snapshots. Both absences are required. - if current != nil || dbr != nil { - return errors.New("native canary backup retirement has not finished") - } - return nil - }); err != nil { - return err - } - if err = p.requireNativeRestoreGone(ctx, restore); err != nil { - return err - } - cm, err := p.c.get(ctx, "configmap", "velero", name, false) - if err != nil || text(cm, "metadata", "uid") != p.record.Resources["Modifier"].UID || text(cm, "data", "rules.yaml") != text(modifier, "data", "rules.yaml") { - return errors.New("canary restore modifier identity changed") - } - return removeScratch(ctx, p.c, p.source, "configmap", "velero", "/api/v1/namespaces/velero/configmaps/"+name, cm) -} diff --git a/scripts/migrate-longhorn-v2/canary_cleanup_test.go b/scripts/migrate-longhorn-v2/canary_cleanup_test.go deleted file mode 100644 index cd5f183a0c..0000000000 --- a/scripts/migrate-longhorn-v2/canary_cleanup_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" -) - -func TestNativeRestoreRetirementRequiresAbsenceWithoutDeletingAnything(t *testing.T) { - for _, remaining := range []string{"", "uid-restore", "replacement"} { - original := resource("restore") - c := client{context: "admin@prod", run: func(_ context.Context, _ []byte, program string, args ...string) ([]byte, error) { - request := strings.Join(args, " ") - if program != "kubectl" || !strings.Contains(request, " get restores.velero.io restore ") { - t.Fatal("native restore cleanup issued something other than its fixed read") - } - if remaining == "" { - if !strings.Contains(request, "--ignore-not-found") { - return nil, errors.New("native deletion already removed the Restore") - } - return nil, nil - } - current := resource("restore") - current["metadata"].(object)["uid"] = remaining - return json.Marshal(current) - }} - p := canaryRun{c: c} - err := p.requireNativeRestoreGone(t.Context(), original) - if (err == nil) != (remaining == "") { - t.Fatalf("remaining identity %s: %v", remaining, err) - } - } -} diff --git a/scripts/migrate-longhorn-v2/canary_cli_test.go b/scripts/migrate-longhorn-v2/canary_cli_test.go deleted file mode 100644 index 0afa043a53..0000000000 --- a/scripts/migrate-longhorn-v2/canary_cli_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "encoding/json" - "os" - "os/exec" - "strings" - "testing" - - "gopkg.in/yaml.v3" -) - -func TestCSICanaryCanBeRenderedWithoutProductionAuthorization(t *testing.T) { - cmd := exec.CommandContext(t.Context(), "go", "run", ".", "--render-csi-canary", "1234") - cmd.Env = append(os.Environ(), "GITHUB_ACTIONS=false", "LONGHORN_V2_CONFIRM=", "LONGHORN_V2_OPERATION=") - output, err := cmd.Output() - if err != nil { - t.Fatalf("offline CSI rendering failed: %v", err) - } - var rendered struct{ Items []object } - if json.Unmarshal(output, &rendered) != nil || len(rendered.Items) != 12 || text(rendered.Items[11], "kind") != "VolumeSnapshotClass" || value(rendered.Items[9], "spec", "snapshotMoveData") != true { - t.Fatal("offline CSI output is not the native Data Mover canary") - } -} - -func TestWorkflowRequiresTheCSIConfirmationAndRejectsTheFSBConfirmation(t *testing.T) { - data, err := os.ReadFile("../../.github/workflows/prepare-longhorn-v2-node.yaml") - if err != nil { - t.Fatal(err) - } - var workflow struct { - Jobs map[string]struct { - Steps []struct{ Run string } - } - } - if err = yaml.Unmarshal(data, &workflow); err != nil { - t.Fatal(err) - } - guard := workflow.Jobs["prepare"].Steps[0].Run - for _, confirmation := range []string{"verify-longhorn-v2-csi-restore", "verify-longhorn-v2-restore", "prepare-longhorn-v2-prod-worker-1"} { - cmd := exec.CommandContext(t.Context(), "bash", "-c", guard) - cmd.Env = append(os.Environ(), "DISPATCH_REF=refs/heads/main", "GITHUB_RUN_ATTEMPT=1", "LONGHORN_V2_OPERATION=canary-csi-v2", "LONGHORN_V2_CONFIRM="+confirmation) - if err = cmd.Run(); (err == nil) != strings.Contains(confirmation, "-csi-") { - t.Fatalf("CSI workflow confirmation %s: %v", confirmation, err) - } - } -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_assets.go b/scripts/migrate-longhorn-v2/canary_csi_assets.go deleted file mode 100644 index b09d9125cf..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_assets.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -// csiCanaryResources selects native snapshot data movement for the same -// disposable byte-recovery probe and gives it a run-owned snapshot class. -func csiCanaryResources(run string) ([]object, error) { - assets, err := canaryResources(run) - if err != nil { - return nil, err - } - delete(assets[4]["metadata"].(object), "annotations") - class := "lh-csi-canary-" + run - assets[9]["metadata"].(object)["annotations"] = object{"velero.io/csi-volumesnapshot-class_driver.longhorn.io": class} - spec := assets[9]["spec"].(object) - spec["snapshotVolumes"], spec["snapshotMoveData"], spec["dataMover"] = true, true, "velero" - assets[10]["spec"].(object)["restorePVs"] = true - return append(assets, object{ - "apiVersion": "snapshot.storage.k8s.io/v1", "kind": "VolumeSnapshotClass", - "metadata": object{"name": class, "labels": object{"storage.platform.devantler.tech/canary-run": run}}, - "driver": "driver.longhorn.io", "deletionPolicy": "Delete", "parameters": object{"type": "snap"}, - }), nil -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_assets_test.go b/scripts/migrate-longhorn-v2/canary_csi_assets_test.go deleted file mode 100644 index 44b32ae926..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_assets_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import "testing" - -func TestCSICanaryUsesItsOwnDisposableSnapshotAndNativeDataMover(t *testing.T) { - assets, err := csiCanaryResources("1234") - if err != nil || len(assets) != 12 { - t.Fatalf("CSI fixture invalid: %v", err) - } - pod, backup, restore, class := assets[4], assets[9], assets[10], assets[11] - if value(pod, "metadata", "annotations", "backup.velero.io/backup-volumes") != nil || - value(backup, "spec", "snapshotVolumes") != true || value(backup, "spec", "snapshotMoveData") != true || text(backup, "spec", "dataMover") != "velero" || - value(backup, "spec", "defaultVolumesToFsBackup") != false || value(restore, "spec", "restorePVs") != true { - t.Fatal("CSI data movement was replaced by FSB or a metadata-only restore") - } - if text(class, "kind") != "VolumeSnapshotClass" || text(class, "metadata", "name") != "lh-csi-canary-1234" || text(class, "driver") != "driver.longhorn.io" || text(class, "deletionPolicy") != "Delete" || text(class, "parameters", "type") != "snap" || - text(backup, "metadata", "annotations", "velero.io/csi-volumesnapshot-class_driver.longhorn.io") != text(class, "metadata", "name") { - t.Fatal("backup could select an original historical snapshot or a default snapshot class") - } - if value(class, "metadata", "labels", "velero.io/csi-volumesnapshot-class") != nil || value(class, "metadata", "annotations", "snapshot.storage.kubernetes.io/is-default-class") != nil { - t.Fatal("disposable canary changed global snapshot-class selection") - } - for _, i := range []int{2, 7} { - if text(assets[i], "reclaimPolicy") != "Retain" { - t.Fatal("source or restored data class lost Retain") - } - } - fsb, err := canaryResources("1234") - if err != nil || value(fsb[9], "spec", "snapshotVolumes") != false || value(fsb[9], "spec", "snapshotMoveData") != nil { - t.Fatal("FSB transport was changed by CSI mode") - } -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_binding.go b/scripts/migrate-longhorn-v2/canary_csi_binding.go deleted file mode 100644 index a54e833931..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_binding.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "errors" - "strings" -) - -type csiScratchEvidence struct { - Claim, PV, Volume benchmarkResource - OperationUID, NativeSource string -} - -// Accepts observed watch objects, including deletion events. These identities -// are proof for later absence checks, never authority to delete by name. -func verifyCSIScratchBinding(claim, pv, volume, operation, sourceVolume object, class string) (csiScratchEvidence, error) { - proof := csiScratchEvidence{Claim: benchmarkIdentity(claim), PV: benchmarkIdentity(pv), Volume: benchmarkIdentity(volume), OperationUID: text(operation, "metadata", "uid")} - kind := text(operation, "kind") - if proof.OperationUID == "" || (kind != "DataUpload" && kind != "DataDownload") || text(operation, "metadata", "namespace") != "velero" || proof.Claim.Namespace != "velero" || - proof.Claim.UID == "" || proof.PV.UID == "" || proof.Volume.UID == "" || proof.PV.Namespace != "" || proof.Volume.Namespace != "longhorn-system" || text(sourceVolume, "metadata", "uid") == "" { - return proof, errors.New("CSI scratch resource identity or operation scope is incomplete") - } - owned := false - refs, _ := value(claim, "metadata", "ownerReferences").([]any) - for _, raw := range refs { - r, _ := raw.(object) - if text(r, "apiVersion") == "velero.io/v2alpha1" && text(r, "kind") == kind && text(r, "name") == text(operation, "metadata", "name") && text(r, "uid") == proof.OperationUID && value(r, "controller") == true { - owned = true - } - } - policy := text(pv, "spec", "persistentVolumeReclaimPolicy") - if !owned || class == "" || text(claim, "spec", "storageClassName") != class || text(pv, "spec", "storageClassName") != class || (policy != "Retain" && policy != "Delete") || - text(claim, "spec", "volumeName") != proof.PV.Name || text(pv, "spec", "claimRef", "name") != proof.Claim.Name || text(pv, "spec", "claimRef", "namespace") != "velero" || text(pv, "spec", "claimRef", "uid") != proof.Claim.UID || - text(pv, "spec", "csi", "driver") != "driver.longhorn.io" || text(pv, "spec", "csi", "volumeHandle") != proof.Volume.Name || proof.Volume.UID == text(sourceVolume, "metadata", "uid") { - return proof, errors.New("CSI scratch PVC, PV and independent Longhorn backend are not bound to this operation") - } - engine := "v1" - if kind == "DataDownload" { - engine = "v2" - } - if text(volume, "spec", "dataEngine") != engine { - return proof, errors.New("CSI scratch backend uses the wrong data engine") - } - proof.NativeSource = text(volume, "spec", "dataSource") - if kind == "DataUpload" { - snapshot, ok := strings.CutPrefix(proof.NativeSource, "snap://"+text(sourceVolume, "metadata", "name")+"/") - if !ok || !kubeName.MatchString(snapshot) { - return proof, errors.New("temporary CSI backup clone is not a snapshot of the exact canary source") - } - } else if proof.NativeSource != "" { - return proof, errors.New("CSI recovery target was pre-populated from another data source") - } - return proof, nil -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_binding_test.go b/scripts/migrate-longhorn-v2/canary_csi_binding_test.go deleted file mode 100644 index a84a2a9e2a..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_binding_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import "testing" - -func csiScratchFixture(kind string) (object, object, object, object, object, string) { - _, _, _, op := csiTransferFixture("Backup") - op["kind"] = kind - claim, pv, backend, source := resource("transfer"), resource("temporary-pv"), resource("temporary-backend"), resource("source-backend") - claim["metadata"].(object)["namespace"] = "velero" - claim["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "velero.io/v2alpha1", "kind": kind, "name": text(op, "metadata", "name"), "uid": text(op, "metadata", "uid"), "controller": true}} - class, engine := "lh-canary-v1-1234", "v1" - if kind == "DataDownload" { - class, engine = "lh-canary-v2-1234", "v2" - } - claim["spec"] = object{"volumeName": text(pv, "metadata", "name"), "storageClassName": class} - pv["spec"] = object{"claimRef": object{"name": text(claim, "metadata", "name"), "namespace": "velero", "uid": text(claim, "metadata", "uid")}, "storageClassName": class, "persistentVolumeReclaimPolicy": "Retain", "csi": object{"driver": "driver.longhorn.io", "volumeHandle": text(backend, "metadata", "name")}} - backend["metadata"].(object)["namespace"] = "longhorn-system" - backend["spec"] = object{"dataEngine": engine} - if kind == "DataUpload" { - backend["spec"].(object)["dataSource"] = "snap://source-backend/native-snapshot" - } - return claim, pv, backend, op, source, class -} - -func TestCSIScratchProofRequiresTheObservedOwnedClaimAndExactBackend(t *testing.T) { - for _, kind := range []string{"DataUpload", "DataDownload"} { - claim, pv, volume, op, source, class := csiScratchFixture(kind) - proof, err := verifyCSIScratchBinding(claim, pv, volume, op, source, class) - if err != nil || proof.Claim.UID == "" || proof.PV.UID == "" || proof.Volume.UID == "" { - t.Fatalf("valid owned scratch rejected: %v", err) - } - for name, change := range map[string]func(object, object, object){ - "claim belongs to another operation": func(c, _, _ object) { value(c, "metadata", "ownerReferences").([]any)[0].(object)["uid"] = "foreign" }, - "PV rebound to another claim": func(_, p, _ object) { value(p, "spec", "claimRef").(object)["uid"] = "replacement" }, - "backend replaced": func(_, _, v object) { v["metadata"].(object)["name"] = "other-backend" }, - "another storage driver": func(_, p, _ object) { value(p, "spec", "csi").(object)["driver"] = "another-driver" }, - "wrong data engine": func(_, _, v object) { v["spec"].(object)["dataEngine"] = "other" }, - } { - t.Run(kind+"/"+name, func(t *testing.T) { - c, p, v, o, s, sc := csiScratchFixture(kind) - change(c, p, v) - if _, err := verifyCSIScratchBinding(c, p, v, o, s, sc); err == nil { - t.Fatal("unowned temporary storage accepted") - } - }) - } - } - c, p, v, o, s, sc := csiScratchFixture("DataUpload") - v["spec"].(object)["dataSource"] = "snap://original-historical-volume/do-not-delete" - if _, err := verifyCSIScratchBinding(c, p, v, o, s, sc); err == nil { - t.Fatal("a snapshot of another source volume was accepted") - } -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_capture.go b/scripts/migrate-longhorn-v2/canary_csi_capture.go deleted file mode 100644 index 01a0c408b6..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_capture.go +++ /dev/null @@ -1,220 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "io" - "net/url" - "os/exec" - "sync" -) - -// A completed DataUpload can outlive its temporary PVC, PV and backend. Start -// gap-free API watches before creating the backup, and retain their observed -// identities. This evidence never authorizes deletion of an observed name. -type csiCapture struct { - mu sync.Mutex - run string - objects map[string]map[string]object - err error - cancel context.CancelFunc - wg sync.WaitGroup -} - -var csiCollections = map[string]string{ - "persistentvolumeclaims": "/api/v1/namespaces/velero/persistentvolumeclaims", - "persistentvolumes": "/api/v1/persistentvolumes", - "volumes.longhorn.io": "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes", -} - -// csiInventory anchors the following fixed-scope watch to a complete native -// collection snapshot rather than kubectl's synthetic List printer. -func csiInventory(ctx context.Context, c client, query string) ([]object, string, error) { - path, ok := csiCollections[query] - if !ok { - return nil, "", errors.New("CSI inventory is outside its fixed scope") - } - // kubectl's ordinary JSON printer synthesizes kind: List with an empty - // resourceVersion. Read the native collection so its snapshot can anchor - // the following watch without missing transient provisioning resources. - data, err := c.kubectl(ctx, nil, "get", "--raw="+path+"?limit=500") - if err != nil { - return nil, "", err - } - var list object - if json.Unmarshal(data, &list) != nil || text(list, "metadata", "resourceVersion") == "" || len(text(list, "metadata", "resourceVersion")) > 256 { - return nil, "", errors.New("CSI watch cannot start without a complete native list resource version") - } - items, err := decodeList(data) - return items, text(list, "metadata", "resourceVersion"), err -} - -// newCSICapture creates a run-scoped inventory for the three storage collections. -func newCSICapture(run string) *csiCapture { - return &csiCapture{run: run, objects: map[string]map[string]object{ - "persistentvolumeclaims": {}, "persistentvolumes": {}, "volumes.longhorn.io": {}, - }} -} - -// observe retains bounded original identities, including short-lived bound -// scratch objects that native transfer cleanup may remove before the next poll. -func (p *csiCapture) observe(query string, o object) error { - p.mu.Lock() - defer p.mu.Unlock() - if p.err != nil { - return p.err - } - uid := text(o, "metadata", "uid") - if uid == "" || text(o, "metadata", "resourceVersion") == "" { - return errors.New("CSI watch object has no complete identity") - } - class := text(o, "spec", "storageClassName") - if query != "volumes.longhorn.io" { - if class != "lh-canary-v1-"+p.run && class != "lh-canary-v2-"+p.run { - return nil - } - if query == "persistentvolumeclaims" && (text(o, "metadata", "namespace") != "velero" || text(o, "spec", "volumeName") == "") { - return nil - } - if query == "persistentvolumes" && text(o, "spec", "claimRef", "namespace") != "velero" { - return nil - } - } - items, ok := p.objects[query] - if !ok { - return errors.New("CSI capture request is outside its fixed scope") - } - if len(items) >= 512 { - return errors.New("CSI capture exceeded its bounded identity inventory") - } - // Keep the original bound graph, including a clone's original dataSource, - // before native cleanup or download rebinding changes those fields. - if _, exists := items[uid]; !exists { - items[uid] = o - } - return nil -} - -// proof joins exactly one owned CSI scratch graph and rejects PV name reuse; -// these observations are evidence and never deletion authority. -func (p *csiCapture) proof(operation, source object, class string) (csiScratchEvidence, error) { - p.mu.Lock() - defer p.mu.Unlock() - if p.err != nil { - return csiScratchEvidence{}, p.err - } - var proofs []csiScratchEvidence - for _, claim := range p.objects["persistentvolumeclaims"] { - for _, pv := range p.objects["persistentvolumes"] { - if text(pv, "metadata", "name") != text(claim, "spec", "volumeName") { - continue - } - for _, volume := range p.objects["volumes.longhorn.io"] { - if text(volume, "metadata", "name") != text(pv, "spec", "csi", "volumeHandle") { - continue - } - if proof, err := verifyCSIScratchBinding(claim, pv, volume, operation, source, class); err == nil { - proofs = append(proofs, proof) - } - } - } - } - if len(proofs) != 1 { - return csiScratchEvidence{}, errors.New("the CSI operation's independent storage graph was not observed exactly once") - } - proof := proofs[0] - for _, pv := range p.objects["persistentvolumes"] { - if text(pv, "metadata", "name") == proof.PV.Name && text(pv, "metadata", "uid") != proof.PV.UID { - return proof, errors.New("the CSI temporary PV name was reused") - } - } - return proof, nil -} - -// readCSIWatch rejects stream gaps, incomplete objects and unsupported events -// instead of accepting a partial identity history. -func readCSIWatch(input io.Reader, observe func(object) error) error { - decoder := json.NewDecoder(input) - for { - var event struct { - Type string `json:"type"` - Object object `json:"object"` - } - if decoder.Decode(&event) != nil { - return errors.New("CSI watch ended before the recovery proof completed") - } - if event.Type == "BOOKMARK" { - continue - } - if event.Type != "ADDED" && event.Type != "MODIFIED" && event.Type != "DELETED" { - return errors.New("CSI watch reported an error or unsupported event") - } - if text(event.Object, "metadata", "uid") == "" || text(event.Object, "metadata", "resourceVersion") == "" { - return errors.New("CSI watch event identity is incomplete") - } - if err := observe(event.Object); err != nil { - return err - } - } -} - -// start lists then streams each fixed collection from its resource version; -// the parent deadline bounds native watches and any failed stream poisons proof. -func (p *csiCapture) start(ctx context.Context, c client) error { - watchCtx, cancel := context.WithCancel(ctx) - p.cancel = cancel - for _, query := range []string{"persistentvolumeclaims", "persistentvolumes", "volumes.longhorn.io"} { - items, rv, err := csiInventory(watchCtx, c, query) - if err != nil { - p.stop() - return err - } - for _, item := range items { - if err = p.observe(query, item); err != nil { - p.stop() - return err - } - } - args := []string{"--context=" + c.context, "--request-timeout=0", "get", "--raw=" + csiCollections[query] + "?watch=true&allowWatchBookmarks=true&resourceVersion=" + url.QueryEscape(rv)} - cmd := exec.CommandContext(watchCtx, "kubectl", args...) - cmd.Stderr = io.Discard - stdout, err := cmd.StdoutPipe() - if err != nil { - p.stop() - return err - } - if err = cmd.Start(); err != nil { - p.stop() - return errors.New("CSI identity watch could not start") - } - p.wg.Go(func() { - err := readCSIWatch(stdout, func(o object) error { return p.observe(query, o) }) - _ = cmd.Process.Kill() - _ = cmd.Wait() - if watchCtx.Err() == nil { - p.mu.Lock() - if p.err == nil { - p.err = err - } - p.mu.Unlock() - } - }) - } - return nil -} - -// stop cancels and joins only the native watch processes owned by this capture. -func (p *csiCapture) stop() { - if p.cancel != nil { - p.cancel() - } - p.wg.Wait() -} - -// health returns any observed stream failure after synchronized shutdown. -func (p *csiCapture) health() error { - p.mu.Lock() - defer p.mu.Unlock() - return p.err -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_capture_test.go b/scripts/migrate-longhorn-v2/canary_csi_capture_test.go deleted file mode 100644 index cd7af2a9de..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_capture_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" -) - -func TestCSIInventoryPreservesNativeListResourceVersion(t *testing.T) { - for _, continuation := range []string{"", "unread-page"} { - calls := 0 - c := client{context: "admin@prod", run: func(_ context.Context, _ []byte, program string, args ...string) ([]byte, error) { - calls++ - if program != "kubectl" || strings.Join(args, " ") != "--context=admin@prod --request-timeout=20s get --raw=/api/v1/persistentvolumes?limit=500" { - t.Fatal("watch inventory did not read the fixed native collection") - } - return []byte(`{"apiVersion":"v1","kind":"PersistentVolumeList","metadata":{"resourceVersion":"12345","continue":"` + continuation + `"},"items":[]}`), nil - }} - items, rv, err := csiInventory(t.Context(), c, "persistentvolumes") - if continuation == "" && (err != nil || rv != "12345" || len(items) != 0 || calls != 1) { - t.Fatalf("lost native list version: rv=%s err=%v", rv, err) - } - if continuation != "" && err == nil { - t.Fatal("a partial list was accepted as a gap-free watch origin") - } - } -} - -func TestCSICaptureKeepsTheOwnedBindingAfterNativeCleanup(t *testing.T) { - for _, kind := range []string{"DataUpload", "DataDownload"} { - claim, pv, volume, op, source, class := csiScratchFixture(kind) - capture := newCSICapture("1234") - // The backend may arrive before its PV and claim. Native cleanup may - // remove all three before a backup's completion becomes observable. - for _, event := range []struct { - query string - value object - }{ - {"volumes.longhorn.io", volume}, {"persistentvolumes", pv}, {"persistentvolumeclaims", claim}, - } { - if err := capture.observe(event.query, event.value); err != nil { - t.Fatal(err) - } - } - proof, err := capture.proof(op, source, class) - if err != nil || proof.Volume.UID != text(volume, "metadata", "uid") { - t.Fatalf("owned transient binding lost: %v", err) - } - encoded, _ := json.Marshal(pv) - var foreign object - if err := json.Unmarshal(encoded, &foreign); err != nil { - t.Fatal(err) - } - foreign["metadata"].(object)["uid"] = "replacement-pv" - foreign["spec"].(object)["csi"].(object)["volumeHandle"] = "unrelated-volume" - if err := capture.observe("persistentvolumes", foreign); err != nil { - t.Fatal(err) - } - if _, err := capture.proof(op, source, class); err == nil { - t.Fatal("a replaced or ambiguous PV name was accepted") - } - } -} - -func TestCSIWatchRefusesGapsAndMalformedEvents(t *testing.T) { - for _, input := range []string{ - `{"type":"ERROR","object":{"code":410}}`, - `{"type":"ADDED","object":{"metadata":{"name":"claim"}}}`, - `{"type":"MODIFIED","object":`, - `{"type":"UNRECOGNIZED","object":{}}`, - "", - } { - if err := readCSIWatch(strings.NewReader(input), func(object) error { return nil }); err == nil { - t.Fatalf("incomplete watch accepted: %q", input) - } - } -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_native_test.go b/scripts/migrate-longhorn-v2/canary_csi_native_test.go deleted file mode 100644 index 7d6b01fe42..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_native_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "net/http/httptest" - "os" - "path/filepath" - "testing" - "time" -) - -// Exercise the actual kubectl raw collection/watch transport, including its -// streaming output. The API fixture has no credentials and is not a cluster. -func TestCSIIdentityCaptureThroughNativeKubectl(t *testing.T) { - if os.Getenv("LONGHORN_V2_NATIVE_TEST") != "1" { - t.Skip("set LONGHORN_V2_NATIVE_TEST=1 with the repository-approved kubectl") - } - claim, pv, volume, op, source, class := csiScratchFixture("DataUpload") - objects := map[string]object{ - "/api/v1/namespaces/velero/persistentvolumeclaims": claim, - "/api/v1/persistentvolumes": pv, - "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes": volume, - } - watching := make(chan struct{}, 3) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - o, ok := objects[r.URL.Path] - if !ok || r.Method != http.MethodGet { - http.Error(w, "unexpected fixture request", http.StatusBadRequest) - return - } - w.Header().Set("Content-Type", "application/json") - if r.URL.Query().Get("watch") != "true" { - if r.URL.Query().Get("limit") != "500" { - http.Error(w, "missing collection bound", http.StatusBadRequest) - return - } - _ = json.NewEncoder(w).Encode(object{"apiVersion": "v1", "kind": "List", "metadata": object{"resourceVersion": "12345"}, "items": []any{}}) - return - } - if r.URL.Query().Get("resourceVersion") != "12345" || r.URL.Query().Get("allowWatchBookmarks") != "true" { - http.Error(w, "watch lost its collection snapshot", http.StatusBadRequest) - return - } - _ = json.NewEncoder(w).Encode(object{"type": "ADDED", "object": o}) - w.(http.Flusher).Flush() - watching <- struct{}{} - <-r.Context().Done() - })) - defer server.Close() - config := fmt.Sprintf("apiVersion: v1\nkind: Config\nclusters:\n- name: fixture\n cluster:\n server: %s\ncontexts:\n- name: admin@prod\n context:\n cluster: fixture\ncurrent-context: admin@prod\n", server.URL) - path := filepath.Join(t.TempDir(), "kubeconfig") - if err := os.WriteFile(path, []byte(config), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv("KUBECONFIG", path) - ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) - defer cancel() - capture := newCSICapture("1234") - if err := capture.start(ctx, client{run: execute, context: "admin@prod"}); err != nil { - t.Fatal(err) - } - defer capture.stop() - for range 3 { - select { - case <-watching: - case <-ctx.Done(): - t.Fatal("native watches did not stream all three storage identities") - } - } - deadline := time.NewTimer(5 * time.Second) - defer deadline.Stop() - tick := time.NewTicker(10 * time.Millisecond) - defer tick.Stop() - for { - if proof, err := capture.proof(op, source, class); err == nil && proof.Volume.UID == text(volume, "metadata", "uid") { - return - } - select { - case <-tick.C: - case <-deadline.C: - t.Fatal("native watch events did not produce the bound CSI storage proof") - } - } -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_runtime.go b/scripts/migrate-longhorn-v2/canary_csi_runtime.go deleted file mode 100644 index 0fe30fa14e..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_runtime.go +++ /dev/null @@ -1,183 +0,0 @@ -package main - -import ( - "context" - "errors" - "strings" -) - -// canaryJournalName separates filesystem and CSI proof ownership at fixed names. -func canaryJournalName(transport string) (string, error) { - switch transport { - case "filesystem": - return "longhorn-v2-canary", nil - case "csi": - return "longhorn-v2-csi-canary", nil - default: - return "", errors.New("canary transport is outside the reviewed procedure") - } -} - -// verifyCSIRestoredBinding requires the destination claim to adopt the exact -// independent backend downloaded by the native data mover. -func verifyCSIRestoredBinding(proof csiScratchEvidence, claim, pv, volume object) error { - if proof.OperationUID == "" || proof.Claim.UID == "" || proof.PV.UID == "" || proof.Volume.UID == "" || - text(claim, "metadata", "namespace") == "velero" || text(claim, "metadata", "uid") == "" || text(claim, "metadata", "uid") == proof.Claim.UID || - text(pv, "metadata", "name") != proof.PV.Name || text(pv, "metadata", "uid") != proof.PV.UID || - text(volume, "metadata", "name") != proof.Volume.Name || text(volume, "metadata", "uid") != proof.Volume.UID { - return errors.New("the restored claim did not adopt the exact downloaded PV and independent backend") - } - return nil -} - -// transferCSI joins the completed native operation to its snapshot class, -// source claim, captured scratch storage and parent backup or restore. -func (p *canaryRun) transferCSI(ctx context.Context, desired object, expectedSnapshot string) (object, string, error) { - kind := text(desired, "kind") - query := "datauploads.velero.io" - if kind == "Restore" { - query = "datadownloads.velero.io" - } - created, err := p.create(ctx, kind, desired) - if err != nil { - return nil, "", err - } - var parent object - var snapshot string - err = waitReady(ctx, func(ctx context.Context) error { - if err = p.csi.health(); err != nil { - return err - } - parent, err = p.c.get(ctx, strings.ToLower(kind)+"s.velero.io", "velero", text(created, "metadata", "name"), false) - if err != nil || text(parent, "metadata", "uid") != text(created, "metadata", "uid") { - return errors.New("CSI backup or restore identity changed") - } - if err = cleanVeleroResult(parent); err != nil { - return err - } - class, readErr := p.c.get(ctx, "volumesnapshotclasses.snapshot.storage.k8s.io", "", text(p.snapshotClass, "metadata", "name"), false) - if readErr != nil || text(class, "metadata", "uid") != text(p.snapshotClass, "metadata", "uid") || text(class, "driver") != "driver.longhorn.io" || text(class, "deletionPolicy") != "Delete" || text(class, "parameters", "type") != "snap" { - return errors.New("the disposable CSI snapshot class changed") - } - sourceClaim, readErr := p.c.get(ctx, "persistentvolumeclaim", text(p.sourceClaim, "metadata", "namespace"), text(p.sourceClaim, "metadata", "name"), false) - if readErr != nil || text(sourceClaim, "metadata", "uid") != text(p.sourceClaim, "metadata", "uid") { - return errors.New("the CSI source claim was replaced") - } - var targetClaim object - if kind == "Restore" { - targetClaim, readErr = p.c.get(ctx, "persistentvolumeclaim", "longhorn-canary-v2-"+p.run, "fio-data", false) - if readErr != nil { - return readErr - } - } - operations, readErr := p.c.list(ctx, query, "velero") - if readErr != nil { - return readErr - } - op, snap, verifyErr := verifyCSITransfer(operations, parent, sourceClaim, targetClaim, expectedSnapshot, text(sourceClaim, "spec", "storageClassName"), text(class, "metadata", "name")) - if verifyErr != nil { - return verifyErr - } - storageClass := text(sourceClaim, "spec", "storageClassName") - if kind == "Restore" { - storageClass = text(targetClaim, "spec", "storageClassName") - } - proof, verifyErr := p.csi.proof(op, p.sourceVolume, storageClass) - if verifyErr != nil { - return verifyErr - } - opKind := text(op, "kind") - if old := p.record.CSIScratch[opKind]; old.OperationUID != "" && old != proof { - return errors.New("the observed CSI operation storage graph changed") - } - p.record.CSIScratch[opKind] = proof - if verifyErr = p.remember(ctx, "csi/"+opKind, op); verifyErr != nil { - return verifyErr - } - snapshot = snap - return nil - }) - return parent, snapshot, err -} - -// requireCSIGone observes native scratch retirement without adopting or deleting -// a replacement that reused a recorded resource name. -func (p *canaryRun) requireCSIGone(ctx context.Context, kind string, identity benchmarkResource) error { - if identity.Name == "" || identity.UID == "" { - return errors.New("CSI cleanup lacks a captured resource identity") - } - current, err := p.c.get(ctx, kind, identity.Namespace, identity.Name, true) - if err != nil { - return err - } - if current != nil { - if text(current, "metadata", "uid") != identity.UID { - return errors.New("a CSI scratch name was reused; do not adopt or delete its replacement") - } - return errors.New("native CSI scratch cleanup has not completed") - } - return nil -} - -// waitCSIUploadGone requires native retirement of all captured upload storage -// before beginning download, while keeping the identity streams healthy. -func (p *canaryRun) waitCSIUploadGone(ctx context.Context) error { - proof := p.record.CSIScratch["DataUpload"] - return waitReady(ctx, func(ctx context.Context) error { - if err := p.csi.health(); err != nil { - return err - } - for _, item := range []struct { - kind string - identity benchmarkResource - }{ - {"persistentvolumeclaim", proof.Claim}, {"persistentvolume", proof.PV}, {"volumes.longhorn.io", proof.Volume}, - } { - if err := p.requireCSIGone(ctx, item.kind, item.identity); err != nil { - return err - } - } - return nil - }) -} - -// waitCSIArtifactsGone verifies native operation and snapshot cleanup while -// retaining the downloaded backend for its separate owned cleanup. -func (p *canaryRun) waitCSIArtifactsGone(ctx context.Context) error { - return waitReady(ctx, func(ctx context.Context) error { - if err := p.csi.health(); err != nil { - return err - } - for _, kind := range []string{"DataUpload", "DataDownload"} { - if err := p.requireCSIGone(ctx, strings.ToLower(kind)+"s.velero.io", p.record.Resources["csi/"+kind]); err != nil { - return err - } - } - if err := p.requireCSIGone(ctx, "persistentvolumeclaim", p.record.CSIScratch["DataDownload"].Claim); err != nil { - return err - } - for _, scope := range []struct{ kind, namespace string }{ - {"volumesnapshots.snapshot.storage.k8s.io", "*"}, {"volumesnapshotcontents.snapshot.storage.k8s.io", ""}, - } { - items, err := p.c.list(ctx, scope.kind, scope.namespace) - if err != nil { - return err - } - for _, item := range items { - if text(item, "spec", "volumeSnapshotClassName") == text(p.snapshotClass, "metadata", "name") { - return errors.New("native CSI snapshot cleanup has not completed") - } - } - } - return nil - }) -} - -// removeCSISnapshotClass conditionally removes only this run's original class. -func (p *canaryRun) removeCSISnapshotClass(ctx context.Context) error { - current, err := p.c.get(ctx, "volumesnapshotclasses.snapshot.storage.k8s.io", "", text(p.snapshotClass, "metadata", "name"), false) - if err != nil || text(current, "metadata", "uid") != text(p.snapshotClass, "metadata", "uid") { - return errors.New("the CSI snapshot class was replaced; preserve it for diagnosis") - } - return removeScratch(ctx, p.c, p.source, "volumesnapshotclasses.snapshot.storage.k8s.io", "", "/apis/snapshot.storage.k8s.io/v1/volumesnapshotclasses/"+text(current, "metadata", "name"), current) -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_runtime_test.go b/scripts/migrate-longhorn-v2/canary_csi_runtime_test.go deleted file mode 100644 index 47e2714ed1..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_runtime_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import "testing" - -func TestCSIAndFilesystemCanariesHaveSeparateConfirmationsAndJournals(t *testing.T) { - env := dispatchFixture() - env["LONGHORN_V2_OPERATION"] = "canary-csi-v2" - env["LONGHORN_V2_CONFIRM"] = "verify-longhorn-v2-restore" - getenv := func(k string) string { return env[k] } - if authorizeCanary(getenv) == nil { - t.Fatal("filesystem confirmation authorized CSI data movement") - } - env["LONGHORN_V2_CONFIRM"] = "verify-longhorn-v2-csi-restore" - if err := authorizeCanary(getenv); err != nil { - t.Fatal(err) - } - if authorize(getenv, "prod-worker-1") == nil || authorizeTemporaryPreparation(getenv, "temporary-v1-1") == nil || authorizeHostReboot(getenv, "prod-worker-1") == nil { - t.Fatal("CSI canary authorized a reset or host reboot") - } - fsb, err := canaryJournalName("filesystem") - if err != nil { - t.Fatal(err) - } - csi, err := canaryJournalName("csi") - if err != nil || csi == fsb { - t.Fatal("one canary could overwrite the other transport's evidence") - } - if _, err = canaryJournalName("arbitrary-journal"); err == nil { - t.Fatal("transport selected an undeclared journal") - } -} - -func TestCSIRestoreKeepsTheDownloadedPVAndIndependentBackend(t *testing.T) { - claim, pv, volume, op, source, class := csiScratchFixture("DataDownload") - proof, err := verifyCSIScratchBinding(claim, pv, volume, op, source, class) - if err != nil { - t.Fatal(err) - } - claim["metadata"].(object)["uid"] = "restored-claim-uid" - claim["metadata"].(object)["namespace"] = "longhorn-canary-v2-1234" - if err = verifyCSIRestoredBinding(proof, claim, pv, volume); err != nil { - t.Fatal(err) - } - volume["metadata"].(object)["uid"] = "replacement-backend" - if verifyCSIRestoredBinding(proof, claim, pv, volume) == nil { - t.Fatal("a different backend substituted for the downloaded data") - } -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_transfer.go b/scripts/migrate-longhorn-v2/canary_csi_transfer.go deleted file mode 100644 index 3517a09701..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_transfer.go +++ /dev/null @@ -1,65 +0,0 @@ -package main - -import "errors" - -// CSI DataUpload/DataDownload have different schemas from PVB/PVR. Bind the -// actual native fields instead of inferring a CSI recovery from FSB evidence. -func verifyCSITransfer(transfers []object, parent, sourceClaim, targetClaim object, expectedSnapshot, sourceClass, snapshotClass string) (object, string, error) { - kind, uid := text(parent, "kind"), text(parent, "metadata", "uid") - if uid == "" || text(parent, "metadata", "namespace") != "velero" || text(sourceClaim, "metadata", "uid") == "" || (kind != "Backup" && kind != "Restore") { - return nil, "", errors.New("CSI transfer parent or source claim identity is missing") - } - label, expectedKind := "velero.io/backup-uid", "DataUpload" - if kind == "Restore" { - label, expectedKind = "velero.io/restore-uid", "DataDownload" - if text(targetClaim, "metadata", "uid") == "" || text(targetClaim, "metadata", "uid") == text(sourceClaim, "metadata", "uid") { - return nil, "", errors.New("CSI restore requires an independent target claim") - } - } - var found object - snapshot := "" - for _, op := range transfers { - if !controlledBy(op, parent) && text(op, "metadata", "labels", label) != uid { - continue - } - if found != nil || !controlledBy(op, parent) || text(op, "metadata", "namespace") != "velero" || text(op, "kind") != expectedKind || text(op, "metadata", "labels", label) != uid || - text(op, "metadata", "uid") == "" || text(op, "spec", "backupStorageLocation") != "default" || text(op, "spec", "datamover") != "velero" || text(op, "spec", "sourceNamespace") != text(sourceClaim, "metadata", "namespace") { - return nil, "", errors.New("CSI transfer ownership, repository or source scope is ambiguous") - } - if err := cleanVeleroResult(op); err != nil { - return nil, "", err - } - done, doneOK := value(op, "status", "progress", "bytesDone").(float64) - total, totalOK := value(op, "status", "progress", "totalBytes").(float64) - if !doneOK || !totalOK || done < 134217728 || done != total { - return nil, "", errors.New("the CSI operation has not transferred the complete canary payload") - } - if kind == "Backup" { - if text(op, "metadata", "labels", "velero.io/pvc-uid") != text(sourceClaim, "metadata", "uid") || text(op, "spec", "snapshotType") != "CSI" || - text(op, "spec", "sourcePVC") != text(sourceClaim, "metadata", "name") || text(op, "spec", "csiSnapshot", "driver") != "driver.longhorn.io" || - text(op, "spec", "csiSnapshot", "storageClass") != sourceClass || text(op, "spec", "csiSnapshot", "snapshotClass") != snapshotClass || - !kubeName.MatchString(text(op, "spec", "csiSnapshot", "volumeSnapshot")) { - return nil, "", errors.New("CSI upload does not bind the exact source claim and disposable snapshot class") - } - snapshot = text(op, "status", "snapshotID") - } else { - if text(op, "spec", "targetVolume", "namespace") != text(targetClaim, "metadata", "namespace") || text(op, "spec", "targetVolume", "pvc") != text(targetClaim, "metadata", "name") || - text(op, "spec", "snapshotID") != expectedSnapshot || expectedSnapshot == "" { - return nil, "", errors.New("CSI download does not bind the exact repository snapshot and target claim") - } - size, ok := value(op, "spec", "snapshotSize").(float64) - if !ok || size < 134217728 { - return nil, "", errors.New("CSI download source size is incomplete") - } - snapshot = text(op, "spec", "snapshotID") - } - if snapshot == "" { - return nil, "", errors.New("CSI repository snapshot identity is missing") - } - found = op - } - if found == nil { - return nil, "", errors.New("exactly one completed CSI volume-data operation is required") - } - return found, snapshot, nil -} diff --git a/scripts/migrate-longhorn-v2/canary_csi_transfer_test.go b/scripts/migrate-longhorn-v2/canary_csi_transfer_test.go deleted file mode 100644 index 84928cfe79..0000000000 --- a/scripts/migrate-longhorn-v2/canary_csi_transfer_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package main - -import "testing" - -func csiTransferFixture(kind string) (object, object, object, object) { - parent := resource("longhorn-v2-canary-1234") - parent["kind"] = kind - parent["metadata"].(object)["namespace"] = "velero" - source, target := resource("fio-data"), resource("fio-data") - source["metadata"].(object)["namespace"], source["metadata"].(object)["uid"] = "longhorn-canary-v1-1234", "source-claim" - target["metadata"].(object)["namespace"], target["metadata"].(object)["uid"] = "longhorn-canary-v2-1234", "target-claim" - op := resource("transfer") - op["metadata"].(object)["namespace"] = "velero" - op["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "velero.io/v1", "kind": kind, "name": text(parent, "metadata", "name"), "uid": text(parent, "metadata", "uid"), "controller": true}} - op["status"] = object{"phase": "Completed", "completionTimestamp": "2026-10-29T22:00:00Z", "snapshotID": "repository-snapshot", "progress": object{"bytesDone": float64(134217800), "totalBytes": float64(134217800)}} - op["spec"] = object{"backupStorageLocation": "default", "datamover": "velero", "sourceNamespace": text(source, "metadata", "namespace")} - spec := op["spec"].(object) - if kind == "Backup" { - op["kind"] = "DataUpload" - op["metadata"].(object)["labels"] = object{"velero.io/backup-uid": text(parent, "metadata", "uid"), "velero.io/pvc-uid": text(source, "metadata", "uid")} - spec["snapshotType"], spec["sourcePVC"] = "CSI", "fio-data" - spec["csiSnapshot"] = object{"driver": "driver.longhorn.io", "storageClass": "lh-canary-v1-1234", "snapshotClass": "lh-csi-canary-1234", "volumeSnapshot": "native-snapshot"} - } else { - op["kind"] = "DataDownload" - op["metadata"].(object)["labels"] = object{"velero.io/restore-uid": text(parent, "metadata", "uid")} - spec["targetVolume"] = object{"pvc": "fio-data", "namespace": text(target, "metadata", "namespace")} - spec["snapshotID"], spec["snapshotSize"] = "repository-snapshot", float64(134217800) - delete(op["status"].(object), "snapshotID") - } - return parent, source, target, op -} - -func TestCSICanaryBindsBothNativeDataMovementOperations(t *testing.T) { - for _, kind := range []string{"Backup", "Restore"} { - parent, source, target, op := csiTransferFixture(kind) - found, snapshot, err := verifyCSITransfer([]object{op}, parent, source, target, "repository-snapshot", "lh-canary-v1-1234", "lh-csi-canary-1234") - if err != nil || found == nil || snapshot != "repository-snapshot" { - t.Fatalf("valid %s transfer rejected: %v", kind, err) - } - } -} - -func TestCSICanaryRejectsForeignOrIncompleteVolumeTransfers(t *testing.T) { - for _, kind := range []string{"Backup", "Restore"} { - for name, change := range map[string]func(object){ - "foreign owner": func(o object) { value(o, "metadata", "ownerReferences").([]any)[0].(object)["uid"] = "another-backup" }, - "foreign repository": func(o object) { o["spec"].(object)["backupStorageLocation"] = "another-destination" }, - "wrong source namespace": func(o object) { o["spec"].(object)["sourceNamespace"] = "production" }, - "incomplete payload": func(o object) { value(o, "status", "progress").(object)["bytesDone"] = float64(128) }, - "uncompleted operation": func(o object) { o["status"].(object)["phase"] = "InProgress" }, - } { - t.Run(kind+"/"+name, func(t *testing.T) { - parent, source, target, op := csiTransferFixture(kind) - change(op) - if _, _, err := verifyCSITransfer([]object{op}, parent, source, target, "repository-snapshot", "lh-canary-v1-1234", "lh-csi-canary-1234"); err == nil { - t.Fatal("unproven CSI recovery accepted") - } - }) - } - } - for _, change := range []func(object){ - func(o object) { o["metadata"].(object)["labels"].(object)["velero.io/pvc-uid"] = "replaced-claim" }, - func(o object) { value(o, "spec", "csiSnapshot").(object)["driver"] = "another-driver" }, - func(o object) { value(o, "spec", "csiSnapshot").(object)["snapshotClass"] = "original-history" }, - func(o object) { o["status"].(object)["snapshotID"] = "" }, - } { - parent, source, target, op := csiTransferFixture("Backup") - change(op) - if _, _, err := verifyCSITransfer([]object{op}, parent, source, target, "repository-snapshot", "lh-canary-v1-1234", "lh-csi-canary-1234"); err == nil { - t.Fatal("foreign upload accepted") - } - } - for _, change := range []func(object){ - func(o object) { value(o, "spec", "targetVolume").(object)["namespace"] = "production" }, - func(o object) { value(o, "spec", "targetVolume").(object)["pvc"] = "another-claim" }, - func(o object) { o["spec"].(object)["snapshotID"] = "another-point-in-time" }, - } { - parent, source, target, op := csiTransferFixture("Restore") - change(op) - if _, _, err := verifyCSITransfer([]object{op}, parent, source, target, "repository-snapshot", "lh-canary-v1-1234", "lh-csi-canary-1234"); err == nil { - t.Fatal("foreign download accepted") - } - } -} diff --git a/scripts/migrate-longhorn-v2/canary_identity_test.go b/scripts/migrate-longhorn-v2/canary_identity_test.go deleted file mode 100644 index be5b406ba4..0000000000 --- a/scripts/migrate-longhorn-v2/canary_identity_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "strings" - "testing" -) - -func canaryPodFixture() object { - assets, err := canaryProbeResources("scratch", "class", "1234", "v1") - if err != nil { - panic(err) - } - o := assets[4] - o["metadata"].(object)["uid"] = "probe" - o["spec"].(object)["nodeName"] = "prod-worker-3" - o["status"] = object{"conditions": readyConditions(), "containerStatuses": []any{object{"name": "fio", "imageID": strings.Split(fioImage, "@")[1]}}} - return o -} - -func TestCanaryCannotExecuteAgainstAReplacedOrUnreadyPod(t *testing.T) { - if err := verifyCanaryPod(canaryPodFixture(), "probe", "prod-worker-3"); err != nil { - t.Fatal(err) - } - for name, change := range map[string]func(object){ - "unready": func(p object) { p["status"].(object)["conditions"] = []any{} }, - "replaced": func(p object) { p["metadata"].(object)["uid"] = "replacement" }, - "wrong client": func(p object) { p["spec"].(object)["nodeName"] = "other" }, - "changed script": func(p object) { value(p, "spec", "containers").([]any)[0].(object)["args"] = []any{"other"} }, - "wrong executable": func(p object) { value(p, "status", "containerStatuses").([]any)[0].(object)["imageID"] = "unpinned" }, - "host users": func(p object) { p["spec"].(object)["hostUsers"] = true }, - } { - t.Run(name, func(t *testing.T) { - p := canaryPodFixture() - change(p) - if verifyCanaryPod(p, "probe", "prod-worker-3") == nil { - t.Fatal("canary may execute against an unverified consumer") - } - }) - } -} - -func TestCanaryBindingCannotAdoptAnUnrelatedClassOrClaim(t *testing.T) { - for _, phase := range []string{"v1", "v2"} { - group := "source" - if phase == "v2" { - group = "target" - } - class, claim := resource("class"), resource("fio-data") - claim["metadata"].(object)["labels"] = object{"velero.io/restore-name": "longhorn-v2-canary-1234", "velero.io/backup-name": "longhorn-v2-canary-1234"} - r := canaryRecord{Run: "1234", Resources: map[string]benchmarkResource{group + "/StorageClass": benchmarkIdentity(class), "source/PersistentVolumeClaim": benchmarkIdentity(claim)}} - if err := verifyCanaryBindingIdentity(phase, r, class, claim); err != nil { - t.Fatal(err) - } - class["metadata"].(object)["uid"] = "replacement" - if verifyCanaryBindingIdentity(phase, r, class, claim) == nil { - t.Fatal("foreign storage class adopted") - } - class["metadata"].(object)["uid"] = "uid-class" - if phase == "v1" { - claim["metadata"].(object)["uid"] = "replacement" - } else { - claim["metadata"].(object)["labels"] = object{} - } - if verifyCanaryBindingIdentity(phase, r, class, claim) == nil { - t.Fatal("unrelated source or restored claim adopted") - } - } -} diff --git a/scripts/migrate-longhorn-v2/canary_repository.go b/scripts/migrate-longhorn-v2/canary_repository.go deleted file mode 100644 index 2cf4391905..0000000000 --- a/scripts/migrate-longhorn-v2/canary_repository.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "errors" - "reflect" -) - -// Velero v1.18.4 selects Kopia repositories by this label tuple. Its Restic -// identifier is empty for Kopia. Bind the supported key, ready native object -// and spec, retaining that same repository across backup and restore. -func (p *canaryRun) verifyFilesystemTransfer(repositories, transfers []object, parent, pod object, expectedSnapshot, sourceNS string) (string, error) { - if sourceNS == "" || text(parent, "kind") == "Backup" && text(pod, "metadata", "namespace") != sourceNS || text(parent, "kind") == "Restore" && p.kopiaRepository == nil { - return "", errors.New("canary Kopia source repository is unbound") - } - var selected object - for _, repository := range repositories { - if text(repository, "spec", "volumeNamespace") != sourceNS && text(repository, "metadata", "labels", "velero.io/volume-namespace") != sourceNS { - continue - } - if selected != nil || text(repository, "apiVersion") != "velero.io/v1" || text(repository, "kind") != "BackupRepository" || - text(repository, "metadata", "namespace") != "velero" || text(repository, "metadata", "name") == "" || text(repository, "metadata", "uid") == "" || text(repository, "metadata", "resourceVersion") == "" || value(repository, "metadata", "deletionTimestamp") != nil || - text(repository, "metadata", "labels", "velero.io/volume-namespace") != sourceNS || text(repository, "metadata", "labels", "velero.io/storage-location") != "default" || text(repository, "metadata", "labels", "velero.io/repository-type") != "kopia" || - text(repository, "spec", "volumeNamespace") != sourceNS || text(repository, "spec", "backupStorageLocation") != "default" || text(repository, "spec", "repositoryType") != "kopia" || text(repository, "spec", "resticIdentifier") != "" || text(repository, "status", "phase") != "Ready" { - return "", errors.New("canary Kopia repository key, identity or readiness is unproven") - } - selected = repository - } - if selected == nil || p.kopiaRepository != nil && (benchmarkIdentity(selected) != benchmarkIdentity(p.kopiaRepository) || !reflect.DeepEqual(value(selected, "spec"), value(p.kopiaRepository, "spec"))) { - return "", errors.New("canary Kopia repository is missing or changed") - } - snapshot, err := verifyCanaryTransfer(transfers, parent, pod, expectedSnapshot, sourceNS) - if err != nil { - return "", err - } - if text(parent, "kind") == "Backup" { - p.kopiaRepository = selected - } - return snapshot, nil -} diff --git a/scripts/migrate-longhorn-v2/canary_repository_test.go b/scripts/migrate-longhorn-v2/canary_repository_test.go deleted file mode 100644 index 3a91a21018..0000000000 --- a/scripts/migrate-longhorn-v2/canary_repository_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package main - -import "testing" - -func canaryRepositoryFixture() object { - repository := resource("native-kopia-repository") - repository["apiVersion"], repository["kind"] = "velero.io/v1", "BackupRepository" - repository["metadata"].(object)["namespace"] = "velero" - repository["metadata"].(object)["labels"] = object{"velero.io/volume-namespace": "longhorn-canary-v1-1234", "velero.io/storage-location": "default", "velero.io/repository-type": "kopia"} - repository["spec"] = object{"volumeNamespace": "longhorn-canary-v1-1234", "backupStorageLocation": "default", "repositoryType": "kopia"} - repository["status"] = object{"phase": "Ready"} - return repository -} - -func TestFilesystemRecoveryUsesTheSameReadyKopiaRepository(t *testing.T) { - var p canaryRun - repository := canaryRepositoryFixture() - parent, pod, transfer := transferFixture("Backup") - if _, err := p.verifyFilesystemTransfer([]object{repository}, []object{transfer}, parent, pod, "", "longhorn-canary-v1-1234"); err != nil { - t.Fatal("native Kopia backup without a Restic identifier was refused:", err) - } - parent, pod, transfer = transferFixture("Restore") - pod["metadata"].(object)["namespace"] = "longhorn-canary-v2-1234" - transfer["spec"].(object)["pod"].(object)["namespace"] = "longhorn-canary-v2-1234" - current := canaryRepositoryFixture() - current["metadata"].(object)["resourceVersion"] = "2" - current["status"].(object)["lastMaintenanceTime"] = "2026-11-03T00:00:00Z" - if _, err := p.verifyFilesystemTransfer([]object{current}, []object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err != nil { - t.Fatal("ordinary repository status updates broke restoration:", err) - } - for name, mutate := range map[string]func(object){ - "replaced repository": func(o object) { o["metadata"].(object)["uid"] = "replacement" }, - "changed repository config": func(o object) { o["spec"].(object)["repositoryConfig"] = object{"changed": "true"} }, - "wrong location": func(o object) { o["spec"].(object)["backupStorageLocation"] = "other" }, - "wrong labels": func(o object) { o["metadata"].(object)["labels"].(object)["velero.io/volume-namespace"] = "other" }, - "wrong source": func(o object) { o["spec"].(object)["volumeNamespace"] = "other" }, - "wrong type": func(o object) { o["spec"].(object)["repositoryType"] = "restic" }, - "not ready": func(o object) { o["status"].(object)["phase"] = "NotReady" }, - "deleting": func(o object) { o["metadata"].(object)["deletionTimestamp"] = "2026-11-03T00:00:00Z" }, - } { - t.Run(name, func(t *testing.T) { - candidate := canaryRepositoryFixture() - mutate(candidate) - if _, err := p.verifyFilesystemTransfer([]object{candidate}, []object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { - t.Fatal("repository replacement or contradictory native evidence was accepted") - } - }) - } - for name, repositories := range map[string][]object{"missing": nil, "ambiguous": {repository, repository}} { - t.Run(name, func(t *testing.T) { - if _, err := p.verifyFilesystemTransfer(repositories, []object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { - t.Fatal("incomplete repository evidence accepted") - } - }) - } - var unbound canaryRun - if _, err := unbound.verifyFilesystemTransfer([]object{repository}, []object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { - t.Fatal("restore accepted without the verified backup repository") - } -} diff --git a/scripts/migrate-longhorn-v2/canary_transfer.go b/scripts/migrate-longhorn-v2/canary_transfer.go deleted file mode 100644 index 6c1b910872..0000000000 --- a/scripts/migrate-longhorn-v2/canary_transfer.go +++ /dev/null @@ -1,59 +0,0 @@ -package main - -import "errors" - -// controlledBy requires the exact native Velero parent as controller owner. -func controlledBy(o, parent object) bool { - refs, _ := value(o, "metadata", "ownerReferences").([]any) - for _, ref := range refs { - r, _ := ref.(object) - if text(r, "apiVersion") == "velero.io/v1" && text(r, "kind") == text(parent, "kind") && text(r, "name") == text(parent, "metadata", "name") && - text(r, "uid") == text(parent, "metadata", "uid") && text(r, "uid") != "" && value(r, "controller") == true { - return true - } - } - return false -} - -// verifyCanaryTransfer binds one complete Kopia payload transfer to the original -// pod and backup destination; restores must name that exact source snapshot. -func verifyCanaryTransfer(transfers []object, parent, pod object, expectedSnapshot, sourceNS string) (string, error) { - if text(parent, "metadata", "uid") == "" || text(pod, "metadata", "uid") == "" || (text(parent, "kind") != "Backup" && text(parent, "kind") != "Restore") { - return "", errors.New("canary transfer ownership is missing") - } - matched, snapshot := 0, "" - for _, transfer := range transfers { - owned := controlledBy(transfer, parent) - if !owned && text(transfer, "spec", "pod", "uid") != text(pod, "metadata", "uid") { - continue - } - matched++ - if !owned || text(transfer, "metadata", "namespace") != "velero" || text(transfer, "spec", "pod", "uid") != text(pod, "metadata", "uid") || - text(transfer, "spec", "pod", "namespace") != text(pod, "metadata", "namespace") || text(transfer, "spec", "pod", "name") != "fio" || - text(transfer, "spec", "volume") != "data" || text(transfer, "spec", "backupStorageLocation") != "default" || text(transfer, "spec", "repoIdentifier") != "" || text(transfer, "spec", "uploaderType") != "kopia" { - return "", errors.New("canary data transfer belongs to another pod, volume, repository or operation") - } - if err := cleanVeleroResult(transfer); err != nil { - return "", err - } - done, doneOK := value(transfer, "status", "progress", "bytesDone").(float64) - total, totalOK := value(transfer, "status", "progress", "totalBytes").(float64) - if !doneOK || !totalOK || done < 134217728 || done != total { - return "", errors.New("complete canary payload transfer is unproven") - } - snapshot = text(transfer, "status", "snapshotID") - if text(parent, "kind") == "Restore" { - snapshot = text(transfer, "spec", "snapshotID") - if snapshot != expectedSnapshot || text(transfer, "spec", "sourceNamespace") != sourceNS { - return "", errors.New("canary restore did not use the exact source snapshot") - } - } - if snapshot == "" { - return "", errors.New("canary volume-data snapshot identity is missing") - } - } - if matched != 1 { - return "", errors.New("exactly one completed canary volume-data transfer is required") - } - return snapshot, nil -} diff --git a/scripts/migrate-longhorn-v2/canary_transfer_test.go b/scripts/migrate-longhorn-v2/canary_transfer_test.go deleted file mode 100644 index 2c1edb0568..0000000000 --- a/scripts/migrate-longhorn-v2/canary_transfer_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import "testing" - -func transferFixture(kind string) (object, object, object) { - parent, pod, transfer := resource("canary"), resource("fio"), resource("transfer") - parent["kind"] = kind - pod["metadata"].(object)["namespace"] = "longhorn-canary-v1-1234" - transfer["metadata"].(object)["namespace"] = "velero" - transfer["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "velero.io/v1", "kind": kind, "name": text(parent, "metadata", "name"), "uid": text(parent, "metadata", "uid"), "controller": true}} - // Velero leaves the Restic-only repoIdentifier field absent for Kopia. - transfer["spec"] = object{"pod": object{"name": "fio", "namespace": text(pod, "metadata", "namespace"), "uid": text(pod, "metadata", "uid")}, "volume": "data", "backupStorageLocation": "default", "uploaderType": "kopia", "snapshotID": "snapshot", "sourceNamespace": "longhorn-canary-v1-1234"} - transfer["status"] = object{"phase": "Completed", "completionTimestamp": "2026-10-29T19:00:00Z", "snapshotID": "snapshot", "progress": object{"bytesDone": float64(134217800), "totalBytes": float64(134217800)}} - return parent, pod, transfer -} - -func TestCanaryRequiresTheExactCompletedDataTransfer(t *testing.T) { - for _, kind := range []string{"Backup", "Restore"} { - parent, pod, transfer := transferFixture(kind) - if _, err := verifyCanaryTransfer([]object{transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err != nil { - t.Fatal(err) - } - for name, mutate := range map[string]func(object){ - "unrelated parent": func(o object) { o["metadata"].(object)["ownerReferences"] = []any{} }, - "wrong pod uid": func(o object) { o["spec"].(object)["pod"].(object)["uid"] = "replacement" }, - "wrong volume": func(o object) { o["spec"].(object)["volume"] = "scratch" }, - "wrong backup location": func(o object) { o["spec"].(object)["backupStorageLocation"] = "other" }, - "unproven progress": func(o object) { o["status"].(object)["progress"] = object{} }, - "incomplete transfer": func(o object) { o["status"].(object)["progress"].(object)["bytesDone"] = float64(32) }, - "failed transfer": func(o object) { o["status"].(object)["phase"] = "Failed" }, - "restic instead of kopia": func(o object) { o["spec"].(object)["uploaderType"] = "restic" }, - "restic repository id": func(o object) { o["spec"].(object)["repoIdentifier"] = "restic-only" }, - } { - t.Run(kind+"/"+name, func(t *testing.T) { - p, pod, transfer := transferFixture(kind) - mutate(transfer) - if _, err := verifyCanaryTransfer([]object{transfer}, p, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { - t.Fatal("backup metadata was mistaken for actual volume-data recovery") - } - }) - } - if _, err := verifyCanaryTransfer([]object{transfer, transfer}, parent, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { - t.Fatal("ambiguous transfer inventory accepted") - } - } - p, pod, transfer := transferFixture("Restore") - transfer["spec"].(object)["snapshotID"] = "older-backup" - if _, err := verifyCanaryTransfer([]object{transfer}, p, pod, "snapshot", "longhorn-canary-v1-1234"); err == nil { - t.Fatal("restore from a different snapshot accepted") - } -} diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index e00b56e93e..504b6118d9 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -12,23 +12,8 @@ import ( // and protected workflow identities before constructing any production client. func main() { if len(os.Args) > 1 { - if len(os.Args) != 3 || (os.Args[1] != "--render-canary" && os.Args[1] != "--render-csi-canary") { - fmt.Fprintln(os.Stderr, "Only the explicit offline canary rendering arguments are supported.") - os.Exit(1) - } - render := canaryResources - if os.Args[1] == "--render-csi-canary" { - render = csiCanaryResources - } - assets, err := render(os.Args[2]) - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - if err = json.NewEncoder(os.Stdout).Encode(object{"apiVersion": "v1", "kind": "List", "items": assets}); err != nil { - os.Exit(1) - } - return + fmt.Fprintln(os.Stderr, "Migration operations accept no command-line arguments.") + os.Exit(1) } operation := os.Getenv("LONGHORN_V2_OPERATION") target := os.Getenv("LONGHORN_V2_NODE") @@ -40,8 +25,6 @@ func main() { authorization = authorizeTemporaryPreparation(os.Getenv, target) case "benchmark-v1", "benchmark-v2": authorization = authorizeBenchmark(os.Getenv, operation[len("benchmark-"):]) - case "canary-v2", "canary-csi-v2": - authorization = authorizeCanary(os.Getenv) case "register-temporary-v2-storage": authorization = authorizeTemporaryActivation(os.Getenv) case "inventory-replicas": @@ -109,18 +92,6 @@ func main() { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - if operation == "canary-v2" || operation == "canary-csi-v2" { - transport := "filesystem" - if operation == "canary-csi-v2" { - transport = "csi" - } - if err = runCanaryWithTransport(ctx, client{run: execute, context: "admin@prod", inventoryPools: pools}, os.Getenv("GITHUB_SHA"), os.Getenv("GITHUB_RUN_ID"), pools, transport); err != nil { - fmt.Fprintln(os.Stderr, "V2 recovery canary held:", err) - fmt.Fprintln(os.Stderr, "Preserve the canary journal and scratch resources; no application cutover is authorized.") - os.Exit(1) - } - return - } if operation == "register-temporary-v2-storage" { activation := temporaryV2Activation{c: client{run: execute, context: "admin@prod", inventoryPools: pools}, source: os.Getenv("GITHUB_SHA"), owner: os.Getenv("GITHUB_RUN_ID"), pools: pools} if err = activation.run(ctx); err != nil { diff --git a/scripts/migrate-longhorn-v2/temporary_activation_authorization_test.go b/scripts/migrate-longhorn-v2/temporary_activation_authorization_test.go index 69fff80a29..82e727d2d4 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_authorization_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_authorization_test.go @@ -8,6 +8,8 @@ import ( "gopkg.in/yaml.v3" ) +// Registration confirmation grants neither host disruption nor another +// operation, and cannot be reused outside the protected first main attempt. func TestTemporaryActivationRequiresItsOwnFirstAttemptMainConfirmation(t *testing.T) { env := dispatchFixture() env["LONGHORN_V2_OPERATION"] = "register-temporary-v2-storage" @@ -26,11 +28,17 @@ func TestTemporaryActivationRequiresItsOwnFirstAttemptMainConfirmation(t *testin } }) } - if authorizeCanary(getenv) == nil || authorizeHostReboot(getenv, "prod-worker-1") == nil { - t.Fatal("disk registration authorized a canary or host disruption") + if authorizeHostReboot(getenv, "prod-worker-1") == nil { + t.Fatal("disk registration authorized host disruption") + } + env["LONGHORN_V2_OPERATION"] = "canary-v2" + if authorizeTemporaryActivation(getenv) == nil { + t.Fatal("recovery data movement was admitted as disk registration") } } +// Execute the actual workflow gate so a different operation's confirmation +// cannot reach production credentials through the registration branch. func TestTemporaryActivationWorkflowRequiresTheDistinctConfirmation(t *testing.T) { data, err := os.ReadFile("../../.github/workflows/prepare-longhorn-v2-node.yaml") if err != nil { diff --git a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go index 2906c8d95b..bdaa0b14c6 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go @@ -26,6 +26,8 @@ type activationAPI struct { preparationFailed bool } +// newActivationAPI models complete fixed-roster reads and successful native +// preparation, with writes restricted to the activation journal and raw nodes. func newActivationAPI(t *testing.T) (*activationAPI, temporaryV2Activation) { t.Helper() s, managers := v2ReadinessFixture() @@ -96,6 +98,8 @@ func newActivationAPI(t *testing.T) (*activationAPI, temporaryV2Activation) { return f, a } +// run enforces the production command boundary and identity-preconditioned +// writes, including an uncertain response after an actual disk registration. func (f *activationAPI) run(_ context.Context, input []byte, program string, args ...string) ([]byte, error) { f.t.Helper() if program == "git" && reflect.DeepEqual(args, []string{"ls-remote", "--exit-code", "https://github.com/devantler-tech/platform.git", "refs/heads/main"}) { @@ -242,6 +246,8 @@ func (f *activationAPI) run(_ context.Context, input []byte, program string, arg return nil, nil } +// Registered disks must remain eligible to the independent publication reader; +// replaying the completed operation must not repeat a disk write. func TestTemporaryActivationRegistersOnlyThreeJournalOwnedRawDisks(t *testing.T) { f, a := newActivationAPI(t) if err := a.run(t.Context()); err != nil { @@ -270,6 +276,7 @@ func TestTemporaryActivationRegistersOnlyThreeJournalOwnedRawDisks(t *testing.T) } } +// No journal or disk may be written without successful native preparation. func TestTemporaryActivationHoldsBeforeWritesWithoutSuccessfulPreparation(t *testing.T) { for _, mode := range []string{"missing preparation", "failed preparation", "unfinished preparation"} { t.Run(mode, func(t *testing.T) { @@ -290,6 +297,8 @@ func TestTemporaryActivationHoldsBeforeWritesWithoutSuccessfulPreparation(t *tes } } +// Only a later owner may resume an uncertain owned patch; an unrelated storage +// owner's annotation cannot be adopted even when the desired disk matches. func TestTemporaryActivationResumesItsUncertainWriteWithoutAdoptingForeignStorage(t *testing.T) { f, a := newActivationAPI(t) f.lostResponse = true @@ -307,6 +316,44 @@ func TestTemporaryActivationResumesItsUncertainWriteWithoutAdoptingForeignStorag } } +// Completed and interrupted registration survive a normal host reboot without +// replaying an owned disk write or rewriting the original observation evidence. +func TestTemporaryActivationPreservesStorageIdentityAcrossReboot(t *testing.T) { + for _, interrupted := range []bool{false, true} { + t.Run(fmt.Sprintf("interrupted=%t", interrupted), func(t *testing.T) { + f, a := newActivationAPI(t) + f.lostResponse = interrupted + if err := a.run(t.Context()); (err != nil) != interrupted { + t.Fatal("initial registration did not reach the selected boundary:", err) + } + _, original, err := a.read(t.Context()) + if err != nil { + t.Fatal(err) + } + for i, node := range f.s.nodes { + if !strings.HasPrefix(text(node, "metadata", "name"), "autoscale-longhorn-") { + continue + } + info := value(node, "status", "nodeInfo").(object) + info["bootID"] = fmt.Sprintf("12345678-1234-4234-8234-%012x", i+500) + info["osImage"] = "Talos upgraded fixture" + } + a.owner = "789" + if err = a.run(t.Context()); err != nil || f.writes != 3 { + t.Fatal("normal reboot invalidated intact prepared storage:", err) + } + if err = a.accepted(t.Context()); err != nil { + t.Fatal("completed storage was not accepted after reboot:", err) + } + _, recovered, err := a.read(t.Context()) + if err != nil || !reflect.DeepEqual(original.Targets, recovered.Targets) { + t.Fatal("recovery rewrote the original registration observations:", err) + } + }) + } +} + +// Matching but unreviewed manager images must fail before disk registration. func TestTemporaryActivationRejectsUnqualifiedClientsBeforeRegisteringDisks(t *testing.T) { f, a := newActivationAPI(t) f.managers[0]["spec"].(object)["image"] = "example.invalid/unreviewed:latest" @@ -318,7 +365,9 @@ func TestTemporaryActivationRejectsUnqualifiedClientsBeforeRegisteringDisks(t *t } } -func TestCanaryRequiresCompletedStillOwnedTemporaryActivation(t *testing.T) { +// Read-only acceptance must revalidate the completed workflow and current +// storage identities rather than trust a cached completion bit. +func TestCompletedTemporaryActivationRequiresCurrentOwnedStorage(t *testing.T) { f, a := newActivationAPI(t) if a.accepted(t.Context()) == nil || f.writes != 0 { t.Fatal("an unregistered pool authorized the canary") diff --git a/scripts/migrate-longhorn-v2/temporary_activation_native_test.go b/scripts/migrate-longhorn-v2/temporary_activation_native_test.go index c01c2e9309..6732527389 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_native_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_native_test.go @@ -14,6 +14,7 @@ type activationBootReader struct { reads int } +// BootID supplies independently selected observations on the two bracket sides. func (r *activationBootReader) BootID(context.Context) (string, error) { r.reads++ if r.reads == 1 { @@ -22,6 +23,8 @@ func (r *activationBootReader) BootID(context.Context) (string, error) { return r.after, nil } +// A binding requires matching native boot, hardware and API identities plus a +// successful reader close; durable identity comparison cannot waive this bracket. func TestTemporaryRawBindingBracketsNativeReadsAndClosesItsReader(t *testing.T) { for _, mode := range []string{"valid", "boot changed", "hardware changed", "node changed", "partition unreadable", "close failed"} { t.Run(mode, func(t *testing.T) { diff --git a/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go b/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go index d166412fa7..ee9826deb7 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go @@ -6,6 +6,8 @@ import ( "testing" ) +// temporaryPreparedFixture records an earlier encrypted filesystem and boot, +// both distinct from the post-reset binding used for activation. func temporaryPreparedFixture(t *testing.T, binding temporaryV2Binding) object { t.Helper() j := journal{Schema: 2, Target: binding.Name, Source: strings.Repeat("1", 40), OwnerRun: "7777", Phase: "prepared", @@ -22,6 +24,8 @@ func temporaryPreparedFixture(t *testing.T, binding temporaryV2Binding) object { return cm } +// Foreign, unfinished, replaced or ambiguous preparation cannot authorize a +// raw disk, and completion must prove the original filesystem was replaced. func TestTemporaryActivationRequiresTheCompletedPreparationIdentity(t *testing.T) { n, lh, raw, eph, pool := temporaryRawBindingFixture() binding, err := projectTemporaryV2Binding("temporary-v1-1", n, lh, raw, eph, pool, text(n, "status", "nodeInfo", "bootID")) diff --git a/scripts/migrate-longhorn-v2/temporary_activation_record_test.go b/scripts/migrate-longhorn-v2/temporary_activation_record_test.go index b2f03a83bc..d9b03c6e36 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_record_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_record_test.go @@ -7,6 +7,8 @@ import ( "testing" ) +// temporaryActivationRecordFixture gives each selected worker independent +// provider, hardware, partition and preparation identities. func temporaryActivationRecordFixture(t *testing.T) temporaryActivationRecord { t.Helper() n, lh, raw, eph, pool := temporaryRawBindingFixture() @@ -30,6 +32,8 @@ func temporaryActivationRecordFixture(t *testing.T) temporaryActivationRecord { return record } +// The decoder must reject missing or reused identities and ambiguous JSON +// rather than let a partial or differently interpreted plan authorize storage. func TestTemporaryActivationRecordRequiresThreeDistinctCompleteBindings(t *testing.T) { record := temporaryActivationRecordFixture(t) data, _ := json.Marshal(record) diff --git a/scripts/migrate-longhorn-v2/temporary_activation_run.go b/scripts/migrate-longhorn-v2/temporary_activation_run.go index fbb34877a8..7057060b58 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_run.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_run.go @@ -65,7 +65,7 @@ func (a temporaryV2Activation) environment(ctx context.Context) (inventory, erro if err := verifyBudgetSettings(ctx, a.c); err != nil { return inventory{}, err } - if err := verifyCanarySettings(ctx, a.c); err != nil { + if err := verifyV2Settings(ctx, a.c); err != nil { return inventory{}, err } s, err := a.c.snapshot(ctx) @@ -73,7 +73,7 @@ func (a temporaryV2Activation) environment(ctx context.Context) (inventory, erro return inventory{}, errTemporaryActivation } managers, err := a.c.list(ctx, "instancemanagers.longhorn.io", "longhorn-system") - if err != nil || verifyV2Clients(s, managers, a.pools, canaryManagerImage) != nil { + if err != nil || verifyV2Clients(s, managers, a.pools, v2ManagerImage) != nil { return inventory{}, errTemporaryActivation } for _, node := range s.nodes { @@ -130,7 +130,7 @@ func (a temporaryV2Activation) targets(ctx context.Context, s inventory, journal return nil, errTemporaryActivation } binding, err = a.prepared(ctx, binding) - if err != nil || expected != nil && expected[selection] != binding { + if err != nil || expected != nil && !expected[selection].SameStorageIdentity(binding) { return nil, errTemporaryActivation } result[selection] = binding @@ -235,7 +235,7 @@ func (a temporaryV2Activation) run(ctx context.Context) error { if _, err = a.targets(ctx, s, uid, record.Targets); err != nil { return errors.Join(errReadinessTerminal, err) } - _, err = canaryEnvironment(ctx, a.c, a.pools) + _, err = v2Environment(ctx, a.c, a.pools) return err }); err != nil { return err @@ -254,7 +254,7 @@ func (a temporaryV2Activation) run(ctx context.Context) error { return nil } -// accepted is a read-only canary admission check. The completed journal must +// accepted is a read-only registration acceptance check. The completed journal must // come from a terminated protected workflow and still match all three native // partitions and journal-owned registrations; its cached completion bit alone // never authorizes IO. diff --git a/scripts/migrate-longhorn-v2/temporary_activation_test.go b/scripts/migrate-longhorn-v2/temporary_activation_test.go index d98421135a..098c465506 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_test.go @@ -4,6 +4,8 @@ import ( "testing" ) +// temporaryRawBindingFixture supplies one pinned worker and separate encrypted +// filesystem and raw partitions without using the registration builder. func temporaryRawBindingFixture() (object, object, object, object, migrationPool) { n := capacityFixture()[3] n["spec"] = object{"providerID": "hcloud://123", "unschedulable": false} @@ -16,6 +18,8 @@ func temporaryRawBindingFixture() (object, object, object, object, migrationPool return n, lh, raw, eph, migrationPool{name: "autoscale-longhorn-v1", role: "v1", serverType: "cpx42", count: 2} } +// Wrong ownership, boot, encryption, capacity or pool selection must prevent a +// native partition observation from becoming registration evidence. func TestTemporaryRawRegistrationBindsTheNativePartitionsAndPoolMember(t *testing.T) { n, lh, raw, eph, pool := temporaryRawBindingFixture() binding, err := projectTemporaryV2Binding("temporary-v1-1", n, lh, raw, eph, pool, text(n, "status", "nodeInfo", "bootID")) @@ -52,6 +56,8 @@ func TestTemporaryRawRegistrationBindsTheNativePartitionsAndPoolMember(t *testin } } +// Resumption recognizes only an empty fence or this journal's exact raw disk; +// matching paths without ownership never authorize adoption. func TestTemporaryRawRegistrationAcceptsOnlyItsOwnPartialActivation(t *testing.T) { storage := func(enabled bool) object { _, lh, _, _, _ := temporaryRawBindingFixture() diff --git a/scripts/migrate-longhorn-v2/v2_environment.go b/scripts/migrate-longhorn-v2/v2_environment.go new file mode 100644 index 0000000000..ba4c3ae47c --- /dev/null +++ b/scripts/migrate-longhorn-v2/v2_environment.go @@ -0,0 +1,58 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" +) + +// v2ManagerImage is qualified against the reviewed Longhorn 1.13.0 chart; +// matching unreviewed live declarations cannot establish image provenance. +const v2ManagerImage = "longhornio/longhorn-instance-manager:v1.13.0" + +// verifyV2Settings binds coexistence and effective SPDK configuration to the +// reviewed profile before either registration or subsequent storage acceptance. +func verifyV2Settings(ctx context.Context, c client) error { + for name, expected := range map[string]string{ + "v1-data-engine": "true", "v2-data-engine": "true", "guaranteed-instance-manager-cpu": `{"v1":"6","v2":"25"}`, + "data-engine-cpu-mask": `{"v2":"0x3"}`, "data-engine-number-of-cpu-cores": `{"v2":"0"}`, "data-engine-cpu-isolation-enabled": `{"v2":"true"}`, + "data-engine-interrupt-mode-enabled": `{"v2":"false"}`, "data-engine-hugepage-enabled": `{"v2":"true"}`, "data-engine-memory-size": `{"v2":"2048"}`, + } { + o, err := c.get(ctx, "settings.longhorn.io", "longhorn-system", name, false) + if err != nil { + return err + } + actual := text(o, "value") + if strings.HasPrefix(expected, "{") { + var want, got map[string]string + if json.Unmarshal([]byte(expected), &want) != nil || json.Unmarshal([]byte(actual), &got) != nil || !reflect.DeepEqual(want, got) { + return errors.New("live V2 settings have not adopted the reviewed configuration") + } + } else if actual != expected { + return errors.New("both engines must remain enabled during storage qualification") + } + } + return nil +} + +// v2Environment proves effective worker clients and three independent ready +// raw disks. Merely enabling the V2 setting cannot establish storage readiness. +func v2Environment(ctx context.Context, c client, pools map[string]migrationPool) (inventory, error) { + if err := verifyV2Settings(ctx, c); err != nil { + return inventory{}, err + } + s, err := c.snapshot(ctx) + if err != nil { + return inventory{}, err + } + managers, err := c.list(ctx, "instancemanagers.longhorn.io", "longhorn-system") + if err != nil { + return inventory{}, err + } + if err = verifyV2Runtime(s, managers, pools, v2ManagerImage); err != nil { + return inventory{}, err + } + return s, nil +} diff --git a/scripts/migrate-longhorn-v2/v2_readiness_test.go b/scripts/migrate-longhorn-v2/v2_readiness_test.go index a7701cb3c3..29940ee895 100644 --- a/scripts/migrate-longhorn-v2/v2_readiness_test.go +++ b/scripts/migrate-longhorn-v2/v2_readiness_test.go @@ -5,10 +5,14 @@ import ( "testing" ) +// v2ReadinessFixture keeps original V1 replicas on permanent workers while +// independent temporary disks supply the prospective V2 storage. func v2ReadinessFixture() (inventory, []object) { return v2ReadinessFixtureForRoute(false) } +// v2ReadinessFixtureForRoute can represent the superseded original-disk route +// solely to prove that it cannot satisfy temporary-only activation admission. func v2ReadinessFixtureForRoute(legacy bool) (inventory, []object) { s := evacuatedFixture() s.nodes = capacityFixture() @@ -55,6 +59,7 @@ func v2ReadinessFixtureForRoute(legacy bool) (inventory, []object) { return s, managers } +// Raw storage on original workers cannot borrow the temporary-route authority. func TestV2RuntimeRejectsTheEarlierRebuiltPermanentWorkerRoute(t *testing.T) { pools, err := configuredMigrationPools(poolConfigFixture()) if err != nil { @@ -66,7 +71,9 @@ func TestV2RuntimeRejectsTheEarlierRebuiltPermanentWorkerRoute(t *testing.T) { } } -func TestV2CanaryRequiresRealRawDisksAndReservedReactors(t *testing.T) { +// Ready disk identities, original redundancy and effective SPDK ownership, +// executable arguments and reservations are all required for storage readiness. +func TestV2RuntimeRequiresRealRawDisksAndReservedReactors(t *testing.T) { pools, err := configuredMigrationPools(poolConfigFixture()) if err != nil { t.Fatal(err) From a0a4502c312e2203ed5af5935873c1876bfedf27 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 9 Nov 2026 22:52:47 +0100 Subject: [PATCH 28/35] feat(storage): fit platform volumes on larger staging disks --- scripts/migrate-longhorn-v2/README.md | 10 ++- scripts/migrate-longhorn-v2/budget.go | 12 ++- scripts/migrate-longhorn-v2/client.go | 6 +- scripts/migrate-longhorn-v2/config.go | 12 ++- scripts/migrate-longhorn-v2/journal.go | 3 +- scripts/migrate-longhorn-v2/main.go | 2 +- scripts/migrate-longhorn-v2/prepare.go | 13 ++- .../migrate-longhorn-v2/staging-layout.yaml | 20 +++++ scripts/migrate-longhorn-v2/staging_layout.go | 67 ++++++++++++++ .../staging_layout_test.go | 49 ++++++++++ .../migrate-longhorn-v2/temporary_prepare.go | 37 ++++++-- .../temporary_prepare_flow_test.go | 89 +++++++++++++++++-- 12 files changed, 289 insertions(+), 31 deletions(-) create mode 100644 scripts/migrate-longhorn-v2/staging-layout.yaml create mode 100644 scripts/migrate-longhorn-v2/staging_layout.go create mode 100644 scripts/migrate-longhorn-v2/staging_layout_test.go diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 4732332c58..a177daf904 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -91,9 +91,15 @@ All three temporary workers receive owned scale-down protection before the selec cordoned. Drain uses eviction and respects disruption budgets. Two fresh storage observations after drain precede the recorded reset intent. +For new preparations, the native original partition must show at least 200 GiB. After drain +and a fresh empty-storage read, the workflow applies the 144 GiB raw profile without rebooting +the selected worker. It checks the loaded configuration and repeats the native empty-storage +read before recording reset intent. The profile retains a 40 GiB minimum for EPHEMERAL and +keeps both encryption keys locked to STATE. Existing journals retain their recorded geometry. + Recovery from `reset-requested` only observes post-state. An uncertain response never causes a -second reset. A new boot and encrypted EPHEMERAL UUID, the reviewed 96 GiB encrypted raw -partition, loaded configuration and huge-page readiness must be observed before releasing the +second reset. A new boot and encrypted EPHEMERAL UUID, the exact encrypted raw size bound +by the journal, loaded configuration and huge-page readiness must be observed before releasing the owned cordon. CPX42 workers retain their larger root capacity; the permanent-worker root-size cap does not apply to this separate profile. The raw disk remains unregistered and fenced; storage activation and volume migration follow separately. Original storage replicas remain on diff --git a/scripts/migrate-longhorn-v2/budget.go b/scripts/migrate-longhorn-v2/budget.go index 46b259a211..9c640cb8de 100644 --- a/scripts/migrate-longhorn-v2/budget.go +++ b/scripts/migrate-longhorn-v2/budget.go @@ -17,6 +17,10 @@ func verifyBudgetSettings(ctx context.Context, c client) error { } func verifyRebuildBudget(target string, s inventory) error { + return verifyRawDiskBudget(target, s, permanentRawDiskGiB) +} + +func verifyRawDiskBudget(target string, s inventory, rawGiB uint64) error { const gib = uint64(1024 * 1024 * 1024) var logical, actual uint64 if len(s.volumes) == 0 { @@ -27,21 +31,21 @@ func verifyRebuildBudget(target string, s inventory) error { return errors.New("the last V1 worker cannot be rebuilt while any V1 volume remains") } size, err := strconv.ParseUint(text(v, "spec", "size"), 10, 64) - if err != nil || size == 0 || size > 96*gib { + if err != nil || size == 0 || size > rawGiB*gib { return errors.New("volume logical size is unknown or outside the raw-disk budget") } // Unlike spec.size, status.actualSize is a JSON number. measured, ok := value(v, "status", "actualSize").(float64) - if !ok || measured < 0 || measured > float64(96*gib) { + if !ok || measured < 0 || measured > float64(rawGiB*gib) { return errors.New("volume actual size is unknown or outside the raw-disk budget") } used := uint64(measured) logical += size actual += used - // 96 GiB per target, 4 GiB disk reserve, 2 GiB metadata margin, + // 4 GiB disk reserve, 2 GiB metadata margin, // and one 4 GiB disposable canary or benchmark at a time. Actual data // must additionally leave the current 25% minimum free-space margin. - if logical > (96-4-2-4)*gib || actual > (96*75/100-4-2-4)*gib { + if logical > (rawGiB-4-2-4)*gib || actual > (rawGiB*75/100-4-2-4)*gib { return errors.New("current volumes leave insufficient raw-disk and canary headroom") } } diff --git a/scripts/migrate-longhorn-v2/client.go b/scripts/migrate-longhorn-v2/client.go index 47e044c648..be9d973099 100644 --- a/scripts/migrate-longhorn-v2/client.go +++ b/scripts/migrate-longhorn-v2/client.go @@ -48,6 +48,10 @@ func verifyResetLayout(raw, ephemeral object, originalUUID string) error { } func verifyEncryptedLocalLayout(raw, ephemeral object) error { + return verifyEncryptedRawDisk(raw, ephemeral, permanentRawDiskGiB) +} + +func verifyEncryptedRawDisk(raw, ephemeral object, rawGiB uint64) error { if text(raw, "metadata", "id") != "r-longhorn" || text(ephemeral, "metadata", "id") != "EPHEMERAL" || text(ephemeral, "spec", "uuid") == "" { return errors.New("encrypted local partition identities are missing") } @@ -68,7 +72,7 @@ func verifyEncryptedLocalLayout(raw, ephemeral object) error { } rawSize, rawOK := value(raw, "spec", "size").(float64) ephemeralSize, ephemeralOK := value(ephemeral, "spec", "size").(float64) - if !rawOK || !ephemeralOK || rawSize != 96*1024*1024*1024 || ephemeralSize < 40*1024*1024*1024 { + if !rawOK || !ephemeralOK || rawSize != float64(rawGiB<<30) || ephemeralSize < 40*1024*1024*1024 { return errors.New("partition capacities do not match the reviewed storage layout") } return nil diff --git a/scripts/migrate-longhorn-v2/config.go b/scripts/migrate-longhorn-v2/config.go index 128bf0617d..3eabbc24d4 100644 --- a/scripts/migrate-longhorn-v2/config.go +++ b/scripts/migrate-longhorn-v2/config.go @@ -2,7 +2,13 @@ package main import "errors" +const permanentRawDiskGiB = 96 + func verifyDesiredLayout(raw, ephemeral object) error { + return verifyRawDiskConfig(raw, ephemeral, permanentRawDiskGiB) +} + +func verifyRawDiskConfig(raw, ephemeral object, rawGiB uint64) error { if text(raw, "metadata", "id") != "r-longhorn" || text(ephemeral, "metadata", "id") != "EPHEMERAL" { return errors.New("the expected Talos volume configurations are missing") } @@ -23,11 +29,11 @@ func verifyDesiredLayout(raw, ephemeral object) error { return errors.New("talos encryption is not locked to the existing state partition") } } - if value(raw, "spec", "provisioning", "partitionSpec", "minSize") != float64(96*1024*1024*1024) || - value(raw, "spec", "provisioning", "partitionSpec", "maxSize") != float64(96*1024*1024*1024) || + if value(raw, "spec", "provisioning", "partitionSpec", "minSize") != float64(rawGiB<<30) || + value(raw, "spec", "provisioning", "partitionSpec", "maxSize") != float64(rawGiB<<30) || value(raw, "spec", "provisioning", "partitionSpec", "negativeMaxSize") == true || value(ephemeral, "spec", "provisioning", "partitionSpec", "minSize") != float64(40*1024*1024*1024) || - value(ephemeral, "spec", "provisioning", "partitionSpec", "maxSize") != float64(104*1024*1024*1024) || + value(ephemeral, "spec", "provisioning", "partitionSpec", "maxSize") != float64((rawGiB+8)<<30) || value(ephemeral, "spec", "provisioning", "partitionSpec", "negativeMaxSize") != true { return errors.New("talos has not reserved the reviewed raw-disk capacity") } diff --git a/scripts/migrate-longhorn-v2/journal.go b/scripts/migrate-longhorn-v2/journal.go index 9fe197e712..d0d3d9545d 100644 --- a/scripts/migrate-longhorn-v2/journal.go +++ b/scripts/migrate-longhorn-v2/journal.go @@ -25,6 +25,7 @@ type journal struct { NativeEmptyDigest string `json:"nativeEmptyDigest,omitempty"` OriginalBootID string `json:"originalBootID,omitempty"` RawVolumeAbsent bool `json:"rawVolumeAbsent,omitempty"` + StagingLayout bool `json:"stagingLayout,omitempty"` } // validateJournal requires complete original identities and a known procedure @@ -50,7 +51,7 @@ func validateJournal(j journal, target string) error { } return nil } - if j.Schema != 1 || !inventoryTarget.MatchString(target) || j.TemporarySelection != "" || j.NativeEmptyDigest != "" || j.OriginalBootID != "" || j.RawVolumeAbsent { + if j.Schema != 1 || !inventoryTarget.MatchString(target) || j.TemporarySelection != "" || j.NativeEmptyDigest != "" || j.OriginalBootID != "" || j.RawVolumeAbsent || j.StagingLayout { return errors.New("journal schema and preparation scope do not match") } if len(j.OriginalDisks) != 1 { diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index 5ab1e103c9..2e46cba189 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -123,7 +123,7 @@ func main() { os.Exit(1) } } - p := temporaryPreparation{preparation: preparation{c: c, target: resolved, name: "longhorn-v2-prepare-" + target, temporarySelection: target, source: os.Getenv("GITHUB_SHA"), owner: os.Getenv("GITHUB_RUN_ID"), pools: pools}} + p := temporaryPreparation{preparation: preparation{c: c, target: resolved, name: "longhorn-v2-prepare-" + target, temporarySelection: target, source: os.Getenv("GITHUB_SHA"), owner: os.Getenv("GITHUB_RUN_ID"), pools: pools}, staging: true} if err = p.prepare(ctx); err != nil { fmt.Fprintln(os.Stderr, "Unused temporary preparation held:", err) fmt.Fprintln(os.Stderr, "Preserve its journal and owned cordon; uncertain resets are never replayed.") diff --git a/scripts/migrate-longhorn-v2/prepare.go b/scripts/migrate-longhorn-v2/prepare.go index 0d95433444..f2e1775706 100644 --- a/scripts/migrate-longhorn-v2/prepare.go +++ b/scripts/migrate-longhorn-v2/prepare.go @@ -28,15 +28,20 @@ func (p preparation) storageNode(ctx context.Context) (object, error) { } func (p preparation) desiredLayout(ctx context.Context, address string) error { - raw, err := p.c.talosResource(ctx, address, "volumeconfig", "r-longhorn") + raw, ephemeral, err := p.readDesiredLayout(ctx, address) if err != nil { return err } - ephemeral, err := p.c.talosResource(ctx, address, "volumeconfig", "EPHEMERAL") + return verifyDesiredLayout(raw, ephemeral) +} + +func (p preparation) readDesiredLayout(ctx context.Context, address string) (object, object, error) { + raw, err := p.c.talosResource(ctx, address, "volumeconfig", "r-longhorn") if err != nil { - return err + return nil, nil, err } - return verifyDesiredLayout(raw, ephemeral) + ephemeral, err := p.c.talosResource(ctx, address, "volumeconfig", "EPHEMERAL") + return raw, ephemeral, err } func (p *preparation) begin(ctx context.Context) error { diff --git a/scripts/migrate-longhorn-v2/staging-layout.yaml b/scripts/migrate-longhorn-v2/staging-layout.yaml new file mode 100644 index 0000000000..a2b32a35af --- /dev/null +++ b/scripts/migrate-longhorn-v2/staging-layout.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + diskSelector: + match: system_disk + minSize: 40GiB + maxSize: -152GiB + grow: false +--- +apiVersion: v1alpha1 +kind: RawVolumeConfig +name: longhorn +provisioning: + diskSelector: + match: system_disk + minSize: 144GiB + maxSize: 144GiB + grow: false diff --git a/scripts/migrate-longhorn-v2/staging_layout.go b/scripts/migrate-longhorn-v2/staging_layout.go new file mode 100644 index 0000000000..b12684b1ef --- /dev/null +++ b/scripts/migrate-longhorn-v2/staging_layout.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + _ "embed" + "errors" + "time" +) + +const stagingRawDiskGiB = 144 + +//go:embed staging-layout.yaml +var stagingLayoutPatch string + +func verifyStagingBudget(s inventory) error { + return verifyRawDiskBudget("", s, stagingRawDiskGiB) +} + +func verifyStagingLayout(raw, ephemeral object) error { + return verifyRawDiskConfig(raw, ephemeral, stagingRawDiskGiB) +} + +// Only an owned, drained temporary worker with positive native absence can +// request this patch. If a previous attempt already installed it, read it back +// without applying again. Configuration alone does not authorize a reset. +func (p temporaryPreparation) installStagingLayout(ctx context.Context, address string) error { + if !p.j.StagingLayout || p.j.Phase != "draining" { + return errInventory + } + raw, ephemeral, err := p.readDesiredLayout(ctx, address) + if err != nil { + return err + } + if verifyStagingLayout(raw, ephemeral) == nil { + return nil + } + if err = verifyDesiredLayout(raw, ephemeral); err != nil { + return err + } + if err = p.checkOwner(ctx); err != nil { + return err + } + node, currentAddress, err := p.node(ctx, true) + if err != nil || currentAddress != address || text(node, "status", "nodeInfo", "bootID") != p.j.OriginalBootID || + value(node, "spec", "unschedulable") != true || text(node, "metadata", "annotations", fenceAnnotation) != p.name { + return errInventory + } + applyCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if _, err = p.c.run(applyCtx, nil, "talosctl", "--nodes="+address, "patch", "machineconfig", "--mode=no-reboot", "--patch="+stagingLayoutPatch); err != nil { + return errors.New("the temporary worker layout change could not be confirmed") + } + return waitReadyWithin(ctx, time.Minute, func(ctx context.Context) error { + return p.verifyUnusedLayout(ctx, address) + }) +} + +func (p temporaryPreparation) verifyUnusedLayout(ctx context.Context, address string) error { + if !p.staging { + return p.desiredLayout(ctx, address) + } + raw, ephemeral, err := p.readDesiredLayout(ctx, address) + if err != nil { + return err + } + return verifyStagingLayout(raw, ephemeral) +} diff --git a/scripts/migrate-longhorn-v2/staging_layout_test.go b/scripts/migrate-longhorn-v2/staging_layout_test.go new file mode 100644 index 0000000000..44821f476a --- /dev/null +++ b/scripts/migrate-longhorn-v2/staging_layout_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "strconv" + "testing" +) + +func TestStagingBudgetPreservesFullCopiesAndWorkingSpace(t *testing.T) { + s := inventory{} + // The current 20 claims total 117 GiB. Use 40 GiB actual to round up. + for _, size := range []uint64{2, 2, 2, 2, 2, 2, 5, 5, 5, 5, 5, 5, 5, 10, 10, 10, 10, 10, 10, 10} { + s.volumes = append(s.volumes, object{ + "spec": object{"size": strconv.FormatUint(size<<30, 10), "dataEngine": "v1"}, + "status": object{"actualSize": float64(2 << 30)}, + }) + } + if err := verifyRebuildBudget("prod-worker-1", s); err == nil { + t.Fatal("117 GiB fit on the original permanent disk") + } + if err := verifyStagingBudget(s); err != nil { + t.Fatalf("larger staging disk rejected the measured migration footprint: %v", err) + } + // Adding 18 GiB to an existing 2 GiB volume crosses the 134 GiB limit. + s.volumes[0]["spec"].(object)["size"] = strconv.FormatUint(20<<30, 10) + if err := verifyStagingBudget(s); err == nil { + t.Fatal("staging budget spent reserved logical headroom") + } + s.volumes[0]["spec"].(object)["size"] = strconv.FormatUint(2<<30, 10) + s.volumes[0]["status"].(object)["actualSize"] = float64(61 << 30) + if err := verifyStagingBudget(s); err == nil { + t.Fatal("staging budget spent the physical free-space floor") + } +} + +func TestStagingLayoutRequiresTheLargerEncryptedPartition(t *testing.T) { + raw, ephemeral := desiredConfigFixture() + if err := verifyStagingLayout(raw, ephemeral); err == nil { + t.Fatal("original layout accepted as larger staging capacity") + } + rawPartition := valueObject(raw, "spec", "provisioning", "partitionSpec") + rawPartition["minSize"], rawPartition["maxSize"] = float64(144<<30), float64(144<<30) + valueObject(ephemeral, "spec", "provisioning", "partitionSpec")["maxSize"] = float64(152 << 30) + if err := verifyStagingLayout(raw, ephemeral); err != nil { + t.Fatal(err) + } + if err := verifyDesiredLayout(raw, ephemeral); err == nil { + t.Fatal("staging layout accepted for a permanent worker") + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_prepare.go b/scripts/migrate-longhorn-v2/temporary_prepare.go index 0a68994ce7..4468a65efd 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare.go @@ -14,7 +14,8 @@ import ( type temporaryPreparation struct { preparation - reader inventoryReaderFactory + reader inventoryReaderFactory + staging bool } // prepare advances the separately journaled unused-worker profile, recording reset @@ -94,8 +95,12 @@ func (p *temporaryPreparation) prepare(ctx context.Context) error { // hugepage capacity, tolerating only this journal's own cordon during preparation. // Once pinned, every temporary pool member must retain the migration-owned pin. func (p temporaryPreparation) unusedCapacity(s inventory, pinned bool) error { - if err := verifyRebuildBudget(p.target, s); err != nil { - return err + checkBudget := verifyRebuildBudget(p.target, s) + if p.staging { + checkBudget = verifyStagingBudget(s) + } + if checkBudget != nil { + return checkBudget } nodes := s.nodes for _, n := range nodes { @@ -140,6 +145,7 @@ func (p *temporaryPreparation) beginUnused(ctx context.Context) error { if err = p.claimRecord(ctx, journal{}); err != nil { return err } + p.staging = p.j.StagingLayout n, _, nodeErr := p.node(ctx, true) if nodeErr != nil { return nodeErr @@ -176,7 +182,7 @@ func (p *temporaryPreparation) beginUnused(ctx context.Context) error { if err != nil { return err } - p.j = journal{Schema: 2, Target: p.target, Source: p.source, OwnerRun: p.owner, Phase: "evacuating", NodeUID: text(n, "metadata", "uid"), ProviderID: text(n, "spec", "providerID"), MachineUUID: text(n, "status", "nodeInfo", "systemUUID"), LonghornUID: text(lh, "metadata", "uid"), OriginalDisks: object{}, TemporarySelection: p.temporarySelection, OriginalBootID: text(n, "status", "nodeInfo", "bootID")} + p.j = journal{Schema: 2, Target: p.target, Source: p.source, OwnerRun: p.owner, Phase: "evacuating", NodeUID: text(n, "metadata", "uid"), ProviderID: text(n, "spec", "providerID"), MachineUUID: text(n, "status", "nodeInfo", "systemUUID"), LonghornUID: text(lh, "metadata", "uid"), OriginalDisks: object{}, TemporarySelection: p.temporarySelection, OriginalBootID: text(n, "status", "nodeInfo", "bootID"), StagingLayout: p.staging} _, address, err := p.node(ctx, true) if err != nil { return err @@ -229,6 +235,11 @@ func (p temporaryPreparation) observeUnused(ctx context.Context, owned bool) (st if receipt.APIProjectionDigest != before.Digest || !unusedParentAbsent(receipt.Report) || receipt.BlockLayout == nil { return "", errInventory } + // The allocated original partition alone must cover the new 152 GiB + // reservation and 40 GiB root minimum, with 8 GiB for geometry overhead. + if p.staging && receipt.BlockLayout.EphemeralSizeBytes < 200<<30 { + return "", errors.New("native storage cannot fit the reviewed staging layout") + } if err = sourceCurrent(ctx, p.c.run, p.source); err != nil { return "", err } @@ -322,7 +333,7 @@ func (p *temporaryPreparation) resetUnused(ctx context.Context) error { if _, err = p.c.kubectl(ctx, nil, "drain", p.target, "--ignore-daemonsets", "--delete-emptydir-data", "--timeout=40m"); err != nil { return errors.New("temporary worker drain was blocked; no reset was requested") } - for range 2 { + for pass := range 2 { s, err := p.c.snapshot(ctx) if err != nil { return err @@ -336,10 +347,14 @@ func (p *temporaryPreparation) resetUnused(ctx context.Context) error { if err = requireBenchmarkBaseline(ctx, p.c, s.nodes); err != nil { return err } - if err = p.desiredLayout(ctx, address); err != nil { + if _, err = p.observeUnused(ctx, true); err != nil { return err } - if _, err = p.observeUnused(ctx, true); err != nil { + if p.staging && pass == 0 { + if err = p.installStagingLayout(ctx, address); err != nil { + return err + } + } else if err = p.verifyUnusedLayout(ctx, address); err != nil { return err } } @@ -387,7 +402,7 @@ func (p temporaryPreparation) unusedPostState(ctx context.Context) (node object, if err != nil || actualBoot != boot { return nil, errInventory } - if err = p.desiredLayout(ctx, address); err != nil { + if err = p.verifyUnusedLayout(ctx, address); err != nil { return nil, err } raw, err := p.c.talosVolume(ctx, address, "r-longhorn") @@ -401,7 +416,11 @@ func (p temporaryPreparation) unusedPostState(ctx context.Context) (node object, if text(eph, "spec", "uuid") == p.j.EphemeralUUID { return nil, errInventory } - if err = verifyEncryptedLocalLayout(raw, eph); err != nil { + rawGiB := uint64(permanentRawDiskGiB) + if p.staging { + rawGiB = stagingRawDiskGiB + } + if err = verifyEncryptedRawDisk(raw, eph, rawGiB); err != nil { return nil, err } lh, err := p.storageNode(ctx) diff --git a/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go index aeb8f34e62..f0b76077cd 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go @@ -18,12 +18,55 @@ type unusedPreparationFixture struct { base *preparationAPI baseline object nativeCalls int + layoutWrites int + layoutInstalled bool + layoutWriteUncertain bool badNativeAfter int missingBaseline, noRecovery bool afterReset func() dropPinWrites bool } +func TestUnusedStagingPreparationRequiresTheLargerNativeLayout(t *testing.T) { + f, p := unusedPreparationAPI(t) + p.staging = true + if err := p.prepare(context.Background()); err != nil { + t.Fatal(err) + } + if f.layoutWrites != 1 || f.base.resets != 1 || p.j.Phase != "prepared" { + t.Fatal("larger capacity did not use one verified layout change and one guarded reset") + } + if err := p.prepare(context.Background()); err != nil || f.layoutWrites != 1 || f.base.resets != 1 { + t.Fatal("completed staging preparation repeated a mutation", err) + } +} + +func TestUncertainStagingConfigurationRequiresNativeReadbackBeforeReset(t *testing.T) { + f, p := unusedPreparationAPI(t) + p.staging, f.layoutWriteUncertain = true, true + if err := p.prepare(context.Background()); err == nil || f.layoutWrites != 1 || f.base.resets != 0 || p.j.Phase != "draining" { + t.Fatal("an uncertain staging change authorized a reset", err) + } + if err := p.prepare(context.Background()); err != nil || f.layoutWrites != 1 || f.base.resets != 1 || p.j.Phase != "prepared" { + t.Fatal("native staging readback did not recover the guarded reset", err) + } +} + +func TestStagingRejectsInsufficientNativePartitionSizeBeforeMutation(t *testing.T) { + f, p := unusedPreparationAPI(t) + p.staging = true + p.reader = func(context.Context, string) (nativeReplicaReader, func() error, error) { + r := &unusedParentReaderFixture{ + temporaryNativeReader: temporaryNativeReader{boot: text(f.base.node, "status", "nodeInfo", "bootID")}, + ephemeralSize: 190 << 30, + } + return r, func() error { return nil }, nil + } + if err := p.prepare(context.Background()); err == nil || f.base.cm != nil || f.layoutWrites != 0 || f.base.resets != 0 { + t.Fatal("smaller native partition allowed a staging change", err) + } +} + func unusedBaselineFixture(t *testing.T, s inventory) object { t.Helper() machines, err := benchmarkMachines(s, "v1") @@ -151,6 +194,30 @@ func (f *unusedPreparationFixture) run(ctx context.Context, input []byte, progra f.base.t.Fatal("unknown node request") } } + if program == "talosctl" && len(args) > 1 && args[1] == "patch" { + want := []string{"--nodes=10.0.99.7", "patch", "machineconfig", "--mode=no-reboot", "--patch=" + stagingLayoutPatch} + var j journal + if !reflect.DeepEqual(args, want) || json.Unmarshal([]byte(text(f.base.cm, "data", "record")), &j) != nil || + !j.StagingLayout || j.Phase != "draining" || f.nativeCalls < 2 || value(f.base.node, "spec", "unschedulable") != true { + f.base.t.Fatal("staging patch lacked the exact target, journal, cordon and fresh native proof") + } + f.layoutWrites++ + f.layoutInstalled = true + if f.layoutWriteUncertain { + return nil, errors.New("staging configuration accepted but response lost") + } + return nil, nil + } + if f.layoutInstalled && program == "talosctl" && len(args) == 5 && args[2] == "volumeconfig" { + raw, ephemeral := desiredConfigFixture() + rawPartition := valueObject(raw, "spec", "provisioning", "partitionSpec") + rawPartition["minSize"], rawPartition["maxSize"] = float64(144<<30), float64(144<<30) + valueObject(ephemeral, "spec", "provisioning", "partitionSpec")["maxSize"] = float64(152 << 30) + if args[3] == "EPHEMERAL" { + return json.Marshal(ephemeral) + } + return json.Marshal(raw) + } if program == "talosctl" && len(args) > 1 && args[1] == "reset" { want := []string{"--nodes=10.0.99.7", "reset", "--system-labels-to-wipe=EPHEMERAL", "--reboot", "--graceful=true", "--wait", "--timeout=15m"} var j journal @@ -171,12 +238,17 @@ func (f *unusedPreparationFixture) run(ctx context.Context, input []byte, progra return nil, nil } data, err := f.base.run(ctx, input, program, args...) - if err == nil && f.base.postReset && program == "talosctl" && len(args) == 5 && args[2] == "volumestatus" && args[3] == "EPHEMERAL" { + if err == nil && f.base.postReset && program == "talosctl" && len(args) == 5 && args[2] == "volumestatus" && + (args[3] == "EPHEMERAL" || f.layoutInstalled && args[3] == "r-longhorn") { var volume object if json.Unmarshal(data, &volume) != nil { f.base.t.Fatal("malformed fixture volume") } - volume["spec"].(object)["size"] = float64(128 << 30) + size := float64(128 << 30) + if args[3] == "r-longhorn" { + size = float64(144 << 30) + } + volume["spec"].(object)["size"] = size return json.Marshal(volume) } return data, err @@ -199,8 +271,9 @@ func TestUnusedTemporaryPreparationPreservesV1AndLeavesRawStorageFenced(t *testi type unusedParentReaderFixture struct { temporaryNativeReader - rootCalls int - mode string + rootCalls int + mode string + ephemeralSize uint64 } func (*unusedParentReaderFixture) ListReplicas(context.Context) (longhorninventory.Listing, error) { @@ -228,8 +301,12 @@ func (r *unusedParentReaderFixture) ListStorageRoot(context.Context) (longhornin return longhorninventory.Listing{Complete: r.mode != "partial", Entries: entries}, nil } -func (*unusedParentReaderFixture) ListBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) { - return longhorninventory.BlockLayoutReport{VolumeStatuses: 2, DiscoveredVolumes: 1, EphemeralPhase: "ready", RawPresent: true, RawPhase: "waiting", Digest: strings.Repeat("a", 64)}, nil +func (r *unusedParentReaderFixture) ListBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) { + size := r.ephemeralSize + if size == 0 { + size = 270 << 30 + } + return longhorninventory.BlockLayoutReport{VolumeStatuses: 2, DiscoveredVolumes: 1, EphemeralPhase: "ready", EphemeralSizeBytes: size, RawPresent: true, RawPhase: "waiting", Digest: strings.Repeat("a", 64)}, nil } func TestUnusedPreparationQualifiesStableParentAbsenceWithoutAReplicaPath(t *testing.T) { From 66b4357ea9dbe56e7d4324373a29b6f5f0bea8d9 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 9 Nov 2026 23:04:49 +0100 Subject: [PATCH 29/35] fix(storage): rebind main before staging writes --- scripts/migrate-longhorn-v2/staging_layout.go | 6 ++-- .../migrate-longhorn-v2/temporary_prepare.go | 3 ++ .../temporary_prepare_flow_test.go | 32 +++++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/scripts/migrate-longhorn-v2/staging_layout.go b/scripts/migrate-longhorn-v2/staging_layout.go index b12684b1ef..5d2cd6a26c 100644 --- a/scripts/migrate-longhorn-v2/staging_layout.go +++ b/scripts/migrate-longhorn-v2/staging_layout.go @@ -37,14 +37,14 @@ func (p temporaryPreparation) installStagingLayout(ctx context.Context, address if err = verifyDesiredLayout(raw, ephemeral); err != nil { return err } - if err = p.checkOwner(ctx); err != nil { - return err - } node, currentAddress, err := p.node(ctx, true) if err != nil || currentAddress != address || text(node, "status", "nodeInfo", "bootID") != p.j.OriginalBootID || value(node, "spec", "unschedulable") != true || text(node, "metadata", "annotations", fenceAnnotation) != p.name { return errInventory } + if err = p.checkOwner(ctx); err != nil { + return err + } applyCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() if _, err = p.c.run(applyCtx, nil, "talosctl", "--nodes="+address, "patch", "machineconfig", "--mode=no-reboot", "--patch="+stagingLayoutPatch); err != nil { diff --git a/scripts/migrate-longhorn-v2/temporary_prepare.go b/scripts/migrate-longhorn-v2/temporary_prepare.go index 4468a65efd..7c8d3189b1 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare.go @@ -364,6 +364,9 @@ func (p *temporaryPreparation) resetUnused(ctx context.Context) error { if err = p.advance(ctx, "reset-requested"); err != nil { return err } + if err = p.checkOwner(ctx); err != nil { + return err + } _, resetErr := p.c.run(ctx, nil, "talosctl", "--nodes="+address, "reset", "--system-labels-to-wipe=EPHEMERAL", "--reboot", "--graceful=true", "--wait", "--timeout=15m") if resetErr != nil { fmt.Println("Temporary reset response was uncertain; only positive recovery can advance the journal.") diff --git a/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go index f0b76077cd..801ec2d833 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare_flow_test.go @@ -67,6 +67,38 @@ func TestStagingRejectsInsufficientNativePartitionSizeBeforeMutation(t *testing. } } +func TestStagingHoldsIfMainMovesAtEitherMutationBoundary(t *testing.T) { + for _, phase := range []string{"layout apply", "reset request"} { + t.Run(phase, func(t *testing.T) { + f, p := unusedPreparationAPI(t) + p.staging = true + oldLayoutRead := false + p.c.run = func(ctx context.Context, input []byte, program string, args ...string) ([]byte, error) { + if phase == "layout apply" && program == "talosctl" && len(args) == 5 && args[1] == "get" { + if args[2] == "volumeconfig" && args[3] == "EPHEMERAL" && f.nativeCalls == 2 { + oldLayoutRead = true + } + if args[2] == "systeminformation" && oldLayoutRead { + f.base.staleSource = true + } + } + response, err := f.run(ctx, input, program, args...) + if phase == "reset request" && program == "kubectl" && len(args) > 3 && args[2] == "patch" && args[3] == "configmap" { + var j journal + if json.Unmarshal([]byte(text(f.base.cm, "data", "record")), &j) == nil && j.Phase == "reset-requested" { + f.base.staleSource = true + } + } + return response, err + } + err := p.prepare(context.Background()) + if err == nil || !f.base.staleSource || f.base.resets != 0 || phase == "layout apply" && f.layoutWrites != 0 { + t.Fatal("moved main allowed a forward storage mutation", err) + } + }) + } +} + func unusedBaselineFixture(t *testing.T, s inventory) object { t.Helper() machines, err := benchmarkMachines(s, "v1") From 20e87d4c5ccdf999e68ca1b8b8a750346bafad68 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Mon, 9 Nov 2026 23:39:18 +0100 Subject: [PATCH 30/35] fix(storage): protect all current databases during worker reboots --- scripts/migrate-longhorn-v2/README.md | 6 ++ scripts/migrate-longhorn-v2/host_reboot.go | 55 +++++++++++++++---- .../migrate-longhorn-v2/host_reboot_test.go | 54 +++++++++++++++++- 3 files changed, 100 insertions(+), 15 deletions(-) diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 4732332c58..067e877861 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -42,6 +42,12 @@ reboot intent. The single reboot request has no reset or disk-wipe selector. An unfinished or unreadable peer journal, another worker's cordon, or lost proof of a peer's recorded recovery prevents starting the next worker. +New journals bind all five production PostgreSQL clusters and their full declared +replica counts. The two-instance kagent and Substrate databases must both recover +before the reboot, alongside Coroot, Umami and Wedding. Existing journals preserve +their original identities. A legacy journal that has yet to request a reboot must +bind the current database roster before moving the worker. + Recovery never repeats an uncertain reboot. It requires a changed boot with the same node, hardware, OS, encrypted partition, registered disks and database system identities, 1,024 native 2 MiB pages, matching 2 GiB Kubernetes capacity, clear node diff --git a/scripts/migrate-longhorn-v2/host_reboot.go b/scripts/migrate-longhorn-v2/host_reboot.go index 17a7f1666b..14521e7d5a 100644 --- a/scripts/migrate-longhorn-v2/host_reboot.go +++ b/scripts/migrate-longhorn-v2/host_reboot.go @@ -49,6 +49,17 @@ type hostRebootRecord struct { type hostDatabaseIdentity struct{ UID, SystemID string } +var hostDatabases = map[string]struct { + instances float64 + legacy bool +}{ + "observability/coroot-db": {instances: 3, legacy: true}, + "umami/umami-db": {instances: 3, legacy: true}, + "wedding-app/wedding-db": {instances: 3, legacy: true}, + "ate-system/substrate-db": {instances: 2}, + "kagent/kagent-db": {instances: 2}, +} + var hostSystemID = regexp.MustCompile(`^[1-9][0-9]{0,19}$`) var errHostDatabasesRecovering = errors.New("the original database quorum is still recovering after eviction") @@ -63,14 +74,22 @@ func (r hostRebootRecord) identity() journal { // validate rejects incomplete or out-of-profile records before their phase can authorize // an action, including incomplete original storage and database identities. func (r hostRebootRecord) validate(target string) error { - if r.Version != 1 || validateJournal(r.identity(), target) != nil || r.OriginalDiskUUID == "" || !inventoryBoot.MatchString(r.OriginalBootID) || r.OSImage == "" || r.Architecture != "amd64" || len(r.Databases) != 3 { + if (r.Version != 1 && r.Version != 2) || validateJournal(r.identity(), target) != nil || r.OriginalDiskUUID == "" || !inventoryBoot.MatchString(r.OriginalBootID) || r.OSImage == "" || r.Architecture != "amd64" { return errHostMemory } - for _, key := range []string{"observability/coroot-db", "umami/umami-db", "wedding-app/wedding-db"} { + want := 0 + for key, profile := range hostDatabases { + if r.Version == 1 && !profile.legacy { + continue + } + want++ if r.Databases[key].UID == "" || !hostSystemID.MatchString(r.Databases[key].SystemID) { return errHostMemory } } + if len(r.Databases) != want { + return errHostMemory + } switch r.Phase { case "draining", "reboot-requested": if r.RecoveredBootID != "" { @@ -203,28 +222,31 @@ func (h hostMemoryReboot) owned(ctx context.Context, expected hostRebootRecord) // preserves its cluster and system identities across the maintenance operation. func (h hostMemoryReboot) databases(ctx context.Context, expected map[string]hostDatabaseIdentity) (map[string]hostDatabaseIdentity, error) { items, err := h.c.list(ctx, "clusters.postgresql.cnpg.io", "*") - if err != nil || len(items) != 3 { + if err != nil || len(items) != len(hostDatabases) { return nil, errHostMemory } result := map[string]hostDatabaseIdentity{} recovering := false for _, db := range items { key := text(db, "metadata", "namespace") + "/" + text(db, "metadata", "name") - if key != "observability/coroot-db" && key != "umami/umami-db" && key != "wedding-app/wedding-db" || result[key].UID != "" || + profile, known := hostDatabases[key] + if !known || result[key].UID != "" || text(db, "metadata", "uid") == "" || text(db, "metadata", "resourceVersion") == "" || value(db, "metadata", "deletionTimestamp") != nil || - value(db, "spec", "instances") != float64(3) || !hostSystemID.MatchString(text(db, "status", "systemID")) { + value(db, "spec", "instances") != profile.instances || !hostSystemID.MatchString(text(db, "status", "systemID")) { return nil, errHostMemory } result[key] = hostDatabaseIdentity{UID: text(db, "metadata", "uid"), SystemID: text(db, "status", "systemID")} ready := value(db, "status", "readyInstances") phase := text(db, "status", "phase") - if (ready != float64(2) && ready != float64(3)) || phase == "" { + if (ready != profile.instances-1 && ready != profile.instances) || phase == "" { return nil, errHostMemory } - recovering = recovering || ready != float64(3) || phase != "Cluster in healthy state" + recovering = recovering || ready != profile.instances || phase != "Cluster in healthy state" } - if expected != nil && !reflect.DeepEqual(result, expected) { - return nil, errHostMemory + for key, identity := range expected { + if result[key] != identity { + return nil, errHostMemory + } } if recovering { return nil, errHostDatabasesRecovering @@ -345,7 +367,7 @@ func (h hostMemoryReboot) initialize(ctx context.Context) (hostRebootRecord, err return hostRebootRecord{}, err } n, lh := nodes[h.target], storage[h.target] - r := hostRebootRecord{Version: 1, Target: h.target, Source: h.source, OwnerRun: h.owner, Phase: "draining", NodeUID: text(n, "metadata", "uid"), ProviderID: text(n, "spec", "providerID"), + r := hostRebootRecord{Version: 2, Target: h.target, Source: h.source, OwnerRun: h.owner, Phase: "draining", NodeUID: text(n, "metadata", "uid"), ProviderID: text(n, "spec", "providerID"), MachineUUID: text(n, "status", "nodeInfo", "systemUUID"), OriginalBootID: text(n, "status", "nodeInfo", "bootID"), LonghornUID: text(lh, "metadata", "uid"), OSImage: text(n, "status", "nodeInfo", "osImage"), Architecture: text(n, "status", "nodeInfo", "architecture"), Databases: databases, OriginalDiskUUID: hostRegisteredDiskUUID(lh)} r.OriginalDisks, _ = value(lh, "spec", "disks").(object) @@ -394,10 +416,19 @@ func (h hostMemoryReboot) run(ctx context.Context) error { if err != nil { return err } - } else if r.OwnerRun != h.owner || r.Source != h.source { - if previousRunCompleted(ctx, h.c, r.OwnerRun, r.Source) != nil { + } else if r.OwnerRun != h.owner || r.Source != h.source || r.Version == 1 && r.Phase == "draining" { + if (r.OwnerRun != h.owner || r.Source != h.source) && previousRunCompleted(ctx, h.c, r.OwnerRun, r.Source) != nil { return errHostMemory } + // Older pre-reboot records must bind the new databases before any + // forward action. Post-request recovery still observes the old record. + if r.Version == 1 && r.Phase == "draining" { + databases, readErr := h.databases(ctx, r.Databases) + if readErr != nil { + return readErr + } + r.Version, r.Databases = 2, databases + } r.OwnerRun, r.Source = h.owner, h.source if err = h.write(ctx, cm, r); err != nil { return err diff --git a/scripts/migrate-longhorn-v2/host_reboot_test.go b/scripts/migrate-longhorn-v2/host_reboot_test.go index 23f6b4374c..47bb14d407 100644 --- a/scripts/migrate-longhorn-v2/host_reboot_test.go +++ b/scripts/migrate-longhorn-v2/host_reboot_test.go @@ -69,11 +69,17 @@ func hostRebootAPI(t *testing.T) (*hostRebootFixture, hostMemoryReboot) { pod["spec"] = object{"nodeName": "prod-worker-1"} pod["status"] = object{"phase": "Running", "conditions": readyConditions()} b.s.pods = append(b.s.pods, pod) - for i, selected := range []struct{ namespace, name string }{{"observability", "coroot-db"}, {"umami", "umami-db"}, {"wedding-app", "wedding-db"}} { + for i, selected := range []struct { + namespace, name string + instances float64 + }{ + {"observability", "coroot-db", 3}, {"umami", "umami-db", 3}, {"wedding-app", "wedding-db", 3}, + {"ate-system", "substrate-db", 2}, {"kagent", "kagent-db", 2}, + } { db := resource(selected.name) db["metadata"].(object)["namespace"] = selected.namespace - db["spec"] = object{"instances": float64(3)} - db["status"] = object{"readyInstances": float64(3), "systemID": fmt.Sprintf("1234567%d", i), "phase": "Cluster in healthy state"} + db["spec"] = object{"instances": selected.instances} + db["status"] = object{"readyInstances": selected.instances, "systemID": fmt.Sprintf("1234567%d", i), "phase": "Cluster in healthy state"} f.databases = append(f.databases, db) } h := hostMemoryReboot{c: client{context: "admin@prod", run: f.run}, target: "prod-worker-1", source: old.source, owner: old.owner, pools: old.pools} @@ -93,6 +99,48 @@ func hostRebootAPI(t *testing.T) (*hostRebootFixture, hostMemoryReboot) { return f, h } +func TestHostRebootRecordsEveryDeclaredDatabase(t *testing.T) { + f, h := hostRebootAPI(t) + r, err := h.initialize(context.Background()) + if err != nil || r.Version != 2 || len(r.Databases) != 5 { + t.Fatalf("current database roster was not bound: version=%d databases=%d err=%v", r.Version, len(r.Databases), err) + } + f.databases[4]["status"].(object)["readyInstances"] = float64(1) + if _, err = h.databases(context.Background(), r.Databases); !errors.Is(err, errHostDatabasesRecovering) { + t.Fatal("a recovering two-instance cluster cleared the reboot gate", err) + } + f.databases[4]["status"].(object)["readyInstances"] = float64(2) + f.databases[4]["status"].(object)["systemID"] = "90000000" + if _, err = h.databases(context.Background(), r.Databases); err == nil { + t.Fatal("new cluster identity was not protected") + } +} + +func TestLegacyPreRebootJournalBindsNewDatabasesBeforeMovingTheWorker(t *testing.T) { + f, h := hostRebootAPI(t) + r, err := h.initialize(context.Background()) + if err != nil { + t.Fatal(err) + } + r.Version = 1 + delete(r.Databases, "ate-system/substrate-db") + delete(r.Databases, "kagent/kagent-db") + encoded, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + valueObject(f.base.cm, "data")["record"] = string(encoded) + f.afterDrain = func() { + var current hostRebootRecord + if json.Unmarshal([]byte(text(f.base.cm, "data", "record")), ¤t) != nil || current.Version != 2 || len(current.Databases) != 5 { + t.Fatal("the worker drained before the journal bound the current database roster") + } + } + if err := h.run(context.Background()); err != nil || f.reboots != 1 { + t.Fatal("the qualified legacy journal could not resume", err) + } +} + func (f *hostRebootFixture) run(ctx context.Context, input []byte, program string, args ...string) ([]byte, error) { f.base.t.Helper() if program == "talosctl" && len(args) >= 2 && args[1] == "reboot" { From 8866386417034b42ab41f25d5f6a52ec2deb5a3a Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 10 Nov 2026 01:18:05 +0100 Subject: [PATCH 31/35] fix(storage): qualify the larger staging disks before activation --- scripts/migrate-longhorn-v2/README.md | 3 +++ .../temporary_activation.go | 2 +- .../temporary_activation_flow_test.go | 24 +++++++++++++++---- .../temporary_activation_prepared.go | 4 ++-- .../temporary_activation_prepared_test.go | 5 +++- .../temporary_activation_run.go | 3 +++ .../temporary_activation_test.go | 3 ++- 7 files changed, 34 insertions(+), 10 deletions(-) diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index a959b3e620..6170c8eb43 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -92,6 +92,9 @@ raw-layout preparation, select `register-temporary-v2-storage` and confirm fixed temporary pool members. It requires intact original V1 replicas, healthy database quorums, the cleaned baseline, reviewed storage capacity, no competing maintenance, and ready V2 clients on all six workers before its first write. +Every selected worker must present its own 144 GiB encrypted raw partition. The +complete volume inventory must fit on each disk with the working reserves and +free-space margin available. The journal `longhorn-v2-temporary-activation` binds each temporary selection to the node, provider, hardware, boot, encrypted EPHEMERAL and raw GPT partition diff --git a/scripts/migrate-longhorn-v2/temporary_activation.go b/scripts/migrate-longhorn-v2/temporary_activation.go index 1a0386e142..64306fa95c 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation.go +++ b/scripts/migrate-longhorn-v2/temporary_activation.go @@ -112,7 +112,7 @@ func projectTemporaryV2Binding(selection string, node, storage, raw, ephemeral o BootID: text(node, "status", "nodeInfo", "bootID"), OSImage: text(node, "status", "nodeInfo", "osImage"), LonghornUID: text(storage, "metadata", "uid"), EphemeralUUID: text(ephemeral, "spec", "uuid"), RawPartitionUUID: text(raw, "spec", "partitionUUID")} if !inventoryTemporarySelection.MatchString(selection) || !strings.HasPrefix(selection, "temporary-"+pool.role+"-") || - !inventoryTemporaryNode.MatchString(b.Name) || verifyMigrationNode(node, pool) != nil || verifyEncryptedLocalLayout(raw, ephemeral) != nil || + !inventoryTemporaryNode.MatchString(b.Name) || verifyMigrationNode(node, pool) != nil || verifyEncryptedRawDisk(raw, ephemeral, stagingRawDiskGiB) != nil || b.NodeUID == "" || !regexp.MustCompile(`^hcloud://[1-9][0-9]*$`).MatchString(b.ProviderID) || b.MachineUUID == "" || b.OSImage == "" || !inventoryBoot.MatchString(b.BootID) || nativeBoot != b.BootID || !inventoryBoot.MatchString(strings.ToLower(b.RawPartitionUUID)) || b.RawPartitionUUID == b.EphemeralUUID || text(node, "status", "nodeInfo", "architecture") != "amd64" || !hugepagesReady(node) || text(node, "status", "allocatable", "cpu") != "8" || diff --git a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go index bdaa0b14c6..479e9e9dde 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go @@ -50,13 +50,18 @@ func newActivationAPI(t *testing.T) (*activationAPI, temporaryV2Activation) { } } f := &activationAPI{t: t, s: s, managers: managers, baseline: unusedBaselineFixture(t, s)} - for _, selected := range []struct{ namespace, name string }{{"observability", "coroot-db"}, {"umami", "umami-db"}, {"wedding-app", "wedding-db"}} { - db := resource(selected.name) - db["metadata"].(object)["namespace"] = selected.namespace - db["spec"] = object{"instances": float64(3)} - db["status"] = object{"readyInstances": float64(3), "systemID": "123456789", "phase": "Cluster in healthy state"} + for key, profile := range hostDatabases { + namespace, name, _ := strings.Cut(key, "/") + db := resource(name) + db["metadata"].(object)["namespace"] = namespace + db["spec"] = object{"instances": profile.instances} + db["status"] = object{"readyInstances": profile.instances, "systemID": "123456789", "phase": "Cluster in healthy state"} f.databases = append(f.databases, db) } + for _, volume := range s.volumes { + valueObject(volume, "spec")["size"] = "1073741824" + valueObject(volume, "status")["actualSize"] = float64(1 << 20) + } pools, err := configuredMigrationPools(poolConfigFixture()) if err != nil { t.Fatal(err) @@ -130,6 +135,7 @@ func (f *activationAPI) run(_ context.Context, input []byte, program string, arg return json.Marshal(object{"metadata": object{"id": "systeminformation", "namespace": "hardware"}, "spec": object{"uuid": text(node, "status", "nodeInfo", "systemUUID")}}) } raw, eph := rawFixture() + valueObject(raw, "spec")["size"] = float64(144 << 30) raw["spec"].(object)["partitionUUID"] = fmt.Sprintf("12345678-1234-4234-8234-%012x", i+100) eph["spec"].(object)["uuid"] = fmt.Sprintf("ephemeral-%d", i) if args[2] == "volumestatus" && args[3] == "r-longhorn" { @@ -297,6 +303,14 @@ func TestTemporaryActivationHoldsBeforeWritesWithoutSuccessfulPreparation(t *tes } } +func TestTemporaryActivationHoldsWhenTheDatasetOutgrowsStaging(t *testing.T) { + f, a := newActivationAPI(t) + valueObject(f.s.volumes[0], "spec")["size"] = fmt.Sprint(135 << 30) + if err := a.run(t.Context()); err == nil || f.writes != 0 || f.record != nil { + t.Fatal("activation accepted data beyond the per-worker storage budget", err) + } +} + // Only a later owner may resume an uncertain owned patch; an unrelated storage // owner's annotation cannot be adopted even when the desired disk matches. func TestTemporaryActivationResumesItsUncertainWriteWithoutAdoptingForeignStorage(t *testing.T) { diff --git a/scripts/migrate-longhorn-v2/temporary_activation_prepared.go b/scripts/migrate-longhorn-v2/temporary_activation_prepared.go index 2bd8c54aa8..62b2b3fbb2 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_prepared.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_prepared.go @@ -17,13 +17,13 @@ func preparedTemporaryBinding(cm object, binding temporaryV2Binding) (journal, e return journal{}, errTemporaryActivation } data := []byte(text(cm, "data", "record")) - fields := map[string][]string{"": {"schema", "target", "source", "ownerRun", "phase", "nodeUID", "providerID", "machineUUID", "longhornUID", "ephemeralUUID", "originalDisks", "temporarySelection", "nativeEmptyDigest", "originalBootID", "rawVolumeAbsent"}} + fields := map[string][]string{"": {"schema", "target", "source", "ownerRun", "phase", "nodeUID", "providerID", "machineUUID", "longhornUID", "ephemeralUUID", "originalDisks", "temporarySelection", "nativeEmptyDigest", "originalBootID", "rawVolumeAbsent", "stagingLayout"}} var record journal if len(data) == 0 || len(data) > 32<<10 || strictjson.Validate(data, fields) != nil || json.Unmarshal(data, &record) != nil { return journal{}, errTemporaryActivation } p := preparation{target: binding.Name, name: name, temporarySelection: binding.Selection} - if p.validateRecord(record) != nil || record.Schema != 2 || record.Phase != "prepared" || record.NodeUID != binding.NodeUID || + if p.validateRecord(record) != nil || record.Schema != 2 || !record.StagingLayout || record.Phase != "prepared" || record.NodeUID != binding.NodeUID || record.ProviderID != binding.ProviderID || record.MachineUUID != binding.MachineUUID || record.LonghornUID != binding.LonghornUID || record.OriginalBootID == binding.BootID || strings.EqualFold(record.EphemeralUUID, binding.EphemeralUUID) { return journal{}, errTemporaryActivation diff --git a/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go b/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go index ee9826deb7..0ed721c590 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go @@ -13,7 +13,7 @@ func temporaryPreparedFixture(t *testing.T, binding temporaryV2Binding) object { j := journal{Schema: 2, Target: binding.Name, Source: strings.Repeat("1", 40), OwnerRun: "7777", Phase: "prepared", NodeUID: binding.NodeUID, ProviderID: binding.ProviderID, MachineUUID: binding.MachineUUID, LonghornUID: binding.LonghornUID, EphemeralUUID: "original-ephemeral", OriginalDisks: object{}, TemporarySelection: binding.Selection, - NativeEmptyDigest: strings.Repeat("c", 64), OriginalBootID: "12345678-1234-4234-8234-000000000099", RawVolumeAbsent: true} + NativeEmptyDigest: strings.Repeat("c", 64), OriginalBootID: "12345678-1234-4234-8234-000000000099", RawVolumeAbsent: true, StagingLayout: true} data, err := json.Marshal(j) if err != nil { t.Fatal(err) @@ -43,6 +43,9 @@ func TestTemporaryActivationRequiresTheCompletedPreparationIdentity(t *testing.T "unfinished preparation": func(cm object) { cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), `"phase":"prepared"`, `"phase":"reset-requested"`, 1) }, + "old smaller preparation": func(cm object) { + cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), `,"stagingLayout":true`, "", 1) + }, "replaced worker": func(cm object) { cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), binding.NodeUID, "other-node", 1) }, diff --git a/scripts/migrate-longhorn-v2/temporary_activation_run.go b/scripts/migrate-longhorn-v2/temporary_activation_run.go index 7057060b58..8e318a3abb 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_run.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_run.go @@ -72,6 +72,9 @@ func (a temporaryV2Activation) environment(ctx context.Context) (inventory, erro if err != nil || verifyMigrationCapacity(s.nodes, a.pools) != nil || verifyRedundancy(s) != nil { return inventory{}, errTemporaryActivation } + if err = verifyStagingBudget(s); err != nil { + return inventory{}, err + } managers, err := a.c.list(ctx, "instancemanagers.longhorn.io", "longhorn-system") if err != nil || verifyV2Clients(s, managers, a.pools, v2ManagerImage) != nil { return inventory{}, errTemporaryActivation diff --git a/scripts/migrate-longhorn-v2/temporary_activation_test.go b/scripts/migrate-longhorn-v2/temporary_activation_test.go index 098c465506..df8eb1b467 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_test.go @@ -14,6 +14,7 @@ func temporaryRawBindingFixture() (object, object, object, object, migrationPool lh := resource(text(n, "metadata", "name")) lh["metadata"].(object)["namespace"] = "longhorn-system" raw, eph := rawFixture() + valueObject(raw, "spec")["size"] = float64(144 << 30) raw["spec"].(object)["partitionUUID"] = "12345678-1234-4234-8234-000000000002" return n, lh, raw, eph, migrationPool{name: "autoscale-longhorn-v1", role: "v1", serverType: "cpx42", count: 2} } @@ -31,7 +32,7 @@ func TestTemporaryRawRegistrationBindsTheNativePartitionsAndPoolMember(t *testin "missing raw identity": func(_, _ object, raw, _ object) { delete(raw["spec"].(object), "partitionUUID") }, "raw and filesystem identity reused": func(_, _ object, raw, eph object) { raw["spec"].(object)["partitionUUID"] = text(eph, "spec", "uuid") }, "unencrypted raw disk": func(_, _ object, raw, _ object) { raw["spec"].(object)["encryptionProvider"] = "" }, - "wrong raw capacity": func(_, _ object, raw, _ object) { raw["spec"].(object)["size"] = float64(95 << 30) }, + "original smaller raw capacity": func(_, _ object, raw, _ object) { raw["spec"].(object)["size"] = float64(96 << 30) }, "cordoned member": func(n, _, _, _ object) { n["spec"].(object)["unschedulable"] = true }, "unowned capacity": func(n, _, _, _ object) { delete(n["metadata"].(object)["annotations"].(object), capacityOwnerAnnotation) From d29475da7877c010fbafefe8d61badba83e0c428 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 10 Nov 2026 09:55:56 +0100 Subject: [PATCH 32/35] fix(storage): verify the retained Longhorn partition extent --- internal/longhorninventory/block_layout.go | 43 ++++++++++++- .../longhorninventory/block_layout_test.go | 61 +++++++++++++++++++ .../storage_root_inventory.go | 28 +++++++-- .../storage_root_inventory_test.go | 40 +++++++++--- 4 files changed, 157 insertions(+), 15 deletions(-) diff --git a/internal/longhorninventory/block_layout.go b/internal/longhorninventory/block_layout.go index 6104e746d3..240c515d4d 100644 --- a/internal/longhorninventory/block_layout.go +++ b/internal/longhorninventory/block_layout.go @@ -6,6 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "math" "slices" "strings" @@ -25,12 +26,23 @@ type BlockLayoutReport struct { RawSizeBytes uint64 RawLUKS2, RawLockedToState, RawNodeIDOnly bool RawDiscoveredPartitions, OtherUserVolumes int + RawExtentObserved bool `json:",omitempty"` Digest string } // ListBlockLayout reads the block layout twice and reports it only when both // complete reads agree. func (n Native) ListBlockLayout(ctx context.Context) (BlockLayoutReport, error) { + return n.listBlockLayout(ctx, false) +} + +// ListRetainedBlockLayout additionally joins the configured r-longhorn status +// to its physical partition and compares the byte extent in both reads. +func (n Native) ListRetainedBlockLayout(ctx context.Context) (BlockLayoutReport, error) { + return n.listBlockLayout(ctx, true) +} + +func (n Native) listBlockLayout(ctx context.Context, retainRaw bool) (BlockLayoutReport, error) { ctx, cancel, err := n.requestContext(ctx) if err != nil { return BlockLayoutReport{}, nativeFailure{code: "request"} @@ -39,11 +51,11 @@ func (n Native) ListBlockLayout(ctx context.Context) (BlockLayoutReport, error) if n.Blocks == nil { return BlockLayoutReport{}, nativeFailure{code: "block-client"} } - first, _, err := n.readBlockLayout(ctx) + first, _, err := n.readBlockLayout(ctx, retainRaw) if err != nil { return BlockLayoutReport{}, err } - second, report, err := n.readBlockLayout(ctx) + second, report, err := n.readBlockLayout(ctx, retainRaw) if err != nil { return BlockLayoutReport{}, err } @@ -62,7 +74,7 @@ func (n Native) ListBlockLayout(ctx context.Context) (BlockLayoutReport, error) // Selected layout metadata is compared privately. The public commitment covers // only reported counters and properties, without private device identities, // key material, configuration contents, filesystem contents or provider errors. -func (n Native) readBlockLayout(ctx context.Context) ([]byte, BlockLayoutReport, error) { +func (n Native) readBlockLayout(ctx context.Context, retainRaw bool) ([]byte, BlockLayoutReport, error) { if !active(ctx) { return nil, BlockLayoutReport{}, nativeFailure{code: "block-list"} } @@ -92,6 +104,7 @@ func (n Native) readBlockLayout(ctx context.Context) ([]byte, BlockLayoutReport, } seen := map[string]bool{} ephFound := false + var rawStatus *block.VolumeStatusSpec for _, item := range statuses.Items { v, ok := item.(*block.VolumeStatus) if !ok || v == nil || v.Metadata().Namespace() != block.NamespaceName || v.Metadata().Phase() != resource.PhaseRunning || v.Metadata().ID() == "" || len(v.Metadata().ID()) > 256 || seen[v.Metadata().ID()] { @@ -131,6 +144,7 @@ func (n Native) readBlockLayout(ctx context.Context) ([]byte, BlockLayoutReport, report.EphemeralLockedToState = s.EncryptionLockedToState report.EphemeralNodeIDOnly = nodeIDOnly case "r-longhorn": + rawStatus = s report.RawPresent = true report.RawPhase = phase report.RawSizeBytes = s.Size @@ -153,6 +167,7 @@ func (n Native) readBlockLayout(ctx context.Context) ([]byte, BlockLayoutReport, return nil, BlockLayoutReport{}, invalid } seen = map[string]bool{} + var rawPartition *block.DiscoveredVolumeSpec for _, item := range discovered.Items { v, ok := item.(*block.DiscoveredVolume) if !ok || v == nil || v.Metadata().Namespace() != block.NamespaceName || v.Metadata().Phase() != resource.PhaseRunning || v.Metadata().ID() == "" || len(v.Metadata().ID()) > 256 || seen[v.Metadata().ID()] { @@ -165,6 +180,7 @@ func (n Native) readBlockLayout(ctx context.Context) ([]byte, BlockLayoutReport, } if s.PartitionLabel == "r-longhorn" { report.RawDiscoveredPartitions++ + rawPartition = s } if !appendRow(struct { Kind, ID string @@ -173,6 +189,17 @@ func (n Native) readBlockLayout(ctx context.Context) ([]byte, BlockLayoutReport, return nil, BlockLayoutReport{}, invalid } } + if retainRaw { + if report.RawDiscoveredPartitions != 1 || !report.RawLUKS2 || !report.RawLockedToState || + !report.RawNodeIDOnly || !retainedRawMatches(rawStatus, rawPartition) || + !appendRow(struct { + Kind, FilesystemUUID string + Offset uint64 + }{"retained-extent", rawPartition.UUID, rawPartition.Offset}) { + return nil, BlockLayoutReport{}, invalid + } + report.RawExtentObserved = true + } slices.SortFunc(rows, bytes.Compare) encoded, err := json.Marshal(rows) if err != nil || len(encoded) > MaxMetadataBytes || !active(ctx) { @@ -180,3 +207,13 @@ func (n Native) readBlockLayout(ctx context.Context) ([]byte, BlockLayoutReport, } return encoded, report, nil } + +func retainedRawMatches(status *block.VolumeStatusSpec, partition *block.DiscoveredVolumeSpec) bool { + return status != nil && partition != nil && status.Phase == block.VolumePhaseReady && + status.Type == block.VolumeTypePartition && partition.Type == "partition" && partition.Name == "luks" && partition.UUID != "" && + status.Location == partition.DevPath && status.ParentLocation != "" && status.ParentLocation == partition.ParentDevPath && + status.PartitionUUID != "" && status.PartitionUUID == partition.PartitionUUID && + status.PartitionIndex > 0 && uint(status.PartitionIndex) == partition.PartitionIndex && + status.Size == 96<<30 && status.Size == partition.Size && + partition.Offset > 0 && partition.Offset%512 == 0 && partition.Offset <= math.MaxUint64-partition.Size +} diff --git a/internal/longhorninventory/block_layout_test.go b/internal/longhorninventory/block_layout_test.go index 45baa3a8ad..bc8ce289ea 100644 --- a/internal/longhorninventory/block_layout_test.go +++ b/internal/longhorninventory/block_layout_test.go @@ -20,6 +20,7 @@ type blockLayoutFixture struct { calls int mode string rawPhase *block.VolumePhase + retained bool } func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, options ...state.ListOption) (resource.List, error) { @@ -45,6 +46,14 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio if f.mode == "changed" && f.calls > 2 { raw.TypedSpec().Phase = block.VolumePhaseReady } + if f.retained { + *raw.TypedSpec() = block.VolumeStatusSpec{ + Phase: block.VolumePhaseReady, Type: block.VolumeTypePartition, Location: "/dev/sda3", + ParentLocation: "/dev/sda", PartitionUUID: "private-raw-uuid-canary", PartitionIndex: 3, + Size: 96 << 30, EncryptionProvider: block.EncryptionProviderLUKS2, + EncryptionLockedToState: true, ConfiguredEncryptionKeys: []string{"nodeID"}, + } + } result.Items = []resource.Resource{eph, raw} if f.mode == "duplicate" { result.Items = append(result.Items, eph) @@ -62,6 +71,37 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio v := block.NewDiscoveredVolume(block.NamespaceName, "private-device-canary") *v.TypedSpec() = block.DiscoveredVolumeSpec{DevPath: "/dev/sda6", ParentDevPath: "/dev/sda", PartitionLabel: "EPHEMERAL", PartitionUUID: "private-partition-canary", Size: 150 << 30} result.Items = []resource.Resource{v} + if f.retained { + raw := block.NewDiscoveredVolume(block.NamespaceName, "sda3") + *raw.TypedSpec() = block.DiscoveredVolumeSpec{ + Type: "partition", DevPath: "/dev/sda3", ParentDevPath: "/dev/sda", PartitionLabel: "r-longhorn", + Name: "luks", UUID: "private-luks-canary", + PartitionUUID: "private-raw-uuid-canary", PartitionIndex: 3, Size: 96 << 30, Offset: 1 << 30, + } + switch f.mode { + case "offset missing": + raw.TypedSpec().Offset = 0 + case "offset changed": + if f.calls > 2 { + raw.TypedSpec().Offset += 512 + } + case "wrong identity": + raw.TypedSpec().PartitionUUID = "other" + case "wrong parent": + raw.TypedSpec().ParentDevPath = "/dev/sdb" + case "wrong size": + raw.TypedSpec().Size = 144 << 30 + case "LUKS header missing": + raw.TypedSpec().UUID = "" + case "LUKS header changed": + if f.calls > 2 { + raw.TypedSpec().UUID = "other" + } + case "duplicate raw label": + v.TypedSpec().PartitionLabel = "r-longhorn" + } + result.Items = append(result.Items, raw) + } } if f.mode == "partial" && f.calls == 2 { return result, errors.New("private-provider-canary") @@ -69,6 +109,27 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio return result, nil } +func TestNativeRetainedBlockLayoutRequiresStablePhysicalExtent(t *testing.T) { + for _, mode := range []string{"stable", "offset missing", "offset changed", "wrong identity", "wrong parent", "wrong size", "LUKS header missing", "LUKS header changed", "duplicate raw label"} { + t.Run(mode, func(t *testing.T) { + f := &blockLayoutFixture{t: t, mode: mode, retained: true} + report, err := (Native{API: fileAPI{}, Target: "10.0.0.8", Blocks: f}).ListRetainedBlockLayout(context.Background()) + if mode != "stable" { + if err == nil || report != (BlockLayoutReport{}) || strings.Contains(err.Error(), "canary") { + t.Fatal("unverified physical extent retained success or leaked identity") + } + return + } + encoded, marshalErr := json.Marshal(report) + if err != nil || marshalErr != nil || !report.RawExtentObserved || f.calls != 4 || + strings.Contains(string(encoded), "canary") || strings.Contains(string(encoded), "/dev/") || + strings.Contains(string(encoded), "1073741824") { + t.Fatalf("physical extent observation was incomplete or exposed private geometry: %v", err) + } + }) + } +} + func TestNativeBlockLayoutReportsObservedStateWithoutPrivateIdentities(t *testing.T) { for _, mode := range []string{"stable", "partial", "changed", "duplicate", "missing ephemeral", "tearing down", "canceled"} { t.Run(mode, func(t *testing.T) { diff --git a/scripts/migrate-longhorn-v2/storage_root_inventory.go b/scripts/migrate-longhorn-v2/storage_root_inventory.go index 2dddd83f78..c81be5ed5b 100644 --- a/scripts/migrate-longhorn-v2/storage_root_inventory.go +++ b/scripts/migrate-longhorn-v2/storage_root_inventory.go @@ -16,6 +16,9 @@ type nativeStorageRootReader interface { type nativeBlockLayoutReader interface { ListBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) } +type nativeRetainedBlockLayoutReader interface { + ListRetainedBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) +} type storageRootReceipt struct { Schema int `json:"schema"` ObservationOnly bool `json:"observationOnly"` @@ -80,11 +83,7 @@ func observeStorageRootScope(ctx context.Context, c client, target string, facto } var blockLayout *longhorninventory.BlockLayoutReport if includeBlocks { - blockReader, ok := reader.(nativeBlockLayoutReader) - if !ok { - return storageRootReceipt{}, stageClient - } - layout, layoutErr := blockReader.ListBlockLayout(ctx) + layout, layoutErr := observeNativeBlockLayout(ctx, reader, target) if layoutErr != nil { return storageRootReceipt{}, errors.Join(stageBlockLayout, layoutErr) } @@ -128,6 +127,25 @@ func observeStorageRootScope(ctx context.Context, c client, target string, facto return storageRootReceipt{Schema: schema, ObservationOnly: true, ResetClearance: false, APIProjectionDigest: before.Digest, Report: report, BlockLayout: blockLayout}, nil } +func observeNativeBlockLayout(ctx context.Context, reader nativeReplicaReader, target string) (longhorninventory.BlockLayoutReport, error) { + if target == "temporary-v2-1" { + retainedReader, ok := reader.(nativeRetainedBlockLayoutReader) + if !ok { + return longhorninventory.BlockLayoutReport{}, stageClient + } + layout, err := retainedReader.ListRetainedBlockLayout(ctx) + if err == nil && !layout.RawExtentObserved { + return longhorninventory.BlockLayoutReport{}, stageBlockLayout + } + return layout, err + } + blockReader, ok := reader.(nativeBlockLayoutReader) + if !ok { + return longhorninventory.BlockLayoutReport{}, stageClient + } + return blockReader.ListBlockLayout(ctx) +} + func authorizeStorageRoot(getenv func(string) string, target string) error { if (!inventoryTarget.MatchString(target) && !inventoryTemporarySelection.MatchString(target)) || getenv("LONGHORN_V2_NODE") != target || getenv("LONGHORN_V2_OPERATION") != "inventory-storage-root" { return errors.New("the fixed storage-root observation selection is required") diff --git a/scripts/migrate-longhorn-v2/storage_root_inventory_test.go b/scripts/migrate-longhorn-v2/storage_root_inventory_test.go index eaba358391..61543038dd 100644 --- a/scripts/migrate-longhorn-v2/storage_root_inventory_test.go +++ b/scripts/migrate-longhorn-v2/storage_root_inventory_test.go @@ -17,6 +17,8 @@ type storageRootReaderFixture struct { change, partial, reboot bool blockCalls int blockError bool + retainedCalls int + retainedObserved bool } func (r *storageRootReaderFixture) ListBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) { @@ -27,6 +29,14 @@ func (r *storageRootReaderFixture) ListBlockLayout(context.Context) (longhorninv return longhorninventory.BlockLayoutReport{VolumeStatuses: 2, DiscoveredVolumes: 1, EphemeralPhase: "ready", RawPhase: "waiting", Digest: strings.Repeat("a", 64)}, nil } +func (r *storageRootReaderFixture) ListRetainedBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) { + r.retainedCalls++ + return longhorninventory.BlockLayoutReport{ + VolumeStatuses: 2, DiscoveredVolumes: 2, EphemeralPhase: "ready", RawPhase: "ready", + RawExtentObserved: r.retainedObserved, Digest: strings.Repeat("a", 64), + }, nil +} + func (r *storageRootReaderFixture) ListStorageRoot(context.Context) (longhorninventory.Listing, error) { r.calls++ entries := []longhorninventory.Entry{{Name: ".", Mode: uint32(os.ModeDir), IsDir: true}, {Name: "engine-binaries", Mode: uint32(os.ModeDir), IsDir: true}, {Name: "private-parent-canary", Mode: uint32(os.ModeDir), IsDir: true}} @@ -44,9 +54,14 @@ func (r *storageRootReaderFixture) BootID(context.Context) (string, error) { } func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t *testing.T) { - for _, scenario := range []string{"complete", "partial", "root changed", "peer changed", "boot changed", "close failed", "layout complete", "layout read failed"} { + for _, scenario := range []string{"complete", "partial", "root changed", "peer changed", "boot changed", "close failed", "layout complete", "layout read failed", "layout retained", "layout retained missing proof"} { t.Run(scenario, func(t *testing.T) { nodes := temporaryInventoryRoster() + target, selected := "temporary-v1-1", nodes[0] + retained := strings.HasPrefix(scenario, "layout retained") + if retained { + target, selected = "temporary-v2-1", nodes[2] + } pools, err := configuredMigrationPools(poolConfigFixture()) if err != nil { t.Fatal(err) @@ -57,7 +72,7 @@ func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t t.Fatal("parent observer escaped fixed read scope") } if program == "talosctl" { - return json.Marshal(object{"metadata": object{"id": "systeminformation", "namespace": "hardware"}, "spec": object{"uuid": text(nodes[0], "status", "nodeInfo", "systemUUID")}}) + return json.Marshal(object{"metadata": object{"id": "systeminformation", "namespace": "hardware"}, "spec": object{"uuid": text(selected, "status", "nodeInfo", "systemUUID")}}) } api, kind := "v1", "NodeList" items := []any{} @@ -73,7 +88,7 @@ func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t case "--raw=" + inventoryStorageNodes: api = "longhorn.io/v1beta2" _, lh := temporaryInventoryFixture() - lh["metadata"].(object)["name"] = text(nodes[0], "metadata", "name") + lh["metadata"].(object)["name"] = text(selected, "metadata", "name") items = []any{lh} case "--raw=" + inventoryReplicas: api, kind = "longhorn.io/v1beta2", "ReplicaList" @@ -82,14 +97,18 @@ func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t } return json.Marshal(object{"apiVersion": api, "kind": kind, "metadata": object{"resourceVersion": "1"}, "items": items}) }} - reader := &storageRootReaderFixture{temporaryNativeReader: temporaryNativeReader{boot: text(nodes[0], "status", "nodeInfo", "bootID")}, change: scenario == "root changed", partial: scenario == "partial", reboot: scenario == "boot changed", blockError: scenario == "layout read failed"} + reader := &storageRootReaderFixture{ + temporaryNativeReader: temporaryNativeReader{boot: text(selected, "status", "nodeInfo", "bootID")}, + change: scenario == "root changed", partial: scenario == "partial", reboot: scenario == "boot changed", + blockError: scenario == "layout read failed", retainedObserved: scenario == "layout retained", + } closed := false observe := observeStorageRoot if strings.HasPrefix(scenario, "layout ") { observe = observeStorageLayout } - receipt, err := observe(context.Background(), c, "temporary-v1-1", func(_ context.Context, address string) (nativeReplicaReader, func() error, error) { - projection, _, projectionErr := c.inventoryProjection(context.Background(), "temporary-v1-1") + receipt, err := observe(context.Background(), c, target, func(_ context.Context, address string) (nativeReplicaReader, func() error, error) { + projection, _, projectionErr := c.inventoryProjection(context.Background(), target) if projectionErr != nil || address != projection.Address { t.Fatal("parent observer selected another endpoint") } @@ -101,9 +120,12 @@ func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t return nil }, nil }) - if !closed || (err == nil) != (scenario == "complete" || scenario == "layout complete") { + if !closed || (err == nil) != (scenario == "complete" || scenario == "layout complete" || scenario == "layout retained") { t.Fatal("incomplete or changed parent metadata was accepted:", err) } + if retained != (reader.retainedCalls == 1) || (retained && reader.blockCalls != 0) { + t.Fatal("retained extent observation escaped its fixed worker") + } encoded, _ := json.Marshal(receipt) if strings.Contains(string(encoded), "private-parent-canary") || strings.Contains(string(encoded), "private-close-canary") { t.Fatal("private names reached the receipt") @@ -118,6 +140,10 @@ func TestRootObserverBracketsNativeParentMetadataAndNeverGrantsResetClearance(t if reader.blockCalls != 1 || receipt.Schema != 2 || receipt.BlockLayout == nil || receipt.BlockLayout.RawPhase != "waiting" { t.Fatal("layout mode omitted native block facts") } + } else if retained { + if receipt.Schema != 2 || receipt.BlockLayout == nil || !receipt.BlockLayout.RawExtentObserved { + t.Fatal("retained layout omitted its native extent proof") + } } else if reader.blockCalls != 0 || receipt.Schema != 1 || receipt.BlockLayout != nil { t.Fatal("legacy root mode expanded its read scope") } From a23212da057009aae687b331313649d5bd9f5a2b Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 10 Nov 2026 10:14:21 +0100 Subject: [PATCH 33/35] fix(storage): bind V2 activation to the preserved staging layout --- internal/longhornmigration/activation.go | 14 ++--- internal/longhornmigration/pools.go | 16 ++++- internal/longhornmigration/pools_test.go | 35 +++++++++++ .../migration_capacity.go | 2 +- .../migration_capacity_test.go | 28 ++++++++- scripts/migrate-longhorn-v2/README.md | 19 +++--- scripts/migrate-longhorn-v2/client.go | 6 +- scripts/migrate-longhorn-v2/journal.go | 59 +++++++++++++------ scripts/migrate-longhorn-v2/node.go | 12 +++- .../temporary_activation.go | 42 ++++++++----- .../temporary_activation_flow_test.go | 14 +++-- .../temporary_activation_native_test.go | 7 ++- .../temporary_activation_prepared.go | 17 +++--- .../temporary_activation_prepared_test.go | 47 +++++++++++++++ .../temporary_activation_record_test.go | 9 ++- .../temporary_activation_run.go | 12 ++-- .../temporary_activation_test.go | 11 ++-- scripts/migrate-longhorn-v2/v2_readiness.go | 11 ++-- .../migrate-longhorn-v2/v2_readiness_test.go | 2 +- 19 files changed, 270 insertions(+), 93 deletions(-) diff --git a/internal/longhornmigration/activation.go b/internal/longhornmigration/activation.go index 18d8900b5c..4b45fec70c 100644 --- a/internal/longhornmigration/activation.go +++ b/internal/longhornmigration/activation.go @@ -15,6 +15,8 @@ const ( ActivationJournal = "longhorn-v2-temporary-activation" // ActivationOwner binds registered storage to that receipt's Kubernetes UID. ActivationOwner = "storage.platform.devantler.tech/v2-activation" + // RetainedStageRawID is the new partition on the worker with existing raw storage. + RetainedStageRawID = "r-longhorn-stage" ) // ActivationBinding retains the native identities verified before registration. @@ -100,16 +102,10 @@ func (r ActivationRecord) Validate() error { !uuid.MatchString(strings.ToLower(b.RawPartitionUUID)) || strings.EqualFold(b.RawPartitionUUID, b.EphemeralUUID) { return errActivation } - if selection == "temporary-v2-1" { - if b.PreparationUID != "" { - return errActivation - } - } else { - if b.PreparationUID == "" || preparations[b.PreparationUID] { - return errActivation - } - preparations[b.PreparationUID] = true + if b.PreparationUID == "" || preparations[b.PreparationUID] { + return errActivation } + preparations[b.PreparationUID] = true for i, identity := range []string{b.Name, b.NodeUID, b.ProviderID, strings.ToLower(b.MachineUUID), b.LonghornUID, strings.ToLower(b.EphemeralUUID), strings.ToLower(b.RawPartitionUUID)} { if seen[i][identity] { return errActivation diff --git a/internal/longhornmigration/pools.go b/internal/longhornmigration/pools.go index 7c138aae2d..8f9e7a230f 100644 --- a/internal/longhornmigration/pools.go +++ b/internal/longhornmigration/pools.go @@ -152,11 +152,14 @@ func Eligible(data []byte, n, lh, activation Object) (bool, error) { return false, errors.New("migration filesystem differs from its reviewed scope") } } else { - if role == "v1" && !registeredRawTransition(n, lh, activation) { + if RequiresActivationRecord(n, lh) && !registeredRawTransition(n, lh, activation) { return false, errors.New("migration raw transition lacks its completed identity-bound registration") } disk, _ = disks["longhorn-v2"].(map[string]any) - if text(disk, "path") != "/dev/mapper/luks2-r-longhorn" || text(disk, "diskType") != "block" || text(disk, "diskDriver") != "aio" { + path := text(disk, "path") + standard := path == "/dev/mapper/luks2-r-longhorn" + staged := role == "v2" && path == "/dev/mapper/luks2-"+RetainedStageRawID + if (!standard && !staged) || text(disk, "diskType") != "block" || text(disk, "diskDriver") != "aio" { return false, errors.New("migration raw disk differs from its reviewed scope") } } @@ -165,3 +168,12 @@ func Eligible(data []byte, n, lh, activation Object) (bool, error) { } return true, nil } + +// RequiresActivationRecord identifies storage transitions that cannot use the +// original pool's scheduling exception without a completed registration. +func RequiresActivationRecord(node, storage Object) bool { + role := text(node, "metadata", "labels", PoolLabel) + disk := value(storage, "spec", "disks", "longhorn-v2") + return disk != nil && (role == "v1" || role == "v2" && + text(storage, "spec", "disks", "longhorn-v2", "path") == "/dev/mapper/luks2-"+RetainedStageRawID) +} diff --git a/internal/longhornmigration/pools_test.go b/internal/longhornmigration/pools_test.go index 0c2a00b5c5..79249434e8 100644 --- a/internal/longhornmigration/pools_test.go +++ b/internal/longhornmigration/pools_test.go @@ -2,6 +2,7 @@ package longhornmigration import ( "encoding/json" + "fmt" "strings" "testing" ) @@ -10,6 +11,40 @@ func sourceFixture() []byte { return []byte(`{"spec":{"cluster":{"controlPlanes":3,"workers":3,"autoscaler":{"node":{"maxNodesTotal":9,"pools":[{"name":"autoscale-longhorn-v1","serverType":"cpx42","location":"fsn1","min":2,"max":2,"labels":{"storage.platform.devantler.tech/migration-pool":"v1"}},{"name":"autoscale-longhorn-v2","serverType":"cpx42","location":"fsn1","min":1,"max":1,"labels":{"storage.platform.devantler.tech/migration-pool":"v2"}}]}}},"provider":{"hetzner":{"serverLimit":10}}}}`) } +func TestStagedRawDiskRequiresItsCompletedRegistration(t *testing.T) { + n, lh := eligibilityFixture("v2") + value(lh, "spec", "disks", "longhorn-v2").(Object)["path"] = "/dev/mapper/luks2-" + RetainedStageRawID + lh["metadata"].(Object)["annotations"] = Object{ActivationOwner: "owned-registration"} + record := ActivationRecord{Version: 1, Source: strings.Repeat("a", 40), OwnerRun: "12", Completed: true, Targets: map[string]ActivationBinding{}} + for i, selection := range ActivationSelections() { + role := strings.Split(selection, "-")[1] + record.Targets[selection] = ActivationBinding{ + Selection: selection, Name: fmt.Sprintf("autoscale-longhorn-%s-%016x", role, i), + NodeUID: fmt.Sprintf("node-%d", i), ProviderID: fmt.Sprintf("hcloud://%d", i+1), + MachineUUID: fmt.Sprintf("machine-%d", i), BootID: "12345678-1234-4234-8234-000000000099", OSImage: "Talos fixture", + LonghornUID: fmt.Sprintf("longhorn-%d", i), EphemeralUUID: fmt.Sprintf("ephemeral-%d", i), + RawPartitionUUID: fmt.Sprintf("12345678-1234-4234-8234-%012x", i), PreparationUID: fmt.Sprintf("prep-%d", i), + } + } + stage := record.Targets["temporary-v2-1"] + stage.Name, stage.NodeUID, stage.ProviderID = text(n, "metadata", "name"), text(n, "metadata", "uid"), text(n, "spec", "providerID") + stage.MachineUUID, stage.LonghornUID = text(n, "status", "nodeInfo", "systemUUID"), text(lh, "metadata", "uid") + record.Targets[stage.Selection] = stage + cm := Object{"metadata": Object{"name": ActivationJournal, "namespace": "longhorn-system", "uid": "owned-registration"}} + for _, complete := range []bool{true, false} { + record.Completed = complete + encoded, err := json.Marshal(record) + if err != nil { + t.Fatal(err) + } + cm["data"] = Object{"record": string(encoded)} + ok, err := Eligible(sourceFixture(), n, lh, cm) + if ok != complete || (err == nil) != complete { + t.Fatalf("staged raw eligibility completed=%t got %t: %v", complete, ok, err) + } + } +} + // eligibilityFixture binds one fixed pool member to owned scale-down protection // and its role-specific storage, independently of the activation writer. func eligibilityFixture(role string) (Object, Object) { diff --git a/scripts/maintain-longhorn-clients/migration_capacity.go b/scripts/maintain-longhorn-clients/migration_capacity.go index 2e0be86145..d9307d9d64 100644 --- a/scripts/maintain-longhorn-clients/migration_capacity.go +++ b/scripts/maintain-longhorn-clients/migration_capacity.go @@ -22,7 +22,7 @@ func (e engine) migrationStorageEligible(ctx context.Context, node, storage obje return false, err } var activation object - if str(node, "metadata", "labels", longhornmigration.PoolLabel) == "v1" && val(storage, "spec", "disks", "longhorn-v2") != nil { + if longhornmigration.RequiresActivationRecord(node, storage) { activation, err = e.c.read(ctx, "configmap", "longhorn-system", longhornmigration.ActivationJournal) if err != nil { return false, err diff --git a/scripts/maintain-longhorn-clients/migration_capacity_test.go b/scripts/maintain-longhorn-clients/migration_capacity_test.go index a40a8c8882..04583601eb 100644 --- a/scripts/maintain-longhorn-clients/migration_capacity_test.go +++ b/scripts/maintain-longhorn-clients/migration_capacity_test.go @@ -70,9 +70,7 @@ func rawMigrationFixture(t *testing.T) (*fakeAPI, engine, object, object, object role = "v2" } binding := object{"selection": selection, "name": fmt.Sprintf("autoscale-longhorn-%s-%016x", role, i+1), "nodeUID": fmt.Sprintf("node-%d", i), "providerID": fmt.Sprintf("hcloud://%d", i+1), "machineUUID": fmt.Sprintf("machine-%d", i), "bootID": "11111111-1111-1111-1111-111111111111", "osImage": "Talos (fixture)", "longhornUID": fmt.Sprintf("longhorn-%d", i), "ephemeralUUID": fmt.Sprintf("ephemeral-%d", i), "rawPartitionUUID": fmt.Sprintf("22222222-2222-2222-2222-%012d", i+1)} - if i < 2 { - binding["preparationUID"] = fmt.Sprintf("preparation-%d", i) - } + binding["preparationUID"] = fmt.Sprintf("preparation-%d", i) if i == 0 { binding["name"], binding["nodeUID"], binding["providerID"] = str(n, "metadata", "name"), str(n, "metadata", "uid"), "hcloud://42" binding["machineUUID"], binding["longhornUID"] = "machine", str(lh, "metadata", "uid") @@ -102,6 +100,30 @@ func TestCompletedRawTransitionPreservesPublicationAndReplicaExclusion(t *testin } } +func TestStagedRawTransitionReadsItsRegistration(t *testing.T) { + a, e, n, lh, cm := rawMigrationFixture(t) + record, err := longhornmigration.DecodeActivationRecord([]byte(str(cm, "data", "record"))) + if err != nil { + t.Fatal(err) + } + stage := record.Targets["temporary-v2-1"] + meta := n["metadata"].(object) + meta["name"], meta["uid"] = stage.Name, stage.NodeUID + meta["labels"].(object)[longhornmigration.PoolLabel] = "v2" + n["spec"].(object)["providerID"] = stage.ProviderID + n["status"].(object)["nodeInfo"].(object)["systemUUID"] = stage.MachineUUID + lhMeta := lh["metadata"].(object) + lhMeta["name"], lhMeta["uid"] = stage.Name, stage.LonghornUID + val(lh, "spec", "disks", "longhorn-v2").(object)["path"] = "/dev/mapper/luks2-" + longhornmigration.RetainedStageRawID + if ok, err := e.migrationStorageEligible(t.Context(), n, lh); !ok || err != nil { + t.Fatalf("completed staged registration did not reach the publication reader: %v", err) + } + delete(a.objects, fakeKey("configmap", "longhorn-system", longhornmigration.ActivationJournal)) + if ok, err := e.migrationStorageEligible(t.Context(), n, lh); ok || err == nil { + t.Fatal("staged raw storage bypassed its registration") + } +} + // An incomplete, replaced or foreign activation must fail both independent // readers rather than turn a matching raw path into storage authorization. func TestRawTransitionCannotBorrowAnotherActivationOrWorkerIdentity(t *testing.T) { diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 6170c8eb43..b171c15f5a 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -86,7 +86,7 @@ Longhorn 1.13.0 instance-manager image and the admitted client resources and arguments must match before registration. Enabling the setting proves neither raw-disk readiness nor application recovery. -After both temporary V1 workers have completed their independently confirmed +After all three temporary workers have completed their independently confirmed raw-layout preparation, select `register-temporary-v2-storage` and confirm `register-longhorn-v2-temporary-storage`. This operation always selects all three fixed temporary pool members. It requires intact original V1 replicas, healthy @@ -102,10 +102,11 @@ identities. Complete native reads bracket the partition observations. Each rebui worker also binds its completed preparation journal UID and the successful first-attempt main workflow that produced it. A prepared-looking disk without that record, a replaced journal or a failed preparation workflow cannot authorize -registration. The V2-born worker has no reset operation and requires its native -encrypted raw-layout proof. Each conditional Longhorn Node update registers only -the canonical raw disk and adds -the journal UID as its ownership annotation. Matching unowned disks are refused. +registration. The third worker needs a completed preservation receipt and a new +encrypted staging partition with a GPT identity distinct from its original raw +partition. Each conditional Longhorn Node update registers the selected worker's +reviewed mapper and adds the journal UID as its ownership annotation. Matching +unowned disks are refused. An uncertain response stops the run. A new first-attempt main dispatch may resume an owned partial registration only after the previous workflow is terminal, and it revalidates all three native bindings and every remaining prerequisite. @@ -114,8 +115,9 @@ it grants no application cutover or original-disk reset authority. The two rebuilt workers retain their original `migration-pool=v1` allocation label. The shared publication and replica-exclusion checks recognize their raw -disks only with the completed activation journal, its matching ownership UID, -and the unchanged Kubernetes, provider, hardware and Longhorn node identities. +disks and the third worker's distinct staging mapper only with the completed +activation journal, its matching ownership UID, and the unchanged Kubernetes, +provider, hardware and Longhorn node identities. An ordinary later reboot or OS upgrade does not revoke this registration. The journal retains the original boot and OS observations for audit. Later @@ -133,7 +135,8 @@ For the unused temporary-worker route (platform#5787), finish the client refresh cleaned V1 baseline first. Select `prepare-unused-temporary-worker`, `temporary-v1-1` or `temporary-v1-2`, and the matching confirmation `prepare-unused-longhorn-v2-temporary-v1-1` or `prepare-unused-longhorn-v2-temporary-v1-2`. Each selection is resolved from the complete, -reviewed two-member V1 pool and one-member V2 pool. The fresh V2 pool has no reset operation. +reviewed two-member V1 pool and one-member V2 pool. This preparation operation +admits only the two V1-born workers. This operation wipes **only the selected temporary worker's EPHEMERAL partition** and reboots it. Its separate journal binds the original node UID, provider and hardware identities, boot, diff --git a/scripts/migrate-longhorn-v2/client.go b/scripts/migrate-longhorn-v2/client.go index be9d973099..8bc47aa79a 100644 --- a/scripts/migrate-longhorn-v2/client.go +++ b/scripts/migrate-longhorn-v2/client.go @@ -52,7 +52,11 @@ func verifyEncryptedLocalLayout(raw, ephemeral object) error { } func verifyEncryptedRawDisk(raw, ephemeral object, rawGiB uint64) error { - if text(raw, "metadata", "id") != "r-longhorn" || text(ephemeral, "metadata", "id") != "EPHEMERAL" || text(ephemeral, "spec", "uuid") == "" { + return verifyNamedEncryptedRawDisk(raw, ephemeral, "r-longhorn", rawGiB) +} + +func verifyNamedEncryptedRawDisk(raw, ephemeral object, rawID string, rawGiB uint64) error { + if text(raw, "metadata", "id") != rawID || text(ephemeral, "metadata", "id") != "EPHEMERAL" || text(ephemeral, "spec", "uuid") == "" { return errors.New("encrypted local partition identities are missing") } for _, volume := range []object{raw, ephemeral} { diff --git a/scripts/migrate-longhorn-v2/journal.go b/scripts/migrate-longhorn-v2/journal.go index d0d3d9545d..c40d2a045a 100644 --- a/scripts/migrate-longhorn-v2/journal.go +++ b/scripts/migrate-longhorn-v2/journal.go @@ -7,25 +7,29 @@ import ( "errors" "io" "regexp" + "strings" ) type journal struct { - Schema int `json:"schema"` - Target string `json:"target"` - Source string `json:"source"` - OwnerRun string `json:"ownerRun"` - Phase string `json:"phase"` - NodeUID string `json:"nodeUID"` - ProviderID string `json:"providerID"` - MachineUUID string `json:"machineUUID"` - LonghornUID string `json:"longhornUID"` - EphemeralUUID string `json:"ephemeralUUID"` - OriginalDisks object `json:"originalDisks"` - TemporarySelection string `json:"temporarySelection,omitempty"` - NativeEmptyDigest string `json:"nativeEmptyDigest,omitempty"` - OriginalBootID string `json:"originalBootID,omitempty"` - RawVolumeAbsent bool `json:"rawVolumeAbsent,omitempty"` - StagingLayout bool `json:"stagingLayout,omitempty"` + Schema int `json:"schema"` + Target string `json:"target"` + Source string `json:"source"` + OwnerRun string `json:"ownerRun"` + Phase string `json:"phase"` + NodeUID string `json:"nodeUID"` + ProviderID string `json:"providerID"` + MachineUUID string `json:"machineUUID"` + LonghornUID string `json:"longhornUID"` + EphemeralUUID string `json:"ephemeralUUID"` + OriginalDisks object `json:"originalDisks"` + TemporarySelection string `json:"temporarySelection,omitempty"` + NativeEmptyDigest string `json:"nativeEmptyDigest,omitempty"` + OriginalBootID string `json:"originalBootID,omitempty"` + RawVolumeAbsent bool `json:"rawVolumeAbsent,omitempty"` + StagingLayout bool `json:"stagingLayout,omitempty"` + RetainedRawUUID string `json:"retainedRawUUID,omitempty"` + RetainedExtentDigest string `json:"retainedExtentDigest,omitempty"` + PreparedRawUUID string `json:"preparedRawUUID,omitempty"` } // validateJournal requires complete original identities and a known procedure @@ -42,6 +46,12 @@ func validateJournal(j journal, target string) error { default: return errors.New("journal phase is outside the node-preparation procedure") } + if j.Schema == 3 { + return validateRetainedPreparation(j) + } + if j.RetainedRawUUID != "" || j.RetainedExtentDigest != "" || j.PreparedRawUUID != "" { + return errors.New("retained extent proof belongs to a different preparation profile") + } if j.Schema == 2 { if !regexp.MustCompile(`^autoscale-longhorn-v1-[0-9a-f]{16}$`).MatchString(target) || (j.TemporarySelection != "temporary-v1-1" && j.TemporarySelection != "temporary-v1-2") || @@ -66,6 +76,21 @@ func validateJournal(j journal, target string) error { return nil } +func validateRetainedPreparation(j journal) error { + preparedUUID := strings.ToLower(j.PreparedRawUUID) + if !regexp.MustCompile(`^autoscale-longhorn-v2-[0-9a-f]{16}$`).MatchString(j.Target) || + j.TemporarySelection != "temporary-v2-1" || !j.StagingLayout || j.RawVolumeAbsent || + j.OriginalDisks == nil || len(j.OriginalDisks) != 0 || !inventoryBoot.MatchString(j.OriginalBootID) || + !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(j.NativeEmptyDigest) || + !regexp.MustCompile(`^[0-9a-f]{64}$`).MatchString(j.RetainedExtentDigest) || + !inventoryBoot.MatchString(strings.ToLower(j.RetainedRawUUID)) || + (j.Phase == "prepared" && (!inventoryBoot.MatchString(preparedUUID) || strings.EqualFold(j.RetainedRawUUID, preparedUUID))) || + (j.Phase != "prepared" && preparedUUID != "") { + return errors.New("retained preparation lacks distinct original and staged storage evidence") + } + return nil +} + type preparation struct { c client target, name string @@ -81,7 +106,7 @@ func (p preparation) validateRecord(j journal) error { if err := validateJournal(j, p.target); err != nil { return err } - if p.temporarySelection != j.TemporarySelection || (j.Schema == 2 && p.name != "longhorn-v2-prepare-"+j.TemporarySelection) { + if p.temporarySelection != j.TemporarySelection || (j.TemporarySelection != "" && p.name != "longhorn-v2-prepare-"+j.TemporarySelection) { return errors.New("journal does not match the explicitly selected preparation profile") } return nil diff --git a/scripts/migrate-longhorn-v2/node.go b/scripts/migrate-longhorn-v2/node.go index bdebb3ff13..0ce32f903f 100644 --- a/scripts/migrate-longhorn-v2/node.go +++ b/scripts/migrate-longhorn-v2/node.go @@ -35,7 +35,11 @@ func verifyNode(n object, j journal, original bool) (string, error) { } func blockDisks() object { - return object{"longhorn-v2": object{"path": "/dev/mapper/luks2-r-longhorn", "diskType": "block", "diskDriver": "aio", "allowScheduling": true, "evictionRequested": false, "storageReserved": float64(4 * 1024 * 1024 * 1024), "tags": []any{}}} + return blockDisksFor("r-longhorn") +} + +func blockDisksFor(rawID string) object { + return object{"longhorn-v2": object{"path": "/dev/mapper/luks2-" + rawID, "diskType": "block", "diskDriver": "aio", "allowScheduling": true, "evictionRequested": false, "storageReserved": float64(4 * 1024 * 1024 * 1024), "tags": []any{}}} } func verifyDiskDefaults(n object) error { @@ -53,12 +57,16 @@ func verifyDiskDefaults(n object) error { } func reviewedBlockDisks(raw any) bool { + return reviewedBlockDisksFor(raw, "r-longhorn") +} + +func reviewedBlockDisksFor(raw any, rawID string) bool { disks, ok := raw.(map[string]any) if !ok || len(disks) != 1 { return false } d, ok := disks["longhorn-v2"].(map[string]any) - expected := blockDisks()["longhorn-v2"].(object) + expected := blockDisksFor(rawID)["longhorn-v2"].(object) if !ok || len(d) != len(expected) { return false } diff --git a/scripts/migrate-longhorn-v2/temporary_activation.go b/scripts/migrate-longhorn-v2/temporary_activation.go index 64306fa95c..a17a9b7974 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation.go +++ b/scripts/migrate-longhorn-v2/temporary_activation.go @@ -32,6 +32,13 @@ type temporaryV2Activation struct { reader inventoryReaderFactory } +func temporaryRawID(selection string) string { + if selection == "temporary-v2-1" { + return longhornmigration.RetainedStageRawID + } + return "r-longhorn" +} + // readBinding brackets the exact native partition reads with hardware and boot // identity, then repeats the Kubernetes projection. An incomplete read or failed // close cannot produce a binding for a subsequent registration mutation. @@ -62,15 +69,18 @@ func (a temporaryV2Activation) readBinding(ctx context.Context, selection string if err != nil { return temporaryV2Binding{}, errTemporaryActivation } - raw, err := a.c.talosVolume(ctx, address, "r-longhorn") - if err != nil { - return temporaryV2Binding{}, errTemporaryActivation - } - ephemeral, err := a.c.talosVolume(ctx, address, "EPHEMERAL") - if err != nil { - return temporaryV2Binding{}, errTemporaryActivation + observe := func(worker object) (temporaryV2Binding, error) { + raw, rawErr := a.c.talosVolume(ctx, address, temporaryRawID(selection)) + if rawErr != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + ephemeral, ephemeralErr := a.c.talosVolume(ctx, address, "EPHEMERAL") + if ephemeralErr != nil { + return temporaryV2Binding{}, errTemporaryActivation + } + return projectTemporaryV2Binding(selection, worker, storage, raw, ephemeral, pool, boot) } - before, err := projectTemporaryV2Binding(selection, node, storage, raw, ephemeral, pool, boot) + before, err := observe(node) if err != nil { return temporaryV2Binding{}, err } @@ -78,12 +88,12 @@ func (a temporaryV2Activation) readBinding(ctx context.Context, selection string if err != nil { return temporaryV2Binding{}, errTemporaryActivation } - boot, err = reader.BootID(ctx) - if err != nil || a.c.verifyHost(ctx, address, fresh) != nil { + after, err := observe(fresh) + if err != nil || before != after || a.c.verifyHost(ctx, address, fresh) != nil { return temporaryV2Binding{}, errTemporaryActivation } - after, err := projectTemporaryV2Binding(selection, fresh, storage, raw, ephemeral, pool, boot) - if err != nil || before != after { + finalBoot, err := reader.BootID(ctx) + if err != nil || finalBoot != boot { return temporaryV2Binding{}, errTemporaryActivation } return after, nil @@ -91,13 +101,14 @@ func (a temporaryV2Activation) readBinding(ctx context.Context, selection string // temporaryRegistrationState accepts an empty fence or this journal's exact // partial activation. A matching disk path without the journal UID is not owned. -func temporaryRegistrationState(storage object, journalUID string) (bool, error) { +func temporaryRegistrationState(storage object, journalUID, selection string) (bool, error) { owner := text(storage, "metadata", "annotations", temporaryActivationOwner) if owner == "" && unusedStorageFenced(storage) { return false, nil } if journalUID == "" || owner != journalUID || text(storage, "metadata", "labels", migrationEnabledLabel) != "true" || - value(storage, "spec", "allowScheduling") != true || value(storage, "spec", "evictionRequested") != false || !reviewedBlockDisks(value(storage, "spec", "disks")) { + value(storage, "spec", "allowScheduling") != true || value(storage, "spec", "evictionRequested") != false || + !reviewedBlockDisksFor(value(storage, "spec", "disks"), temporaryRawID(selection)) { return false, errTemporaryActivation } return true, nil @@ -112,7 +123,8 @@ func projectTemporaryV2Binding(selection string, node, storage, raw, ephemeral o BootID: text(node, "status", "nodeInfo", "bootID"), OSImage: text(node, "status", "nodeInfo", "osImage"), LonghornUID: text(storage, "metadata", "uid"), EphemeralUUID: text(ephemeral, "spec", "uuid"), RawPartitionUUID: text(raw, "spec", "partitionUUID")} if !inventoryTemporarySelection.MatchString(selection) || !strings.HasPrefix(selection, "temporary-"+pool.role+"-") || - !inventoryTemporaryNode.MatchString(b.Name) || verifyMigrationNode(node, pool) != nil || verifyEncryptedRawDisk(raw, ephemeral, stagingRawDiskGiB) != nil || + !inventoryTemporaryNode.MatchString(b.Name) || verifyMigrationNode(node, pool) != nil || + verifyNamedEncryptedRawDisk(raw, ephemeral, temporaryRawID(selection), stagingRawDiskGiB) != nil || b.NodeUID == "" || !regexp.MustCompile(`^hcloud://[1-9][0-9]*$`).MatchString(b.ProviderID) || b.MachineUUID == "" || b.OSImage == "" || !inventoryBoot.MatchString(b.BootID) || nativeBoot != b.BootID || !inventoryBoot.MatchString(strings.ToLower(b.RawPartitionUUID)) || b.RawPartitionUUID == b.EphemeralUUID || text(node, "status", "nodeInfo", "architecture") != "amd64" || !hugepagesReady(node) || text(node, "status", "allocatable", "cpu") != "8" || diff --git a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go index 479e9e9dde..a41f12df6a 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_flow_test.go @@ -67,12 +67,12 @@ func newActivationAPI(t *testing.T) (*activationAPI, temporaryV2Activation) { t.Fatal(err) } f.preparations = map[string]object{} - for ordinal, selection := range []string{"temporary-v1-1", "temporary-v1-2"} { + for ordinal, selection := range []string{"temporary-v1-1", "temporary-v1-2", "temporary-v2-1"} { name, _, err := resolveTemporaryInventoryTarget(selection, s.nodes, pools) if err != nil { t.Fatal(err) } - for _, node := range s.nodes { + for nodeIndex, node := range s.nodes { if text(node, "metadata", "name") != name { continue } @@ -83,7 +83,8 @@ func newActivationAPI(t *testing.T) (*activationAPI, temporaryV2Activation) { } } binding := temporaryV2Binding{Selection: selection, Name: name, NodeUID: text(node, "metadata", "uid"), ProviderID: text(node, "spec", "providerID"), - MachineUUID: text(node, "status", "nodeInfo", "systemUUID"), LonghornUID: text(lh, "metadata", "uid")} + MachineUUID: text(node, "status", "nodeInfo", "systemUUID"), LonghornUID: text(lh, "metadata", "uid"), + RawPartitionUUID: fmt.Sprintf("12345678-1234-4234-8234-%012x", nodeIndex+100)} cm := temporaryPreparedFixture(t, binding) cm["data"].(object)["record"] = strings.Replace(text(cm, "data", "record"), `"ownerRun":"7777"`, fmt.Sprintf(`"ownerRun":"%d"`, 7777+ordinal), 1) f.preparations[selection] = cm @@ -114,7 +115,7 @@ func (f *activationAPI) run(_ context.Context, input []byte, program string, arg source := strings.Repeat("1", 40) status := "completed" conclusion := "success" - if strings.Contains(args[1], "/7777/") || strings.Contains(args[1], "/7778/") { + if strings.Contains(args[1], "/7777/") || strings.Contains(args[1], "/7778/") || strings.Contains(args[1], "/7779/") { if f.preparationFailed { conclusion = "failure" } @@ -138,7 +139,10 @@ func (f *activationAPI) run(_ context.Context, input []byte, program string, arg valueObject(raw, "spec")["size"] = float64(144 << 30) raw["spec"].(object)["partitionUUID"] = fmt.Sprintf("12345678-1234-4234-8234-%012x", i+100) eph["spec"].(object)["uuid"] = fmt.Sprintf("ephemeral-%d", i) - if args[2] == "volumestatus" && args[3] == "r-longhorn" { + role := text(node, "metadata", "labels", migrationPoolLabel) + rawID := temporaryRawID("temporary-" + role + "-1") + if args[2] == "volumestatus" && args[3] == rawID { + raw["metadata"].(object)["id"] = rawID return json.Marshal(raw) } if args[2] == "volumestatus" && args[3] == "EPHEMERAL" { diff --git a/scripts/migrate-longhorn-v2/temporary_activation_native_test.go b/scripts/migrate-longhorn-v2/temporary_activation_native_test.go index 6732527389..cd9c3a8a21 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_native_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_native_test.go @@ -26,7 +26,7 @@ func (r *activationBootReader) BootID(context.Context) (string, error) { // A binding requires matching native boot, hardware and API identities plus a // successful reader close; durable identity comparison cannot waive this bracket. func TestTemporaryRawBindingBracketsNativeReadsAndClosesItsReader(t *testing.T) { - for _, mode := range []string{"valid", "boot changed", "hardware changed", "node changed", "partition unreadable", "close failed"} { + for _, mode := range []string{"valid", "boot changed", "hardware changed", "node changed", "partition unreadable", "partition changed", "close failed"} { t.Run(mode, func(t *testing.T) { n, storage, raw, eph, pool := temporaryRawBindingFixture() n["status"].(object)["addresses"] = []any{object{"type": "InternalIP", "address": "10.0.99.10"}} @@ -59,6 +59,9 @@ func TestTemporaryRawBindingBracketsNativeReadsAndClosesItsReader(t *testing.T) if mode == "partition unreadable" { return nil, errors.New("native read failed") } + if mode == "partition changed" && volumes > 2 { + raw["spec"].(object)["partitionUUID"] = "12345678-1234-4234-8234-000000000099" + } return json.Marshal(raw) case args[2] == "volumestatus" && args[3] == "EPHEMERAL": volumes++ @@ -78,7 +81,7 @@ func TestTemporaryRawBindingBracketsNativeReadsAndClosesItsReader(t *testing.T) if (err == nil) != (mode == "valid") { t.Fatal("native binding validity did not match its complete identity bracket:", err) } - if mode == "valid" && (closed != 1 || reader.reads != 2 || volumes != 2 || binding.BootID != boot) { + if mode == "valid" && (closed != 1 || reader.reads != 2 || volumes != 4 || binding.BootID != boot) { t.Fatal("successful binding omitted a native read or close") } }) diff --git a/scripts/migrate-longhorn-v2/temporary_activation_prepared.go b/scripts/migrate-longhorn-v2/temporary_activation_prepared.go index 62b2b3fbb2..e6805956ce 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_prepared.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_prepared.go @@ -17,28 +17,27 @@ func preparedTemporaryBinding(cm object, binding temporaryV2Binding) (journal, e return journal{}, errTemporaryActivation } data := []byte(text(cm, "data", "record")) - fields := map[string][]string{"": {"schema", "target", "source", "ownerRun", "phase", "nodeUID", "providerID", "machineUUID", "longhornUID", "ephemeralUUID", "originalDisks", "temporarySelection", "nativeEmptyDigest", "originalBootID", "rawVolumeAbsent", "stagingLayout"}} + fields := map[string][]string{"": {"schema", "target", "source", "ownerRun", "phase", "nodeUID", "providerID", "machineUUID", "longhornUID", "ephemeralUUID", "originalDisks", "temporarySelection", "nativeEmptyDigest", "originalBootID", "rawVolumeAbsent", "stagingLayout", "retainedRawUUID", "retainedExtentDigest", "preparedRawUUID"}} var record journal if len(data) == 0 || len(data) > 32<<10 || strictjson.Validate(data, fields) != nil || json.Unmarshal(data, &record) != nil { return journal{}, errTemporaryActivation } p := preparation{target: binding.Name, name: name, temporarySelection: binding.Selection} - if p.validateRecord(record) != nil || record.Schema != 2 || !record.StagingLayout || record.Phase != "prepared" || record.NodeUID != binding.NodeUID || + if p.validateRecord(record) != nil || (record.Schema != 2 && record.Schema != 3) || !record.StagingLayout || record.Phase != "prepared" || record.NodeUID != binding.NodeUID || record.ProviderID != binding.ProviderID || record.MachineUUID != binding.MachineUUID || record.LonghornUID != binding.LonghornUID || record.OriginalBootID == binding.BootID || strings.EqualFold(record.EphemeralUUID, binding.EphemeralUUID) { return journal{}, errTemporaryActivation } + if record.Schema == 3 && !strings.EqualFold(record.PreparedRawUUID, binding.RawPartitionUUID) { + return journal{}, errTemporaryActivation + } return record, nil } -// prepared requires successful completion of the original protected preparation -// before enabling a rebuilt worker's raw disk. The V2-born pool has no reset -// operation; its independently checked native raw layout remains mandatory. +// prepared requires successful completion of the matching protected preparation +// before enabling a rebuilt worker's raw disk. func (a temporaryV2Activation) prepared(ctx context.Context, binding temporaryV2Binding) (temporaryV2Binding, error) { - if binding.Selection == "temporary-v2-1" { - return binding, nil - } - if binding.Selection != "temporary-v1-1" && binding.Selection != "temporary-v1-2" { + if !inventoryTemporarySelection.MatchString(binding.Selection) { return temporaryV2Binding{}, errTemporaryActivation } cm, err := a.c.get(ctx, "configmap", "longhorn-system", "longhorn-v2-prepare-"+binding.Selection, false) diff --git a/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go b/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go index 0ed721c590..b537891cfb 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_prepared_test.go @@ -14,6 +14,12 @@ func temporaryPreparedFixture(t *testing.T, binding temporaryV2Binding) object { NodeUID: binding.NodeUID, ProviderID: binding.ProviderID, MachineUUID: binding.MachineUUID, LonghornUID: binding.LonghornUID, EphemeralUUID: "original-ephemeral", OriginalDisks: object{}, TemporarySelection: binding.Selection, NativeEmptyDigest: strings.Repeat("c", 64), OriginalBootID: "12345678-1234-4234-8234-000000000099", RawVolumeAbsent: true, StagingLayout: true} + if binding.Selection == "temporary-v2-1" { + j.Schema, j.RawVolumeAbsent = 3, false + j.RetainedExtentDigest = strings.Repeat("f", 64) + j.RetainedRawUUID = "12345678-1234-4234-8234-000000000089" + j.PreparedRawUUID = binding.RawPartitionUUID + } data, err := json.Marshal(j) if err != nil { t.Fatal(err) @@ -68,3 +74,44 @@ func TestTemporaryActivationRequiresTheCompletedPreparationIdentity(t *testing.T }) } } + +func TestTemporaryActivationRequiresThePreservingPreparationIdentity(t *testing.T) { + n, lh, raw, eph, pool := temporaryRawBindingFixture() + binding, err := projectTemporaryV2Binding("temporary-v1-1", n, lh, raw, eph, pool, text(n, "status", "nodeInfo", "bootID")) + if err != nil { + t.Fatal(err) + } + binding.Selection = "temporary-v2-1" + binding.Name = strings.Replace(binding.Name, "-v1-", "-v2-", 1) + cm := temporaryPreparedFixture(t, binding) + var record journal + if err = json.Unmarshal([]byte(text(cm, "data", "record")), &record); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + change func(*journal) + }{ + {"complete", nil}, + {"missing extent proof", func(r *journal) { r.RetainedExtentDigest = "" }}, + {"unexpected raw absence", func(r *journal) { r.RawVolumeAbsent = true }}, + {"wrong destination", func(r *journal) { r.PreparedRawUUID = "12345678-1234-4234-8234-000000000099" }}, + {"retained partition reused", func(r *journal) { r.RetainedRawUUID = r.PreparedRawUUID }}, + {"unfinished reset", func(r *journal) { r.Phase = "reset-requested" }}, + } { + t.Run(tc.name, func(t *testing.T) { + candidate := record + if tc.change != nil { + tc.change(&candidate) + } + data, marshalErr := json.Marshal(candidate) + if marshalErr != nil { + t.Fatal(marshalErr) + } + cm["data"] = object{"record": string(data)} + if _, err := preparedTemporaryBinding(cm, binding); (err == nil) != (tc.change == nil) { + t.Fatalf("unexpected preserving-preparation verdict: %v", err) + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_activation_record_test.go b/scripts/migrate-longhorn-v2/temporary_activation_record_test.go index d9b03c6e36..a928e2856f 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_record_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_record_test.go @@ -24,9 +24,7 @@ func temporaryActivationRecordFixture(t *testing.T) temporaryActivationRecord { b.NodeUID, b.ProviderID, b.MachineUUID = fmt.Sprintf("node-%d", i), fmt.Sprintf("hcloud://%d", 100+i), fmt.Sprintf("hardware-%d", i) b.LonghornUID, b.EphemeralUUID = fmt.Sprintf("longhorn-%d", i), fmt.Sprintf("ephemeral-%d", i) b.RawPartitionUUID = fmt.Sprintf("12345678-1234-4234-8234-%012x", i+10) - if selection != "temporary-v2-1" { - b.PreparationUID = fmt.Sprintf("preparation-%d", i) - } + b.PreparationUID = fmt.Sprintf("preparation-%d", i) record.Targets[selection] = b } return record @@ -49,6 +47,11 @@ func TestTemporaryActivationRecordRequiresThreeDistinctCompleteBindings(t *testi b.PreparationUID = "" r.Targets["temporary-v1-1"] = b }, + "missing preserving preparation": func(r *temporaryActivationRecord) { + b := r.Targets["temporary-v2-1"] + b.PreparationUID = "" + r.Targets["temporary-v2-1"] = b + }, "duplicated preparation identity": func(r *temporaryActivationRecord) { b := r.Targets["temporary-v1-2"] b.PreparationUID = r.Targets["temporary-v1-1"].PreparationUID diff --git a/scripts/migrate-longhorn-v2/temporary_activation_run.go b/scripts/migrate-longhorn-v2/temporary_activation_run.go index 8e318a3abb..17a55351cb 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_run.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_run.go @@ -124,7 +124,7 @@ func (a temporaryV2Activation) targets(ctx context.Context, s inventory, journal return nil, errTemporaryActivation } } - if _, err = temporaryRegistrationState(storage[name], journalUID); err != nil { + if _, err = temporaryRegistrationState(storage[name], journalUID, selection); err != nil { return nil, err } role := text(nodes[name], "metadata", "labels", migrationPoolLabel) @@ -143,7 +143,7 @@ func (a temporaryV2Activation) targets(ctx context.Context, s inventory, journal // registrationChanges adds ownership and the canonical raw disk in one // conditional Longhorn-node patch, preserving unrelated metadata. -func registrationChanges(storage object, journalUID string) []object { +func registrationChanges(storage object, journalUID, selection string) []object { changes := []object{} if value(storage, "metadata", "labels") == nil { changes = append(changes, object{"op": "add", "path": "/metadata/labels", "value": object{}}) @@ -154,7 +154,7 @@ func registrationChanges(storage object, journalUID string) []object { changes = append(changes, object{"op": "add", "path": "/metadata/labels/" + strings.ReplaceAll(migrationEnabledLabel, "/", "~1"), "value": "true"}, object{"op": "add", "path": "/metadata/annotations/" + strings.ReplaceAll(temporaryActivationOwner, "/", "~1"), "value": journalUID}, - object{"op": "replace", "path": "/spec/disks", "value": blockDisks()}, + object{"op": "replace", "path": "/spec/disks", "value": blockDisksFor(temporaryRawID(selection))}, object{"op": "replace", "path": "/spec/allowScheduling", "value": true}, object{"op": "replace", "path": "/spec/evictionRequested", "value": false}) return changes @@ -213,7 +213,7 @@ func (a temporaryV2Activation) run(ctx context.Context) error { if err != nil || text(storage, "metadata", "uid") != record.Targets[selection].LonghornUID { return errTemporaryActivation } - registered, err := temporaryRegistrationState(storage, uid) + registered, err := temporaryRegistrationState(storage, uid, selection) if err != nil { return err } @@ -223,7 +223,7 @@ func (a temporaryV2Activation) run(ctx context.Context) error { if _, err = a.owned(ctx, uid, record); err != nil { return err } - if err = a.c.patch(ctx, "nodes.longhorn.io", "longhorn-system", storage, registrationChanges(storage, uid)...); err != nil { + if err = a.c.patch(ctx, "nodes.longhorn.io", "longhorn-system", storage, registrationChanges(storage, uid, selection)...); err != nil { return err } } @@ -279,7 +279,7 @@ func (a temporaryV2Activation) accepted(ctx context.Context) error { return err } for _, binding := range record.Targets { - registered, err := temporaryRegistrationState(storage[binding.Name], uid) + registered, err := temporaryRegistrationState(storage[binding.Name], uid, binding.Selection) if err != nil || !registered { return errTemporaryActivation } diff --git a/scripts/migrate-longhorn-v2/temporary_activation_test.go b/scripts/migrate-longhorn-v2/temporary_activation_test.go index df8eb1b467..d7fef9de52 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_test.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_test.go @@ -73,12 +73,15 @@ func TestTemporaryRawRegistrationAcceptsOnlyItsOwnPartialActivation(t *testing.T } return lh } - if registered, err := temporaryRegistrationState(storage(false), "journal-uid"); err != nil || registered { + if registered, err := temporaryRegistrationState(storage(false), "journal-uid", "temporary-v1-1"); err != nil || registered { t.Fatal("empty fenced storage could not be registered:", err) } - if registered, err := temporaryRegistrationState(storage(true), "journal-uid"); err != nil || !registered { + if registered, err := temporaryRegistrationState(storage(true), "journal-uid", "temporary-v1-1"); err != nil || !registered { t.Fatal("the journal's exact existing activation could not resume:", err) } + if _, err := temporaryRegistrationState(storage(true), "journal-uid", "temporary-v2-1"); err == nil { + t.Fatal("the retained mapper qualified as staged storage") + } for name, change := range map[string]func(object){ "foreign journal": func(lh object) { lh["metadata"].(object)["annotations"].(object)[temporaryActivationOwner] = "foreign" }, "missing ownership": func(lh object) { delete(lh["metadata"].(object)["annotations"].(object), temporaryActivationOwner) }, @@ -90,12 +93,12 @@ func TestTemporaryRawRegistrationAcceptsOnlyItsOwnPartialActivation(t *testing.T t.Run(name, func(t *testing.T) { lh := storage(true) change(lh) - if _, err := temporaryRegistrationState(lh, "journal-uid"); err == nil { + if _, err := temporaryRegistrationState(lh, "journal-uid", "temporary-v1-1"); err == nil { t.Fatal("a foreign or changed disk registration was adopted") } }) } - if _, err := temporaryRegistrationState(storage(true), ""); err == nil { + if _, err := temporaryRegistrationState(storage(true), "", "temporary-v1-1"); err == nil { t.Fatal("an unjournaled existing registration was adopted") } } diff --git a/scripts/migrate-longhorn-v2/v2_readiness.go b/scripts/migrate-longhorn-v2/v2_readiness.go index d13d496e03..ca5ed7f3e1 100644 --- a/scripts/migrate-longhorn-v2/v2_readiness.go +++ b/scripts/migrate-longhorn-v2/v2_readiness.go @@ -29,7 +29,7 @@ func verifyV2Runtime(s inventory, managers []object, pools map[string]migrationP if _, err = indexed(managers); err != nil { return err } - workers, raw := map[string]bool{}, map[string]bool{} + workers, raw := map[string]bool{}, map[string]string{} for name, n := range nodes { if strings.HasPrefix(name, "prod-control-plane-") { continue @@ -38,21 +38,22 @@ func verifyV2Runtime(s inventory, managers []object, pools map[string]migrationP return errors.New("V2 migration workers must be schedulable with eight allocatable CPUs") } workers[name] = true - if _, temporary := pools[text(n, "metadata", "labels", migrationPoolLabel)]; temporary { - raw[name] = true + role := text(n, "metadata", "labels", migrationPoolLabel) + if _, temporary := pools[role]; temporary { + raw[name] = temporaryRawID("temporary-" + role + "-1") } } if len(workers) != 6 || len(raw) != 3 { return errors.New("V2 canary storage and client roster is incomplete") } diskIDs := map[string]bool{} - for name := range raw { + for name, rawID := range raw { lh := storage[name] status, _ := value(lh, "status").(object) diskStatus, _ := value(lh, "status", "diskStatus").(object) disk, _ := diskStatus["longhorn-v2"].(object) id := text(disk, "diskUUID") - if !reviewedBlockDisks(value(lh, "spec", "disks")) || value(lh, "spec", "allowScheduling") != true || value(lh, "spec", "evictionRequested") != false || + if !reviewedBlockDisksFor(value(lh, "spec", "disks"), rawID) || value(lh, "spec", "allowScheduling") != true || value(lh, "spec", "evictionRequested") != false || !ready(status) || len(diskStatus) != 1 || !ready(disk) || id == "" || diskIDs[id] { return errors.New("three distinct ready canonical raw disks are not proven") } diff --git a/scripts/migrate-longhorn-v2/v2_readiness_test.go b/scripts/migrate-longhorn-v2/v2_readiness_test.go index 29940ee895..8299e617e7 100644 --- a/scripts/migrate-longhorn-v2/v2_readiness_test.go +++ b/scripts/migrate-longhorn-v2/v2_readiness_test.go @@ -36,7 +36,7 @@ func v2ReadinessFixtureForRoute(legacy bool) (inventory, []object) { continue } lh := resource(text(n, "metadata", "name")) - lh["spec"] = object{"disks": blockDisks(), "allowScheduling": true, "evictionRequested": false, "instanceManagerCPURequest": float64(0)} + lh["spec"] = object{"disks": blockDisksFor(temporaryRawID("temporary-" + role + "-1")), "allowScheduling": true, "evictionRequested": false, "instanceManagerCPURequest": float64(0)} lh["status"] = object{"conditions": readyConditions(), "diskStatus": object{"longhorn-v2": object{"diskUUID": "raw-" + text(n, "metadata", "name"), "conditions": readyConditions()}}} s.storageNodes = append(s.storageNodes, lh) } From 57c038bd4e16172901a78a192ede1b1eb75dad45 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 10 Nov 2026 10:21:42 +0100 Subject: [PATCH 34/35] fix(storage): bind retained raw and Talos state extents --- internal/longhorninventory/block_layout.go | 102 +++++++++++++----- .../longhorninventory/block_layout_test.go | 84 +++++++++++++-- internal/longhorninventory/native.go | 28 +++++ .../storage_root_inventory.go | 6 +- .../storage_root_inventory_test.go | 7 +- 5 files changed, 190 insertions(+), 37 deletions(-) diff --git a/internal/longhorninventory/block_layout.go b/internal/longhorninventory/block_layout.go index 240c515d4d..e71595b5f9 100644 --- a/internal/longhorninventory/block_layout.go +++ b/internal/longhorninventory/block_layout.go @@ -28,21 +28,43 @@ type BlockLayoutReport struct { RawDiscoveredPartitions, OtherUserVolumes int RawExtentObserved bool `json:",omitempty"` Digest string + retained RetainedLayoutIdentity } +// RetainedLayoutIdentity binds the raw and STATE physical extents across +// boots. It is private journal evidence, separate from the public report. +type RetainedLayoutIdentity struct { + PartitionUUID, ExtentDigest string +} + +type blockLayoutMode int + +const ( + ordinaryLayout blockLayoutMode = iota + retainedOpen + retainedClosed +) + // ListBlockLayout reads the block layout twice and reports it only when both // complete reads agree. func (n Native) ListBlockLayout(ctx context.Context) (BlockLayoutReport, error) { - return n.listBlockLayout(ctx, false) + return n.listBlockLayout(ctx, ordinaryLayout) } -// ListRetainedBlockLayout additionally joins the configured r-longhorn status -// to its physical partition and compares the byte extent in both reads. -func (n Native) ListRetainedBlockLayout(ctx context.Context) (BlockLayoutReport, error) { - return n.listBlockLayout(ctx, true) +// ListRetainedBlockLayout compares the raw and STATE extents and the expected +// raw-volume controller state in both reads. +func (n Native) ListRetainedBlockLayout(ctx context.Context, closed bool) (BlockLayoutReport, RetainedLayoutIdentity, error) { + mode := retainedOpen + if closed { + mode = retainedClosed + } + report, err := n.listBlockLayout(ctx, mode) + identity := report.retained + report.retained = RetainedLayoutIdentity{} + return report, identity, err } -func (n Native) listBlockLayout(ctx context.Context, retainRaw bool) (BlockLayoutReport, error) { +func (n Native) listBlockLayout(ctx context.Context, mode blockLayoutMode) (BlockLayoutReport, error) { ctx, cancel, err := n.requestContext(ctx) if err != nil { return BlockLayoutReport{}, nativeFailure{code: "request"} @@ -51,11 +73,11 @@ func (n Native) listBlockLayout(ctx context.Context, retainRaw bool) (BlockLayou if n.Blocks == nil { return BlockLayoutReport{}, nativeFailure{code: "block-client"} } - first, _, err := n.readBlockLayout(ctx, retainRaw) + first, _, err := n.readBlockLayout(ctx, mode) if err != nil { return BlockLayoutReport{}, err } - second, report, err := n.readBlockLayout(ctx, retainRaw) + second, report, err := n.readBlockLayout(ctx, mode) if err != nil { return BlockLayoutReport{}, err } @@ -74,7 +96,7 @@ func (n Native) listBlockLayout(ctx context.Context, retainRaw bool) (BlockLayou // Selected layout metadata is compared privately. The public commitment covers // only reported counters and properties, without private device identities, // key material, configuration contents, filesystem contents or provider errors. -func (n Native) readBlockLayout(ctx context.Context, retainRaw bool) ([]byte, BlockLayoutReport, error) { +func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]byte, BlockLayoutReport, error) { if !active(ctx) { return nil, BlockLayoutReport{}, nativeFailure{code: "block-list"} } @@ -167,7 +189,7 @@ func (n Native) readBlockLayout(ctx context.Context, retainRaw bool) ([]byte, Bl return nil, BlockLayoutReport{}, invalid } seen = map[string]bool{} - var rawPartition *block.DiscoveredVolumeSpec + var rawPartition, statePartition *block.DiscoveredVolumeSpec for _, item := range discovered.Items { v, ok := item.(*block.DiscoveredVolume) if !ok || v == nil || v.Metadata().Namespace() != block.NamespaceName || v.Metadata().Phase() != resource.PhaseRunning || v.Metadata().ID() == "" || len(v.Metadata().ID()) > 256 || seen[v.Metadata().ID()] { @@ -182,6 +204,12 @@ func (n Native) readBlockLayout(ctx context.Context, retainRaw bool) ([]byte, Bl report.RawDiscoveredPartitions++ rawPartition = s } + if mode != ordinaryLayout && s.PartitionLabel == "STATE" { + if statePartition != nil { + return nil, BlockLayoutReport{}, invalid + } + statePartition = s + } if !appendRow(struct { Kind, ID string Metadata any @@ -189,15 +217,29 @@ func (n Native) readBlockLayout(ctx context.Context, retainRaw bool) ([]byte, Bl return nil, BlockLayoutReport{}, invalid } } - if retainRaw { - if report.RawDiscoveredPartitions != 1 || !report.RawLUKS2 || !report.RawLockedToState || - !report.RawNodeIDOnly || !retainedRawMatches(rawStatus, rawPartition) || - !appendRow(struct { - Kind, FilesystemUUID string - Offset uint64 - }{"retained-extent", rawPartition.UUID, rawPartition.Offset}) { + if mode != ordinaryLayout { + if report.RawDiscoveredPartitions != 1 || !validRetainedRaw(rawPartition) || + !retainedStateMatches(statePartition, rawPartition) { + return nil, BlockLayoutReport{}, invalid + } + if (mode == retainedOpen && (!report.RawLUKS2 || !report.RawLockedToState || !report.RawNodeIDOnly || + !retainedRawMatches(rawStatus, rawPartition))) || (mode == retainedClosed && rawStatus != nil) { return nil, BlockLayoutReport{}, invalid } + if mode == retainedClosed { + if err := n.verifyEmptyPartitionHolders(ctx, rawPartition.DevPath); err != nil { + return nil, BlockLayoutReport{}, err + } + } + extent := []any{"retained-extent", rawPartition.ParentDevPath, + rawPartition.PartitionUUID, rawPartition.UUID, rawPartition.PartitionType, rawPartition.Offset, rawPartition.Size, + statePartition.PartitionUUID, statePartition.PartitionType, statePartition.Offset, statePartition.Size} + encoded, err := json.Marshal(extent) + if err != nil || !appendRow(extent) { + return nil, BlockLayoutReport{}, invalid + } + digest := sha256.Sum256(encoded) + report.retained = RetainedLayoutIdentity{PartitionUUID: rawPartition.PartitionUUID, ExtentDigest: hex.EncodeToString(digest[:])} report.RawExtentObserved = true } slices.SortFunc(rows, bytes.Compare) @@ -208,12 +250,24 @@ func (n Native) readBlockLayout(ctx context.Context, retainRaw bool) ([]byte, Bl return encoded, report, nil } +func validRetainedRaw(partition *block.DiscoveredVolumeSpec) bool { + return partition != nil && partition.Type == "partition" && partition.Name == "luks" && partition.UUID != "" && + partition.PartitionUUID != "" && partition.ParentDevPath != "" && partition.Size == 96<<30 && validPartitionExtent(partition) +} + func retainedRawMatches(status *block.VolumeStatusSpec, partition *block.DiscoveredVolumeSpec) bool { - return status != nil && partition != nil && status.Phase == block.VolumePhaseReady && - status.Type == block.VolumeTypePartition && partition.Type == "partition" && partition.Name == "luks" && partition.UUID != "" && - status.Location == partition.DevPath && status.ParentLocation != "" && status.ParentLocation == partition.ParentDevPath && - status.PartitionUUID != "" && status.PartitionUUID == partition.PartitionUUID && - status.PartitionIndex > 0 && uint(status.PartitionIndex) == partition.PartitionIndex && - status.Size == 96<<30 && status.Size == partition.Size && - partition.Offset > 0 && partition.Offset%512 == 0 && partition.Offset <= math.MaxUint64-partition.Size + return status != nil && status.Phase == block.VolumePhaseReady && status.Type == block.VolumeTypePartition && + status.Location == partition.DevPath && status.ParentLocation == partition.ParentDevPath && + status.PartitionUUID == partition.PartitionUUID && status.PartitionIndex > 0 && + uint(status.PartitionIndex) == partition.PartitionIndex && status.Size == partition.Size +} + +func retainedStateMatches(state, raw *block.DiscoveredVolumeSpec) bool { + return state != nil && state.Type == "partition" && state.PartitionUUID != "" && state.PartitionUUID != raw.PartitionUUID && + state.DevPath != raw.DevPath && state.ParentDevPath == raw.ParentDevPath && validPartitionExtent(state) && + (state.Offset+state.Size <= raw.Offset || raw.Offset+raw.Size <= state.Offset) +} + +func validPartitionExtent(p *block.DiscoveredVolumeSpec) bool { + return p.Offset > 0 && p.Offset%512 == 0 && p.Size > 0 && p.Size%512 == 0 && p.Offset <= math.MaxUint64-p.Size } diff --git a/internal/longhorninventory/block_layout_test.go b/internal/longhorninventory/block_layout_test.go index bc8ce289ea..c9772d6eb2 100644 --- a/internal/longhorninventory/block_layout_test.go +++ b/internal/longhorninventory/block_layout_test.go @@ -4,13 +4,17 @@ import ( "context" "encoding/json" "errors" + "io" + "os" "strings" "testing" "time" resource "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/state" + machine "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/resources/block" + "google.golang.org/grpc" "google.golang.org/grpc/metadata" ) @@ -21,6 +25,8 @@ type blockLayoutFixture struct { mode string rawPhase *block.VolumePhase retained bool + closed bool + shifted bool } func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, options ...state.ListOption) (resource.List, error) { @@ -55,6 +61,9 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio } } result.Items = []resource.Resource{eph, raw} + if f.closed { + result.Items = []resource.Resource{eph} + } if f.mode == "duplicate" { result.Items = append(result.Items, eph) } @@ -72,11 +81,20 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio *v.TypedSpec() = block.DiscoveredVolumeSpec{DevPath: "/dev/sda6", ParentDevPath: "/dev/sda", PartitionLabel: "EPHEMERAL", PartitionUUID: "private-partition-canary", Size: 150 << 30} result.Items = []resource.Resource{v} if f.retained { - raw := block.NewDiscoveredVolume(block.NamespaceName, "sda3") + rawID, rawIndex := "sda3", uint(3) + if f.shifted { + rawID, rawIndex = "sda4", 4 + } + raw := block.NewDiscoveredVolume(block.NamespaceName, rawID) + state := block.NewDiscoveredVolume(block.NamespaceName, "sda2") + *state.TypedSpec() = block.DiscoveredVolumeSpec{ + Type: "partition", DevPath: "/dev/sda2", ParentDevPath: "/dev/sda", PartitionLabel: "STATE", + PartitionUUID: "private-state-uuid-canary", PartitionIndex: 2, Size: 100 << 20, Offset: 500 << 20, + } *raw.TypedSpec() = block.DiscoveredVolumeSpec{ - Type: "partition", DevPath: "/dev/sda3", ParentDevPath: "/dev/sda", PartitionLabel: "r-longhorn", + Type: "partition", DevPath: "/dev/" + rawID, ParentDevPath: "/dev/sda", PartitionLabel: "r-longhorn", Name: "luks", UUID: "private-luks-canary", - PartitionUUID: "private-raw-uuid-canary", PartitionIndex: 3, Size: 96 << 30, Offset: 1 << 30, + PartitionUUID: "private-raw-uuid-canary", PartitionIndex: rawIndex, Size: 96 << 30, Offset: 1 << 30, } switch f.mode { case "offset missing": @@ -99,8 +117,19 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio } case "duplicate raw label": v.TypedSpec().PartitionLabel = "r-longhorn" + case "STATE changed": + if f.calls > 2 { + state.TypedSpec().Offset += 512 + } + case "STATE overlaps raw": + state.TypedSpec().Offset = raw.TypedSpec().Offset + case "STATE on other disk": + state.TypedSpec().ParentDevPath = "/dev/sdb" } result.Items = append(result.Items, raw) + if f.mode != "STATE missing" { + result.Items = append(result.Items, state) + } } } if f.mode == "partial" && f.calls == 2 { @@ -110,26 +139,65 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio } func TestNativeRetainedBlockLayoutRequiresStablePhysicalExtent(t *testing.T) { - for _, mode := range []string{"stable", "offset missing", "offset changed", "wrong identity", "wrong parent", "wrong size", "LUKS header missing", "LUKS header changed", "duplicate raw label"} { + for _, mode := range []string{"stable", "offset missing", "offset changed", "wrong identity", "wrong parent", "wrong size", "LUKS header missing", "LUKS header changed", "duplicate raw label", "STATE changed", "STATE overlaps raw", "STATE on other disk", "STATE missing"} { t.Run(mode, func(t *testing.T) { f := &blockLayoutFixture{t: t, mode: mode, retained: true} - report, err := (Native{API: fileAPI{}, Target: "10.0.0.8", Blocks: f}).ListRetainedBlockLayout(context.Background()) + report, identity, err := (Native{API: fileAPI{}, Target: "10.0.0.8", Blocks: f}).ListRetainedBlockLayout(context.Background(), false) if mode != "stable" { - if err == nil || report != (BlockLayoutReport{}) || strings.Contains(err.Error(), "canary") { + if err == nil || report != (BlockLayoutReport{}) || identity != (RetainedLayoutIdentity{}) || strings.Contains(err.Error(), "canary") { t.Fatal("unverified physical extent retained success or leaked identity") } return } encoded, marshalErr := json.Marshal(report) - if err != nil || marshalErr != nil || !report.RawExtentObserved || f.calls != 4 || + if err != nil || marshalErr != nil || !report.RawExtentObserved || identity.PartitionUUID == "" || len(identity.ExtentDigest) != 64 || f.calls != 4 || strings.Contains(string(encoded), "canary") || strings.Contains(string(encoded), "/dev/") || - strings.Contains(string(encoded), "1073741824") { + strings.Contains(string(encoded), "1073741824") || strings.Contains(string(encoded), identity.ExtentDigest) { t.Fatalf("physical extent observation was incomplete or exposed private geometry: %v", err) } }) } } +func TestRetainedLayoutDigestSurvivesOnlyAnUnchangedPhysicalExtent(t *testing.T) { + observe := func(f *blockLayoutFixture, closed bool) (RetainedLayoutIdentity, error) { + api := fileAPI{list: func(ctx context.Context, req *machine.ListRequest) (grpc.ServerStreamingClient[machine.FileInfo], error) { + const root = "/sys/class/block/sda4/holders" + md, _ := metadata.FromOutgoingContext(ctx) + if !closed || req.GetRoot() != root || req.GetRecursionDepth() != 1 || len(md.Get("node")) != 1 || md.Get("node")[0] != "10.0.0.8" { + t.Fatal("retained-volume inspection escaped the selected partition") + } + items := []*machine.FileInfo{{Name: root, RelativeName: ".", Mode: uint32(os.ModeDir), IsDir: true}} + end := io.EOF + if f.mode == "live holder" { + items = append(items, &machine.FileInfo{Name: root + "/dm-2", RelativeName: "dm-2", Link: "../../dm-2"}) + } + if f.mode == "incomplete holders" { + end = errors.Join(io.EOF, errors.New("private-transport")) + } + return &fileStream{items: items, err: end}, nil + }} + _, identity, err := (Native{API: api, Target: "10.0.0.8", Blocks: f}).ListRetainedBlockLayout(t.Context(), closed) + return identity, err + } + open, err := observe(&blockLayoutFixture{t: t, retained: true}, false) + if err != nil { + t.Fatal(err) + } + closed, err := observe(&blockLayoutFixture{t: t, retained: true, closed: true, shifted: true}, true) + if err != nil || closed != open { + t.Fatal("closing the mapping and changing its GPT index lost the retained identity:", err) + } + if _, err = observe(&blockLayoutFixture{t: t, retained: true}, true); err == nil { + t.Fatal("an open mapper qualified as closed") + } + for _, mode := range []string{"live holder", "incomplete holders"} { + if _, err = observe(&blockLayoutFixture{t: t, retained: true, closed: true, shifted: true, mode: mode}, true); err == nil || strings.Contains(err.Error(), "private-transport") { + t.Fatalf("unproven holders %s qualified or leaked provider data", mode) + } + } +} + func TestNativeBlockLayoutReportsObservedStateWithoutPrivateIdentities(t *testing.T) { for _, mode := range []string{"stable", "partial", "changed", "duplicate", "missing ephemeral", "tearing down", "canceled"} { t.Run(mode, func(t *testing.T) { diff --git a/internal/longhorninventory/native.go b/internal/longhorninventory/native.go index c81dd3a1eb..a8e6f23885 100644 --- a/internal/longhorninventory/native.go +++ b/internal/longhorninventory/native.go @@ -5,6 +5,7 @@ import ( "errors" "io" "net/netip" + "os" "path" "regexp" "strings" @@ -40,6 +41,7 @@ const maxMessageBytes = 64 << 10 var errNative = errors.New("native file observation is incomplete, changed or outside its reviewed scope") var bootFormat = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\n$`) +var kernelPartitionName = regexp.MustCompile(`^[a-z][a-z0-9]{0,62}$`) func (n Native) requestContext(ctx context.Context) (context.Context, context.CancelFunc, error) { address, err := netip.ParseAddr(n.Target) @@ -126,6 +128,32 @@ func classifyNativeFailure(err error, fallback string) nativeFailure { } } +// verifyEmptyPartitionHolders admits only the complete, empty sysfs directory +// for the physical partition named by the native discovery record. It does not +// assert that every possible raw file descriptor on the node is closed. +func (n Native) verifyEmptyPartitionHolders(ctx context.Context, devicePath string) error { + device, ok := strings.CutPrefix(devicePath, "/dev/") + refused := nativeFailure{code: "block-holders"} + if !ok || !kernelPartitionName.MatchString(device) || !active(ctx) { + return refused + } + root := path.Join("/sys/class/block", device, "holders") + stream, err := n.API.List(ctx, &machine.ListRequest{Root: root, RecursionDepth: 1}, grpc.MaxCallRecvMsgSize(maxMessageBytes)) + if err != nil || stream == nil { + return refused + } + entry, err := stream.Recv() + if err != nil || !active(ctx) || entry == nil || proto.Size(entry) > maxMessageBytes || + !n.validMetadata(entry.GetMetadata()) || entry.GetError() != "" || entry.GetLink() != "" || + entry.GetRelativeName() != "." || entry.GetName() != root || !entry.GetIsDir() || !os.FileMode(entry.GetMode()).IsDir() { + return refused + } + if _, err = stream.Recv(); err != io.EOF || !active(ctx) { + return refused + } + return nil +} + // BootID reads a fixed, exact-format kernel identity. It does not accept a // successful stream ending after a partial file, or legacy error envelopes. func (n Native) BootID(ctx context.Context) (string, error) { diff --git a/scripts/migrate-longhorn-v2/storage_root_inventory.go b/scripts/migrate-longhorn-v2/storage_root_inventory.go index c81be5ed5b..1239a5465a 100644 --- a/scripts/migrate-longhorn-v2/storage_root_inventory.go +++ b/scripts/migrate-longhorn-v2/storage_root_inventory.go @@ -17,7 +17,7 @@ type nativeBlockLayoutReader interface { ListBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) } type nativeRetainedBlockLayoutReader interface { - ListRetainedBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) + ListRetainedBlockLayout(context.Context, bool) (longhorninventory.BlockLayoutReport, longhorninventory.RetainedLayoutIdentity, error) } type storageRootReceipt struct { Schema int `json:"schema"` @@ -133,8 +133,8 @@ func observeNativeBlockLayout(ctx context.Context, reader nativeReplicaReader, t if !ok { return longhorninventory.BlockLayoutReport{}, stageClient } - layout, err := retainedReader.ListRetainedBlockLayout(ctx) - if err == nil && !layout.RawExtentObserved { + layout, identity, err := retainedReader.ListRetainedBlockLayout(ctx, false) + if err == nil && (!layout.RawExtentObserved || !inventoryBoot.MatchString(identity.PartitionUUID) || len(identity.ExtentDigest) != 64) { return longhorninventory.BlockLayoutReport{}, stageBlockLayout } return layout, err diff --git a/scripts/migrate-longhorn-v2/storage_root_inventory_test.go b/scripts/migrate-longhorn-v2/storage_root_inventory_test.go index 61543038dd..b8b9846fe3 100644 --- a/scripts/migrate-longhorn-v2/storage_root_inventory_test.go +++ b/scripts/migrate-longhorn-v2/storage_root_inventory_test.go @@ -29,12 +29,15 @@ func (r *storageRootReaderFixture) ListBlockLayout(context.Context) (longhorninv return longhorninventory.BlockLayoutReport{VolumeStatuses: 2, DiscoveredVolumes: 1, EphemeralPhase: "ready", RawPhase: "waiting", Digest: strings.Repeat("a", 64)}, nil } -func (r *storageRootReaderFixture) ListRetainedBlockLayout(context.Context) (longhorninventory.BlockLayoutReport, error) { +func (r *storageRootReaderFixture) ListRetainedBlockLayout(_ context.Context, closed bool) (longhorninventory.BlockLayoutReport, longhorninventory.RetainedLayoutIdentity, error) { r.retainedCalls++ + if closed { + return longhorninventory.BlockLayoutReport{}, longhorninventory.RetainedLayoutIdentity{}, errors.New("public observation must require the original mapper") + } return longhorninventory.BlockLayoutReport{ VolumeStatuses: 2, DiscoveredVolumes: 2, EphemeralPhase: "ready", RawPhase: "ready", RawExtentObserved: r.retainedObserved, Digest: strings.Repeat("a", 64), - }, nil + }, longhorninventory.RetainedLayoutIdentity{PartitionUUID: "12345678-1234-4234-8234-000000000099", ExtentDigest: strings.Repeat("b", 64)}, nil } func (r *storageRootReaderFixture) ListStorageRoot(context.Context) (longhorninventory.Listing, error) { From 5f009f78f891c0da6b1077650323b26a8480cd73 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Tue, 10 Nov 2026 10:39:05 +0100 Subject: [PATCH 35/35] feat(storage): prepare the third migration worker without losing its raw disk --- .../workflows/prepare-longhorn-v2-node.yaml | 4 + internal/longhorninventory/block_layout.go | 103 +++++++++- .../longhorninventory/block_layout_test.go | 64 ++++++- internal/longhorninventory/block_wire.go | 5 +- internal/longhornmigration/activation.go | 2 - internal/longhornmigration/pools.go | 3 + scripts/migrate-longhorn-v2/README.md | 16 ++ scripts/migrate-longhorn-v2/config.go | 35 ++-- .../inventory_temporary_test.go | 4 + scripts/migrate-longhorn-v2/journal.go | 31 ++- scripts/migrate-longhorn-v2/main.go | 11 +- .../migrate-longhorn-v2/retained_prepare.go | 178 ++++++++++++++++++ .../retained_prepare_test.go | 153 +++++++++++++++ .../storage_root_inventory.go | 51 +++-- .../temporary_activation_prepared.go | 8 +- .../temporary_authorization.go | 7 + .../temporary_authorization_test.go | 17 ++ .../migrate-longhorn-v2/temporary_cordon.go | 2 +- .../migrate-longhorn-v2/temporary_prepare.go | 107 +++++++++-- .../migrate-longhorn-v2/temporary_recovery.go | 8 +- 20 files changed, 723 insertions(+), 86 deletions(-) create mode 100644 scripts/migrate-longhorn-v2/retained_prepare.go create mode 100644 scripts/migrate-longhorn-v2/retained_prepare_test.go diff --git a/.github/workflows/prepare-longhorn-v2-node.yaml b/.github/workflows/prepare-longhorn-v2-node.yaml index e5242d39de..ae8627fa11 100644 --- a/.github/workflows/prepare-longhorn-v2-node.yaml +++ b/.github/workflows/prepare-longhorn-v2-node.yaml @@ -22,6 +22,7 @@ name: Prepare Longhorn V2 Migration [ prepare-worker, prepare-unused-temporary-worker, + prepare-retained-temporary-worker, refresh-clients, benchmark-v1, benchmark-v2, @@ -105,6 +106,9 @@ jobs: prepare-unused-temporary-worker) [[ "${LONGHORN_V2_NODE}" =~ ^temporary-v1-[12]$ && "${LONGHORN_V2_CONFIRM}" == "prepare-unused-longhorn-v2-${LONGHORN_V2_NODE}" ]] ;; + prepare-retained-temporary-worker) + [[ "${LONGHORN_V2_NODE}" == temporary-v2-1 && "${LONGHORN_V2_CONFIRM}" == prepare-retained-longhorn-v2-temporary-v2-1 ]] + ;; inventory-replicas) [[ "${LONGHORN_V2_NODE}" =~ ^(prod-worker-[123]|temporary-v1-[12]|temporary-v2-1)$ && "${LONGHORN_V2_CONFIRM}" == "inventory-longhorn-replicas-${LONGHORN_V2_NODE}" ]] ;; diff --git a/internal/longhorninventory/block_layout.go b/internal/longhorninventory/block_layout.go index e71595b5f9..998ba762d9 100644 --- a/internal/longhorninventory/block_layout.go +++ b/internal/longhorninventory/block_layout.go @@ -35,6 +35,9 @@ type BlockLayoutReport struct { // boots. It is private journal evidence, separate from the public report. type RetainedLayoutIdentity struct { PartitionUUID, ExtentDigest string + StagePartitionUUID string + StageStatusPresent bool + EphemeralUUID string } type blockLayoutMode int @@ -126,7 +129,8 @@ func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]by } seen := map[string]bool{} ephFound := false - var rawStatus *block.VolumeStatusSpec + var ephemeralUUID string + var rawStatus, stageStatus, ephemeralStatus *block.VolumeStatusSpec for _, item := range statuses.Items { v, ok := item.(*block.VolumeStatus) if !ok || v == nil || v.Metadata().Namespace() != block.NamespaceName || v.Metadata().Phase() != resource.PhaseRunning || v.Metadata().ID() == "" || len(v.Metadata().ID()) > 256 || seen[v.Metadata().ID()] { @@ -160,6 +164,8 @@ func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]by switch id { case "EPHEMERAL": ephFound = true + ephemeralUUID = s.UUID + ephemeralStatus = s report.EphemeralPhase = phase report.EphemeralSizeBytes = s.Size report.EphemeralLUKS2 = s.EncryptionProvider == block.EncryptionProviderLUKS2 @@ -173,8 +179,14 @@ func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]by report.RawLUKS2 = s.EncryptionProvider == block.EncryptionProviderLUKS2 report.RawLockedToState = s.EncryptionLockedToState report.RawNodeIDOnly = nodeIDOnly + case "r-longhorn-stage": + stageStatus = s + report.OtherUserVolumes++ default: if strings.HasPrefix(id, "r-") || strings.HasPrefix(id, "u-") { + if mode != ordinaryLayout { + return nil, BlockLayoutReport{}, invalid + } report.OtherUserVolumes++ } } @@ -189,7 +201,7 @@ func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]by return nil, BlockLayoutReport{}, invalid } seen = map[string]bool{} - var rawPartition, statePartition *block.DiscoveredVolumeSpec + var rawPartition, statePartition, stagePartition, ephemeralPartition *block.DiscoveredVolumeSpec for _, item := range discovered.Items { v, ok := item.(*block.DiscoveredVolume) if !ok || v == nil || v.Metadata().Namespace() != block.NamespaceName || v.Metadata().Phase() != resource.PhaseRunning || v.Metadata().ID() == "" || len(v.Metadata().ID()) > 256 || seen[v.Metadata().ID()] { @@ -210,6 +222,19 @@ func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]by } statePartition = s } + if mode != ordinaryLayout && s.PartitionLabel == "EPHEMERAL" { + if ephemeralPartition != nil { + return nil, BlockLayoutReport{}, invalid + } + ephemeralPartition = s + } + if mode != ordinaryLayout && s.PartitionLabel != "r-longhorn" && + (strings.HasPrefix(s.PartitionLabel, "r-") || strings.HasPrefix(s.PartitionLabel, "u-")) { + if s.PartitionLabel != "r-longhorn-stage" || stagePartition != nil { + return nil, BlockLayoutReport{}, invalid + } + stagePartition = s + } if !appendRow(struct { Kind, ID string Metadata any @@ -218,8 +243,8 @@ func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]by } } if mode != ordinaryLayout { - if report.RawDiscoveredPartitions != 1 || !validRetainedRaw(rawPartition) || - !retainedStateMatches(statePartition, rawPartition) { + if ephemeralUUID == "" || report.RawDiscoveredPartitions != 1 || !validRetainedRaw(rawPartition) || + !retainedStateMatches(statePartition, rawPartition) || !retainedEphemeralMatches(ephemeralStatus, ephemeralPartition, rawPartition, statePartition) { return nil, BlockLayoutReport{}, invalid } if (mode == retainedOpen && (!report.RawLUKS2 || !report.RawLockedToState || !report.RawNodeIDOnly || @@ -227,6 +252,9 @@ func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]by return nil, BlockLayoutReport{}, invalid } if mode == retainedClosed { + if err := n.verifyRetainedConfigClosed(ctx); err != nil { + return nil, BlockLayoutReport{}, err + } if err := n.verifyEmptyPartitionHolders(ctx, rawPartition.DevPath); err != nil { return nil, BlockLayoutReport{}, err } @@ -235,11 +263,21 @@ func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]by rawPartition.PartitionUUID, rawPartition.UUID, rawPartition.PartitionType, rawPartition.Offset, rawPartition.Size, statePartition.PartitionUUID, statePartition.PartitionType, statePartition.Offset, statePartition.Size} encoded, err := json.Marshal(extent) - if err != nil || !appendRow(extent) { + if err != nil || !appendRow(extent) || !appendRow([]any{"ephemeral-extent", ephemeralPartition.Offset}) || + !validStagedPartition(stageStatus, stagePartition, rawPartition, statePartition) || + (stagePartition != nil && (stagePartition.PartitionUUID == ephemeralPartition.PartitionUUID || !extentsDisjoint(stagePartition, ephemeralPartition))) { return nil, BlockLayoutReport{}, invalid } digest := sha256.Sum256(encoded) report.retained = RetainedLayoutIdentity{PartitionUUID: rawPartition.PartitionUUID, ExtentDigest: hex.EncodeToString(digest[:])} + report.retained.EphemeralUUID = ephemeralUUID + report.retained.StageStatusPresent = stageStatus != nil + if stagePartition != nil { + report.retained.StagePartitionUUID = stagePartition.PartitionUUID + if !appendRow([]any{"staging-extent", stagePartition.UUID, stagePartition.PartitionType, stagePartition.Offset}) { + return nil, BlockLayoutReport{}, invalid + } + } report.RawExtentObserved = true } slices.SortFunc(rows, bytes.Compare) @@ -250,6 +288,30 @@ func (n Native) readBlockLayout(ctx context.Context, mode blockLayoutMode) ([]by return encoded, report, nil } +func (n Native) verifyRetainedConfigClosed(ctx context.Context) error { + refused := nativeFailure{code: "block-config"} + configs, err := n.Blocks.List(ctx, block.NewVolumeConfig(block.NamespaceName, "").Metadata()) + if err != nil || !active(ctx) || len(configs.Items) == 0 || len(configs.Items) > 128 { + return refused + } + seen := map[string]bool{} + for _, item := range configs.Items { + v, ok := item.(*block.VolumeConfig) + if !ok || v == nil || v.Metadata().Namespace() != block.NamespaceName || v.Metadata().Phase() != resource.PhaseRunning { + return refused + } + id := v.Metadata().ID() + if id == "" || id == "r-longhorn" || seen[id] { + return refused + } + seen[id] = true + } + if !seen["r-longhorn-stage"] || !seen["EPHEMERAL"] { + return refused + } + return nil +} + func validRetainedRaw(partition *block.DiscoveredVolumeSpec) bool { return partition != nil && partition.Type == "partition" && partition.Name == "luks" && partition.UUID != "" && partition.PartitionUUID != "" && partition.ParentDevPath != "" && partition.Size == 96<<30 && validPartitionExtent(partition) @@ -258,6 +320,7 @@ func validRetainedRaw(partition *block.DiscoveredVolumeSpec) bool { func retainedRawMatches(status *block.VolumeStatusSpec, partition *block.DiscoveredVolumeSpec) bool { return status != nil && status.Phase == block.VolumePhaseReady && status.Type == block.VolumeTypePartition && status.Location == partition.DevPath && status.ParentLocation == partition.ParentDevPath && + strings.HasPrefix(status.MountLocation, "/dev/dm-") && status.PartitionUUID == partition.PartitionUUID && status.PartitionIndex > 0 && uint(status.PartitionIndex) == partition.PartitionIndex && status.Size == partition.Size } @@ -265,7 +328,35 @@ func retainedRawMatches(status *block.VolumeStatusSpec, partition *block.Discove func retainedStateMatches(state, raw *block.DiscoveredVolumeSpec) bool { return state != nil && state.Type == "partition" && state.PartitionUUID != "" && state.PartitionUUID != raw.PartitionUUID && state.DevPath != raw.DevPath && state.ParentDevPath == raw.ParentDevPath && validPartitionExtent(state) && - (state.Offset+state.Size <= raw.Offset || raw.Offset+raw.Size <= state.Offset) + extentsDisjoint(state, raw) +} + +func retainedEphemeralMatches(status *block.VolumeStatusSpec, ephemeral, raw, state *block.DiscoveredVolumeSpec) bool { + return ephemeral != nil && ephemeral.Type == "partition" && ephemeral.PartitionUUID != "" && + ephemeral.PartitionUUID != raw.PartitionUUID && ephemeral.PartitionUUID != state.PartitionUUID && + ephemeral.ParentDevPath == raw.ParentDevPath && validPartitionExtent(ephemeral) && + extentsDisjoint(ephemeral, raw) && extentsDisjoint(ephemeral, state) && retainedRawMatches(status, ephemeral) +} + +func validStagedPartition(status *block.VolumeStatusSpec, stage, raw, state *block.DiscoveredVolumeSpec) bool { + if stage == nil { + if status == nil { + return true + } + waiting := status.Phase == block.VolumePhaseWaiting || status.Phase == block.VolumePhaseFailed && + (status.PreFailPhase == block.VolumePhaseWaiting || status.PreFailPhase == block.VolumePhaseMissing) + return waiting && status.Type == block.VolumeTypePartition && status.Size == 0 && status.Location == "" && + status.MountLocation == "" && status.ParentLocation == "" && status.UUID == "" && status.PartitionUUID == "" && status.PartitionIndex == 0 + } + return stage.Type == "partition" && stage.Name == "luks" && stage.UUID != "" && stage.Size == 144<<30 && + stage.PartitionUUID != "" && stage.PartitionUUID != raw.PartitionUUID && stage.PartitionUUID != state.PartitionUUID && + stage.ParentDevPath == raw.ParentDevPath && validPartitionExtent(stage) && extentsDisjoint(stage, raw) && extentsDisjoint(stage, state) && + retainedRawMatches(status, stage) && status.EncryptionProvider == block.EncryptionProviderLUKS2 && status.EncryptionLockedToState && + len(status.ConfiguredEncryptionKeys) == 1 && status.ConfiguredEncryptionKeys[0] == "nodeID" +} + +func extentsDisjoint(a, b *block.DiscoveredVolumeSpec) bool { + return a.Offset+a.Size <= b.Offset || b.Offset+b.Size <= a.Offset } func validPartitionExtent(p *block.DiscoveredVolumeSpec) bool { diff --git a/internal/longhorninventory/block_layout_test.go b/internal/longhorninventory/block_layout_test.go index c9772d6eb2..4cc5664200 100644 --- a/internal/longhorninventory/block_layout_test.go +++ b/internal/longhorninventory/block_layout_test.go @@ -27,16 +27,26 @@ type blockLayoutFixture struct { retained bool closed bool shifted bool + staged bool } func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, options ...state.ListOption) (resource.List, error) { f.t.Helper() - f.calls++ deadline, bounded := ctx.Deadline() md, _ := metadata.FromOutgoingContext(ctx) if !bounded || time.Until(deadline) > 15*time.Second || len(options) != 0 || kind.Namespace() != block.NamespaceName || len(md.Get("node")) != 1 || md.Get("node")[0] != "10.0.0.8" || len(md.Get("nodes")) != 0 { f.t.Fatal("block observation escaped its bounded fixed-node scope") } + if kind.Type() == block.VolumeConfigType { + result := resource.List{Items: []resource.Resource{ + block.NewVolumeConfig(block.NamespaceName, "r-longhorn-stage"), block.NewVolumeConfig(block.NamespaceName, "EPHEMERAL"), + }} + if f.mode == "old config remains" { + result.Items = append(result.Items, block.NewVolumeConfig(block.NamespaceName, "r-longhorn")) + } + return result, nil + } + f.calls++ var result resource.List if f.calls%2 == 1 { if kind.Type() != block.VolumeStatusType { @@ -53,9 +63,14 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio raw.TypedSpec().Phase = block.VolumePhaseReady } if f.retained { + eph.TypedSpec().PartitionUUID = "private-partition-canary" + eph.TypedSpec().PartitionIndex = 6 + if f.staged { + eph.TypedSpec().Size = 42 << 30 + } *raw.TypedSpec() = block.VolumeStatusSpec{ Phase: block.VolumePhaseReady, Type: block.VolumeTypePartition, Location: "/dev/sda3", - ParentLocation: "/dev/sda", PartitionUUID: "private-raw-uuid-canary", PartitionIndex: 3, + ParentLocation: "/dev/sda", MountLocation: "/dev/dm-1", PartitionUUID: "private-raw-uuid-canary", PartitionIndex: 3, Size: 96 << 30, EncryptionProvider: block.EncryptionProviderLUKS2, EncryptionLockedToState: true, ConfiguredEncryptionKeys: []string{"nodeID"}, } @@ -64,6 +79,16 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio if f.closed { result.Items = []resource.Resource{eph} } + if f.staged { + stage := block.NewVolumeStatus(block.NamespaceName, "r-longhorn-stage") + *stage.TypedSpec() = block.VolumeStatusSpec{Phase: block.VolumePhaseReady, Type: block.VolumeTypePartition, Location: "/dev/sda5", + ParentLocation: "/dev/sda", MountLocation: "/dev/dm-2", PartitionUUID: "private-stage-canary", PartitionIndex: 5, + Size: 144 << 30, EncryptionProvider: block.EncryptionProviderLUKS2, EncryptionLockedToState: true, ConfiguredEncryptionKeys: []string{"nodeID"}} + if f.mode == "staging mapper missing" { + stage.TypedSpec().MountLocation = "" + } + result.Items = append(result.Items, stage) + } if f.mode == "duplicate" { result.Items = append(result.Items, eph) } @@ -81,6 +106,13 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio *v.TypedSpec() = block.DiscoveredVolumeSpec{DevPath: "/dev/sda6", ParentDevPath: "/dev/sda", PartitionLabel: "EPHEMERAL", PartitionUUID: "private-partition-canary", Size: 150 << 30} result.Items = []resource.Resource{v} if f.retained { + v.TypedSpec().Type, v.TypedSpec().PartitionIndex, v.TypedSpec().Offset = "partition", 6, 100<<30 + if f.staged { + v.TypedSpec().Size, v.TypedSpec().Offset = 42<<30, 244<<30 + if f.mode == "staging overlaps ephemeral" { + v.TypedSpec().Offset -= 512 + } + } rawID, rawIndex := "sda3", uint(3) if f.shifted { rawID, rawIndex = "sda4", 4 @@ -130,6 +162,16 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio if f.mode != "STATE missing" { result.Items = append(result.Items, state) } + if f.staged { + stage := block.NewDiscoveredVolume(block.NamespaceName, "sda5") + *stage.TypedSpec() = block.DiscoveredVolumeSpec{Type: "partition", DevPath: "/dev/sda5", ParentDevPath: "/dev/sda", + PartitionLabel: "r-longhorn-stage", Name: "luks", UUID: "private-stage-luks-canary", + PartitionUUID: "private-stage-canary", PartitionIndex: 5, Size: 144 << 30, Offset: 100 << 30} + if f.mode == "staging overlaps old" { + stage.TypedSpec().Offset = raw.TypedSpec().Offset + } + result.Items = append(result.Items, stage) + } } } if f.mode == "partial" && f.calls == 2 { @@ -138,6 +180,22 @@ func (f *blockLayoutFixture) List(ctx context.Context, kind resource.Kind, optio return result, nil } +func TestRetainedLayoutOnlyReturnsASeparateEncryptedStagingPartition(t *testing.T) { + for _, mode := range []string{"staged", "staging overlaps old", "staging overlaps ephemeral", "staging mapper missing"} { + t.Run(mode, func(t *testing.T) { + f := &blockLayoutFixture{t: t, retained: true, staged: true, mode: mode} + _, identity, err := (Native{API: fileAPI{}, Target: "10.0.0.8", Blocks: f}).ListRetainedBlockLayout(t.Context(), false) + if mode == "staged" { + if err != nil || identity.StagePartitionUUID != "private-stage-canary" || !identity.StageStatusPresent { + t.Fatal("separate staging partition did not qualify", err) + } + } else if err == nil || identity.StagePartitionUUID != "" { + t.Fatal("unqualified staging partition reached the retained identity") + } + }) + } +} + func TestNativeRetainedBlockLayoutRequiresStablePhysicalExtent(t *testing.T) { for _, mode := range []string{"stable", "offset missing", "offset changed", "wrong identity", "wrong parent", "wrong size", "LUKS header missing", "LUKS header changed", "duplicate raw label", "STATE changed", "STATE overlaps raw", "STATE on other disk", "STATE missing"} { t.Run(mode, func(t *testing.T) { @@ -191,7 +249,7 @@ func TestRetainedLayoutDigestSurvivesOnlyAnUnchangedPhysicalExtent(t *testing.T) if _, err = observe(&blockLayoutFixture{t: t, retained: true}, true); err == nil { t.Fatal("an open mapper qualified as closed") } - for _, mode := range []string{"live holder", "incomplete holders"} { + for _, mode := range []string{"live holder", "incomplete holders", "old config remains"} { if _, err = observe(&blockLayoutFixture{t: t, retained: true, closed: true, shifted: true, mode: mode}, true); err == nil || strings.Contains(err.Error(), "private-transport") { t.Fatalf("unproven holders %s qualified or leaked provider data", mode) } diff --git a/internal/longhorninventory/block_wire.go b/internal/longhorninventory/block_wire.go index 965d2a27af..99b3472b56 100644 --- a/internal/longhorninventory/block_wire.go +++ b/internal/longhorninventory/block_wire.go @@ -26,8 +26,8 @@ type BlockListAPI interface { List(context.Context, *v1alpha1.ListRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[v1alpha1.ListResponse], error) } -// BoundedBlocks is a BlockCollections limited to the block namespace's volume -// status and discovered-volume kinds on the one node named by Target. +// BoundedBlocks limits reads to the block namespace's volume statuses, +// configurations and discovered volumes on the one node named by Target. type BoundedBlocks struct { API BlockListAPI Target string @@ -49,6 +49,7 @@ func (r BoundedBlocks) List(ctx context.Context, kind resource.Kind, options ... limit := 128 switch kind.Type() { case block.VolumeStatusType: + case block.VolumeConfigType: case block.DiscoveredVolumeType: limit = 512 default: diff --git a/internal/longhornmigration/activation.go b/internal/longhornmigration/activation.go index 4b45fec70c..9d7c18f519 100644 --- a/internal/longhornmigration/activation.go +++ b/internal/longhornmigration/activation.go @@ -15,8 +15,6 @@ const ( ActivationJournal = "longhorn-v2-temporary-activation" // ActivationOwner binds registered storage to that receipt's Kubernetes UID. ActivationOwner = "storage.platform.devantler.tech/v2-activation" - // RetainedStageRawID is the new partition on the worker with existing raw storage. - RetainedStageRawID = "r-longhorn-stage" ) // ActivationBinding retains the native identities verified before registration. diff --git a/internal/longhornmigration/pools.go b/internal/longhornmigration/pools.go index 8f9e7a230f..9e98931ab3 100644 --- a/internal/longhornmigration/pools.go +++ b/internal/longhornmigration/pools.go @@ -24,6 +24,9 @@ const ( ScaleDownDisabled = "cluster-autoscaler.kubernetes.io/scale-down-disabled" ) +// RetainedStageRawID names the new partition on the worker with existing raw storage. +const RetainedStageRawID = "r-longhorn-stage" + // Pool is one reviewed fixed-size migration pool from the cluster config. type Pool struct { Name, Role, ServerType string diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index b171c15f5a..d01ed402fc 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -170,6 +170,22 @@ cap does not apply to this separate profile. The raw disk remains unregistered a storage activation and volume migration follow separately. Original storage replicas remain on the permanent workers throughout this preparation route. +For the V2-born worker, use `prepare-retained-temporary-worker`, +`temporary-v2-1` and `prepare-retained-longhorn-v2-temporary-v2-1`. The +reviewed pool configuration must already declare the original raw volume, +the distinct 144 GiB staging volume and a 40–44 GiB EPHEMERAL range. The +protected operation verifies the original 96 GiB raw partition and STATE +extents, drains the worker and closes the original mapper. It records reset +intent only after complete native reads show the old configuration, volume +status and partition holders cleared while the physical extents remain. + +After the EPHEMERAL-only reset, the worker must report a new boot, a replaced +encrypted EPHEMERAL partition and a separate encrypted staging partition. +The workflow reopens the original mapper only after rechecking the retained +extents. It requires the same extents and an open original mapper before +recording preparation complete. A lost reset response permits observation +of recovery only. The original mapper remains outside Longhorn scheduling. + When a replica-directory observation holds, `inventory-storage-root` inspects the fixed parent `/var/lib/longhorn` directory. Use the same fixed node selections with the matching confirmation `inventory-longhorn-storage-root-`. It brackets two complete native diff --git a/scripts/migrate-longhorn-v2/config.go b/scripts/migrate-longhorn-v2/config.go index 3eabbc24d4..45d3129fa9 100644 --- a/scripts/migrate-longhorn-v2/config.go +++ b/scripts/migrate-longhorn-v2/config.go @@ -13,20 +13,8 @@ func verifyRawDiskConfig(raw, ephemeral object, rawGiB uint64) error { return errors.New("the expected Talos volume configurations are missing") } for _, volume := range []object{raw, ephemeral} { - if text(volume, "spec", "type") != "partition" || text(volume, "spec", "provisioning", "diskSelector", "match") != "system_disk" || - value(volume, "spec", "provisioning", "partitionSpec", "grow") != false || text(volume, "spec", "encryption", "provider") != "luks2" { - return errors.New("talos has not loaded the reviewed local-disk layout") - } - if relative := value(volume, "spec", "provisioning", "partitionSpec", "relativeMaxSize"); relative != nil && relative != float64(0) { - return errors.New("relative partition sizing is outside this migration") - } - keys, ok := value(volume, "spec", "encryption", "keys").([]any) - if !ok || len(keys) != 1 { - return errors.New("talos encryption keys differ from the reviewed layout") - } - key, ok := keys[0].(map[string]any) - if !ok || text(key, "type") != "nodeID" || value(key, "slot") != float64(0) || value(key, "lockToState") != true { - return errors.New("talos encryption is not locked to the existing state partition") + if err := verifyEncryptedPartitionConfig(volume); err != nil { + return err } } if value(raw, "spec", "provisioning", "partitionSpec", "minSize") != float64(rawGiB<<30) || @@ -39,3 +27,22 @@ func verifyRawDiskConfig(raw, ephemeral object, rawGiB uint64) error { } return nil } + +func verifyEncryptedPartitionConfig(volume object) error { + if text(volume, "spec", "type") != "partition" || text(volume, "spec", "provisioning", "diskSelector", "match") != "system_disk" || + value(volume, "spec", "provisioning", "partitionSpec", "grow") != false || text(volume, "spec", "encryption", "provider") != "luks2" { + return errors.New("talos has not loaded the reviewed local-disk layout") + } + if relative := value(volume, "spec", "provisioning", "partitionSpec", "relativeMaxSize"); relative != nil && relative != float64(0) { + return errors.New("relative partition sizing is outside this migration") + } + keys, ok := value(volume, "spec", "encryption", "keys").([]any) + if !ok || len(keys) != 1 { + return errors.New("talos encryption keys differ from the reviewed layout") + } + key, ok := keys[0].(map[string]any) + if !ok || text(key, "type") != "nodeID" || value(key, "slot") != float64(0) || value(key, "lockToState") != true { + return errors.New("talos encryption is not locked to the existing state partition") + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/inventory_temporary_test.go b/scripts/migrate-longhorn-v2/inventory_temporary_test.go index 64a17ee84f..b2f1ac1ca7 100644 --- a/scripts/migrate-longhorn-v2/inventory_temporary_test.go +++ b/scripts/migrate-longhorn-v2/inventory_temporary_test.go @@ -258,6 +258,10 @@ func TestTemporaryInventoryNativeWorkflowGuardPreservesResetBoundary(t *testing. {"prepare-unused-temporary-worker", "temporary-v1-1", "refs/heads/feature", "1", "prepare-unused-longhorn-v2-temporary-v1-1", false}, {"prepare-unused-temporary-worker", "temporary-v1-1", "refs/heads/main", "2", "prepare-unused-longhorn-v2-temporary-v1-1", false}, {"prepare-unused-temporary-worker", "temporary-v1-1", "refs/heads/main", "1", "inventory-longhorn-replicas-temporary-v1-1", false}, + {"prepare-retained-temporary-worker", "temporary-v2-1", "refs/heads/main", "1", "prepare-retained-longhorn-v2-temporary-v2-1", true}, + {"prepare-retained-temporary-worker", "temporary-v1-1", "refs/heads/main", "1", "prepare-retained-longhorn-v2-temporary-v2-1", false}, + {"prepare-retained-temporary-worker", "temporary-v2-1", "refs/heads/main", "2", "prepare-retained-longhorn-v2-temporary-v2-1", false}, + {"prepare-retained-temporary-worker", "temporary-v2-1", "refs/heads/main", "1", "prepare-unused-longhorn-v2-temporary-v2-1", false}, {"reboot-host-for-hugepages", "prod-worker-1", "refs/heads/main", "1", "reboot-longhorn-v2-host-prod-worker-1", true}, {"reboot-host-for-hugepages", "temporary-v1-1", "refs/heads/main", "1", "reboot-longhorn-v2-host-temporary-v1-1", false}, {"reboot-host-for-hugepages", "prod-worker-4", "refs/heads/main", "1", "reboot-longhorn-v2-host-prod-worker-4", false}, diff --git a/scripts/migrate-longhorn-v2/journal.go b/scripts/migrate-longhorn-v2/journal.go index c40d2a045a..df3933acb2 100644 --- a/scripts/migrate-longhorn-v2/journal.go +++ b/scripts/migrate-longhorn-v2/journal.go @@ -5,9 +5,10 @@ import ( "context" "encoding/json" "errors" - "io" "regexp" "strings" + + "github.com/devantler-tech/platform/internal/strictjson" ) type journal struct { @@ -119,17 +120,26 @@ func (p preparation) readRecord(ctx context.Context) (object, journal, error) { if err != nil || cm == nil { return cm, journal{}, err } + j, err := decodePreparationJournal([]byte(text(cm, "data", "record"))) + if err != nil { + return nil, journal{}, err + } + return cm, j, p.validateRecord(j) +} + +func decodePreparationJournal(data []byte) (journal, error) { + fields := map[string][]string{"": {"schema", "target", "source", "ownerRun", "phase", "nodeUID", "providerID", "machineUUID", "longhornUID", "ephemeralUUID", "originalDisks", "temporarySelection", "nativeEmptyDigest", "originalBootID", "rawVolumeAbsent", "stagingLayout", "retainedRawUUID", "retainedExtentDigest", "preparedRawUUID"}} + refused := errors.New("node-preparation journal cannot be decoded") + if len(data) == 0 || len(data) > 32<<10 || strictjson.Validate(data, fields) != nil { + return journal{}, refused + } var j journal - d := json.NewDecoder(bytes.NewBufferString(text(cm, "data", "record"))) + d := json.NewDecoder(bytes.NewReader(data)) d.DisallowUnknownFields() if d.Decode(&j) != nil { - return nil, journal{}, errors.New("node-preparation journal cannot be decoded") - } - var extra any - if d.Decode(&extra) != io.EOF { - return nil, journal{}, errors.New("node-preparation journal is not singular") + return journal{}, refused } - return cm, j, p.validateRecord(j) + return j, nil } // reviewedWorkflowPath accepts the bare path GitHub returns today and the suffixed @@ -207,6 +217,10 @@ func (p preparation) mutate(ctx context.Context, kind, namespace string, o objec } func (p *preparation) advance(ctx context.Context, phase string) error { + return p.advanceWithStage(ctx, phase, "") +} + +func (p *preparation) advanceWithStage(ctx context.Context, phase, stagedUUID string) error { cm, j, err := p.readRecord(ctx) if err != nil { return err @@ -215,6 +229,7 @@ func (p *preparation) advance(ctx context.Context, phase string) error { return errors.New("node-preparation journal ownership changed") } j.Phase = phase + j.PreparedRawUUID = stagedUUID if err = validateJournal(j, p.target); err != nil { return err } diff --git a/scripts/migrate-longhorn-v2/main.go b/scripts/migrate-longhorn-v2/main.go index a6b233cd84..072929cde5 100644 --- a/scripts/migrate-longhorn-v2/main.go +++ b/scripts/migrate-longhorn-v2/main.go @@ -3,7 +3,6 @@ package main import ( "context" - "encoding/json" "fmt" "os" "time" @@ -24,6 +23,8 @@ func main() { authorization = authorize(os.Getenv, target) case "prepare-unused-temporary-worker": authorization = authorizeTemporaryPreparation(os.Getenv, target) + case "prepare-retained-temporary-worker": + authorization = authorizeRetainedTemporaryPreparation(os.Getenv, target) case "benchmark-v1", "benchmark-v2": authorization = authorizeBenchmark(os.Getenv, operation[len("benchmark-"):]) case "register-temporary-v2-storage": @@ -102,7 +103,7 @@ func main() { } return } - if operation == "prepare-unused-temporary-worker" { + if operation == "prepare-unused-temporary-worker" || operation == "prepare-retained-temporary-worker" { c := client{run: execute, context: "admin@prod", inventoryPools: pools} nodes, readErr := c.list(ctx, "nodes", "") if readErr != nil { @@ -118,8 +119,8 @@ func main() { fmt.Fprintln(os.Stderr, "Temporary preparation held: reviewed pool selection is unproven.") os.Exit(1) } - var record journal - if json.Unmarshal([]byte(text(cm, "data", "record")), &record) != nil || record.TemporarySelection != target { + record, recordErr := decodePreparationJournal([]byte(text(cm, "data", "record"))) + if recordErr != nil || record.TemporarySelection != target { fmt.Fprintln(os.Stderr, "Temporary preparation held: recovery identity is unproven.") os.Exit(1) } @@ -140,7 +141,7 @@ func main() { } p := temporaryPreparation{preparation: preparation{c: c, target: resolved, name: "longhorn-v2-prepare-" + target, temporarySelection: target, source: os.Getenv("GITHUB_SHA"), owner: os.Getenv("GITHUB_RUN_ID"), pools: pools}, staging: true} if err = p.prepare(ctx); err != nil { - fmt.Fprintln(os.Stderr, "Unused temporary preparation held:", err) + fmt.Fprintln(os.Stderr, "Temporary worker preparation held:", err) fmt.Fprintln(os.Stderr, "Preserve its journal and owned cordon; uncertain resets are never replayed.") os.Exit(1) } diff --git a/scripts/migrate-longhorn-v2/retained_prepare.go b/scripts/migrate-longhorn-v2/retained_prepare.go new file mode 100644 index 0000000000..183c1b56a0 --- /dev/null +++ b/scripts/migrate-longhorn-v2/retained_prepare.go @@ -0,0 +1,178 @@ +package main + +import ( + "context" + "errors" + "time" + + "github.com/devantler-tech/platform/internal/longhorninventory" + "github.com/devantler-tech/platform/internal/longhornmigration" +) + +const closeRetainedRawConfig = `apiVersion: v1alpha1 +kind: RawVolumeConfig +name: longhorn +$patch: delete +` + +func (p temporaryPreparation) retainsRaw() bool { + return p.temporarySelection == "temporary-v2-1" +} + +// The staging and root configuration must already be loaded by the reviewed +// deployment. This operation only closes and later restores the old definition. +func (p temporaryPreparation) verifyRetainedLayoutConfig(ctx context.Context, address string, oldOpen bool) error { + staged, err := p.c.talosResource(ctx, address, "volumeconfig", longhornmigration.RetainedStageRawID) + if err != nil || !retainedConfigRange(staged, stagingRawDiskGiB, stagingRawDiskGiB) { + return errInventory + } + ephemeral, err := p.c.talosResource(ctx, address, "volumeconfig", "EPHEMERAL") + if err != nil || !retainedConfigRange(ephemeral, 40, 44) { + return errInventory + } + if oldOpen { + raw, err := p.c.talosResource(ctx, address, "volumeconfig", "r-longhorn") + if err != nil || !retainedConfigRange(raw, permanentRawDiskGiB, permanentRawDiskGiB) { + return errInventory + } + } + return nil +} + +func retainedConfigRange(config object, minGiB, maxGiB uint64) bool { + return verifyEncryptedPartitionConfig(config) == nil && + value(config, "spec", "provisioning", "partitionSpec", "minSize") == float64(minGiB<<30) && + value(config, "spec", "provisioning", "partitionSpec", "maxSize") == float64(maxGiB<<30) && + value(config, "spec", "provisioning", "partitionSpec", "negativeMaxSize") != true +} + +func (p temporaryPreparation) observeRetainedPreparation(ctx context.Context, owned, closed bool) (storageRootReceipt, error) { + scope := storageLayout + if closed { + scope = storageRetainedClosed + } + receipt, err := p.observeTemporaryStorage(ctx, owned, scope) + if err != nil { + return storageRootReceipt{}, err + } + layout, identity := receipt.BlockLayout, receipt.retainedLayout + // 144 GiB for new storage, 44 GiB for root and 8 GiB for placement. + if layout.EphemeralPhase != "ready" || !layout.EphemeralLUKS2 || !layout.EphemeralLockedToState || + !layout.EphemeralNodeIDOnly || layout.EphemeralSizeBytes < 196<<30 || + identity.EphemeralUUID != p.j.EphemeralUUID || identity.StagePartitionUUID != "" || !identity.StageStatusPresent || + (p.j.RetainedRawUUID != "" && !p.sameRetainedExtent(identity)) { + return storageRootReceipt{}, errInventory + } + return receipt, nil +} + +func (p temporaryPreparation) sameRetainedExtent(identity longhorninventory.RetainedLayoutIdentity) bool { + return identity.PartitionUUID == p.j.RetainedRawUUID && identity.ExtentDigest == p.j.RetainedExtentDigest +} + +func (p temporaryPreparation) closeRetainedForReset(ctx context.Context, address string) error { + if p.j.Schema != 3 || p.j.Phase != "draining" || p.verifyRetainedLayoutConfig(ctx, address, false) != nil { + return errInventory + } + if _, err := p.observeRetainedPreparation(ctx, true, true); err == nil { + return nil + } + if _, err := p.observeRetainedPreparation(ctx, true, false); err != nil { + return err + } + if err := p.verifyRetainedLayoutConfig(ctx, address, true); err != nil { + return err + } + if err := p.checkOwner(ctx); err != nil { + return err + } + if _, err := p.c.run(ctx, nil, "talosctl", "--nodes="+address, "patch", "machineconfig", "--mode=no-reboot", "--patch="+closeRetainedRawConfig); err != nil { + return errors.New("the original raw mapper closure could not be confirmed") + } + return waitReadyWithin(ctx, time.Minute, func(ctx context.Context) error { + _, err := p.observeRetainedPreparation(ctx, true, true) + return err + }) +} + +// Read only after a new boot. An absent old mapper can qualify the physical +// recovery, while the completed journal always requires it to be reopened. +func (p temporaryPreparation) readRetainedRecovery(ctx context.Context, address, boot string, closed bool) (identity longhorninventory.RetainedLayoutIdentity, err error) { + ctx, cancel := context.WithTimeout(ctx, 45*time.Second) + defer cancel() + factory := p.reader + if factory == nil { + factory = newInventoryReader + } + reader, closeReader, err := factory(ctx, address) + if err != nil || reader == nil || closeReader == nil { + return longhorninventory.RetainedLayoutIdentity{}, errInventory + } + defer func() { + if closeReader() != nil || !commandContextActive(ctx) { + identity, err = longhorninventory.RetainedLayoutIdentity{}, errInventory + } + }() + before, err := reader.BootID(ctx) + layoutReader, ok := reader.(nativeRetainedBlockLayoutReader) + if err != nil || before != boot || before == p.j.OriginalBootID || !ok { + return longhorninventory.RetainedLayoutIdentity{}, errInventory + } + if err = p.verifyRetainedLayoutConfig(ctx, address, !closed); err != nil { + return longhorninventory.RetainedLayoutIdentity{}, err + } + layout, identity, err := layoutReader.ListRetainedBlockLayout(ctx, closed) + if err != nil || !layout.RawExtentObserved || !p.sameRetainedExtent(identity) || + !inventoryBoot.MatchString(identity.StagePartitionUUID) || identity.EphemeralUUID == "" || + identity.EphemeralUUID == p.j.EphemeralUUID || layout.EphemeralPhase != "ready" || + layout.EphemeralSizeBytes < 40<<30 || layout.EphemeralSizeBytes > 44<<30 || + !layout.EphemeralLUKS2 || !layout.EphemeralLockedToState || !layout.EphemeralNodeIDOnly || + (p.j.PreparedRawUUID != "" && p.j.PreparedRawUUID != identity.StagePartitionUUID) { + return longhorninventory.RetainedLayoutIdentity{}, errInventory + } + after, err := reader.BootID(ctx) + if err != nil || after != before { + return longhorninventory.RetainedLayoutIdentity{}, errInventory + } + return identity, nil +} + +func (p temporaryPreparation) reopenRetained(ctx context.Context) (string, error) { + node, address, err := p.node(ctx, true) + if err != nil || p.j.Schema != 3 || p.j.Phase != "reset-requested" { + return "", errInventory + } + boot := text(node, "status", "nodeInfo", "bootID") + open, err := p.readRetainedRecovery(ctx, address, boot, false) + if err == nil { + return open.StagePartitionUUID, nil + } + closed, err := p.readRetainedRecovery(ctx, address, boot, true) + if err != nil { + return "", err + } + current, currentAddress, err := p.node(ctx, true) + if err != nil || currentAddress != address || text(current, "status", "nodeInfo", "bootID") != boot || + value(current, "spec", "unschedulable") != true || text(current, "metadata", "annotations", fenceAnnotation) != p.name { + return "", errInventory + } + storage, err := p.storageNode(ctx) + if err != nil || !unusedStorageFenced(storage) { + return "", errInventory + } + if err = p.checkOwner(ctx); err != nil { + return "", err + } + if _, err = p.c.run(ctx, nil, "talosctl", "--nodes="+address, "patch", "machineconfig", "--mode=no-reboot", + "--patch=@talos/workers/reserve-longhorn-data.yaml"); err != nil { + return "", errors.New("the original raw mapper reopening could not be confirmed") + } + err = waitReadyWithin(ctx, time.Minute, func(ctx context.Context) error { + open, err = p.readRetainedRecovery(ctx, address, boot, false) + if err != nil || open.StagePartitionUUID != closed.StagePartitionUUID { + return errInventory + } + return nil + }) + return open.StagePartitionUUID, err +} diff --git a/scripts/migrate-longhorn-v2/retained_prepare_test.go b/scripts/migrate-longhorn-v2/retained_prepare_test.go new file mode 100644 index 0000000000..41bec087d4 --- /dev/null +++ b/scripts/migrate-longhorn-v2/retained_prepare_test.go @@ -0,0 +1,153 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "github.com/devantler-tech/platform/internal/longhorninventory" + "github.com/devantler-tech/platform/internal/longhornmigration" +) + +const retainedFixtureStage = "12345678-1234-4234-8234-000000000090" + +type retainingFixture struct { + *unusedPreparationFixture + closed, uncertainClose, changedExtent bool + closes, reopens int +} + +type retainingReader struct { + *unusedParentReaderFixture + fixture *retainingFixture +} + +func (r *retainingReader) ListRetainedBlockLayout(_ context.Context, closed bool) (longhorninventory.BlockLayoutReport, longhorninventory.RetainedLayoutIdentity, error) { + f := r.fixture + if closed != f.closed { + return longhorninventory.BlockLayoutReport{}, longhorninventory.RetainedLayoutIdentity{}, errInventory + } + report := longhorninventory.BlockLayoutReport{RawExtentObserved: true, VolumeStatuses: 3, DiscoveredVolumes: 4, + EphemeralPhase: "ready", EphemeralSizeBytes: 197 << 30, EphemeralLUKS2: true, EphemeralLockedToState: true, + EphemeralNodeIDOnly: true, Digest: strings.Repeat("a", 64)} + identity := longhorninventory.RetainedLayoutIdentity{ + PartitionUUID: "12345678-1234-4234-8234-000000000080", ExtentDigest: strings.Repeat("b", 64), + StageStatusPresent: true, EphemeralUUID: journalFixture().EphemeralUUID, + } + if f.base.postReset { + identity.EphemeralUUID, identity.StagePartitionUUID = "recovered-ephemeral", retainedFixtureStage + report.EphemeralSizeBytes = 42 << 30 + } + if f.changedExtent { + identity.ExtentDigest = strings.Repeat("c", 64) + } + return report, identity, nil +} + +func retainingPreparationAPI(t *testing.T) (*retainingFixture, temporaryPreparation) { + t.Helper() + ordinary, p := unusedPreparationAPI(t) + f := &retainingFixture{unusedPreparationFixture: ordinary} + f.base.node, f.base.storage = f.base.s.nodes[5], f.base.s.storageNodes[5] + p.target, p.name, p.temporarySelection = text(f.base.node, "metadata", "name"), "longhorn-v2-prepare-temporary-v2-1", "temporary-v2-1" + p.staging, p.c.run = true, f.run + p.reader = func(_ context.Context, address string) (nativeReplicaReader, func() error, error) { + if address != "10.0.99.9" { + t.Fatal("retaining preparation selected another host") + } + f.nativeCalls++ + boot := text(f.base.node, "status", "nodeInfo", "bootID") + return &retainingReader{&unusedParentReaderFixture{temporaryNativeReader: temporaryNativeReader{boot: boot}, mode: "cache directories only"}, f}, func() error { return nil }, nil + } + return f, p +} + +func (f *retainingFixture) run(ctx context.Context, input []byte, program string, args ...string) ([]byte, error) { + if program != "talosctl" || len(args) < 2 { + return f.unusedPreparationFixture.run(ctx, input, program, args...) + } + if args[0] != "--nodes=10.0.99.9" { + f.base.t.Fatal("retaining preparation reached an unrelated host") + } + switch { + case args[1] == "patch": + var j journal + if len(args) != 5 || args[2] != "machineconfig" || args[3] != "--mode=no-reboot" || + json.Unmarshal([]byte(text(f.base.cm, "data", "record")), &j) != nil || j.Schema != 3 || + value(f.base.node, "spec", "unschedulable") != true { + f.base.t.Fatal("layout mutation lacked its original cordon, journal or host") + } + switch args[4] { + case "--patch=" + closeRetainedRawConfig: + if f.closed || f.base.postReset || j.Phase != "draining" { + f.base.t.Fatal("unsafe old mapper closure") + } + f.closed = true + f.closes++ + if f.uncertainClose { + return nil, errors.New("closure accepted but response lost") + } + case "--patch=@talos/workers/reserve-longhorn-data.yaml": + if !f.closed || !f.base.postReset || j.Phase != "reset-requested" { + f.base.t.Fatal("old mapper reopened before verified recovery") + } + f.closed = false + f.reopens++ + default: + f.base.t.Fatal("unexpected configuration mutation") + } + return nil, nil + case args[1] == "reset": + want := []string{"--nodes=10.0.99.9", "reset", "--system-labels-to-wipe=EPHEMERAL", "--reboot", "--graceful=true", "--wait", "--timeout=15m"} + var j journal + if !reflect.DeepEqual(args, want) || json.Unmarshal([]byte(text(f.base.cm, "data", "record")), &j) != nil || + j.Schema != 3 || j.Phase != "reset-requested" || !f.closed || f.base.resets != 0 { + f.base.t.Fatal("reset did not preserve both old partitions or preceded durable intent") + } + f.base.resets++ + f.base.postReset = true + valueObject(f.base.node, "status", "nodeInfo")["bootID"] = "12345678-1234-4234-8234-000000000099" + return nil, errors.New("reset accepted but response lost") + case len(args) == 5 && args[2] == "volumeconfig": + raw, ephemeral := desiredConfigFixture() + switch args[3] { + case longhornmigration.RetainedStageRawID: + raw["metadata"].(object)["id"] = longhornmigration.RetainedStageRawID + spec := valueObject(raw, "spec", "provisioning", "partitionSpec") + spec["minSize"], spec["maxSize"] = float64(144<<30), float64(144<<30) + return json.Marshal(raw) + case "EPHEMERAL": + spec := valueObject(ephemeral, "spec", "provisioning", "partitionSpec") + spec["maxSize"], spec["negativeMaxSize"] = float64(44<<30), false + return json.Marshal(ephemeral) + case "r-longhorn": + if f.closed { + return nil, errInventory + } + return json.Marshal(raw) + } + } + return f.unusedPreparationFixture.run(ctx, input, program, args...) +} + +func TestRetainingPreparationRecoversAnUncertainCloseAndNeverReplaysReset(t *testing.T) { + f, p := retainingPreparationAPI(t) + f.uncertainClose = true + if err := p.prepare(t.Context()); err == nil || f.closes != 1 || f.base.resets != 0 || p.j.Phase != "draining" { + t.Fatal("uncertain closure reached a reset", err) + } + if err := p.prepare(t.Context()); err != nil || f.closes != 1 || f.base.resets != 1 || f.reopens != 1 || + p.j.Phase != "prepared" || p.j.PreparedRawUUID != retainedFixtureStage || !unusedStorageFenced(f.base.storage) { + t.Fatalf("preserving recovery failed: phase=%s closes=%d resets=%d reopens=%d: %v", p.j.Phase, f.closes, f.base.resets, f.reopens, err) + } + if err := p.prepare(t.Context()); err != nil || f.base.resets != 1 || f.closes != 1 || f.reopens != 1 { + t.Fatal("a completed preparation replayed a host write", err) + } + f.changedExtent = true + if err := p.prepare(t.Context()); err == nil || f.base.resets != 1 { + t.Fatal("a changed retained extent qualified or replayed reset") + } +} diff --git a/scripts/migrate-longhorn-v2/storage_root_inventory.go b/scripts/migrate-longhorn-v2/storage_root_inventory.go index 1239a5465a..98977002e3 100644 --- a/scripts/migrate-longhorn-v2/storage_root_inventory.go +++ b/scripts/migrate-longhorn-v2/storage_root_inventory.go @@ -26,17 +26,26 @@ type storageRootReceipt struct { APIProjectionDigest string `json:"apiProjectionDigest"` Report longhorninventory.StorageRootReport `json:"report"` BlockLayout *longhorninventory.BlockLayoutReport `json:"blockLayout,omitempty"` + retainedLayout longhorninventory.RetainedLayoutIdentity } func observeStorageLayout(ctx context.Context, c client, target string, factory inventoryReaderFactory) (storageRootReceipt, error) { - return observeStorageRootScope(ctx, c, target, factory, true) + return observeStorageRootScope(ctx, c, target, factory, storageLayout) } func observeStorageRoot(ctx context.Context, c client, target string, factory inventoryReaderFactory) (storageRootReceipt, error) { - return observeStorageRootScope(ctx, c, target, factory, false) + return observeStorageRootScope(ctx, c, target, factory, storageRootOnly) } -func observeStorageRootScope(ctx context.Context, c client, target string, factory inventoryReaderFactory, includeBlocks bool) (receipt storageRootReceipt, err error) { +type storageObservationScope int + +const ( + storageRootOnly storageObservationScope = iota + storageLayout + storageRetainedClosed +) + +func observeStorageRootScope(ctx context.Context, c client, target string, factory inventoryReaderFactory, scope storageObservationScope) (receipt storageRootReceipt, err error) { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() expires, _ := ctx.Deadline() @@ -82,8 +91,9 @@ func observeStorageRootScope(ctx context.Context, c client, target string, facto return storageRootReceipt{}, stageOwnershipBefore } var blockLayout *longhorninventory.BlockLayoutReport - if includeBlocks { - layout, layoutErr := observeNativeBlockLayout(ctx, reader, target) + var retainedLayout longhorninventory.RetainedLayoutIdentity + if scope != storageRootOnly { + layout, retained, layoutErr := observeNativeBlockLayout(ctx, reader, target, scope == storageRetainedClosed) if layoutErr != nil { return storageRootReceipt{}, errors.Join(stageBlockLayout, layoutErr) } @@ -91,6 +101,7 @@ func observeStorageRootScope(ctx context.Context, c client, target string, facto return storageRootReceipt{}, stageBlockLayout } blockLayout = &layout + retainedLayout = retained } second, err := rootReader.ListStorageRoot(ctx) if err != nil { @@ -121,29 +132,31 @@ func observeStorageRootScope(ctx context.Context, c client, target string, facto return storageRootReceipt{}, stageDeadline } schema := 1 - if includeBlocks { + if scope != storageRootOnly { schema = 2 } - return storageRootReceipt{Schema: schema, ObservationOnly: true, ResetClearance: false, APIProjectionDigest: before.Digest, Report: report, BlockLayout: blockLayout}, nil + return storageRootReceipt{Schema: schema, ObservationOnly: true, ResetClearance: false, + APIProjectionDigest: before.Digest, Report: report, BlockLayout: blockLayout, retainedLayout: retainedLayout}, nil } -func observeNativeBlockLayout(ctx context.Context, reader nativeReplicaReader, target string) (longhorninventory.BlockLayoutReport, error) { +func observeNativeBlockLayout(ctx context.Context, reader nativeReplicaReader, target string, closed bool) (longhorninventory.BlockLayoutReport, longhorninventory.RetainedLayoutIdentity, error) { if target == "temporary-v2-1" { retainedReader, ok := reader.(nativeRetainedBlockLayoutReader) if !ok { - return longhorninventory.BlockLayoutReport{}, stageClient + return longhorninventory.BlockLayoutReport{}, longhorninventory.RetainedLayoutIdentity{}, stageClient } - layout, identity, err := retainedReader.ListRetainedBlockLayout(ctx, false) + layout, identity, err := retainedReader.ListRetainedBlockLayout(ctx, closed) if err == nil && (!layout.RawExtentObserved || !inventoryBoot.MatchString(identity.PartitionUUID) || len(identity.ExtentDigest) != 64) { - return longhorninventory.BlockLayoutReport{}, stageBlockLayout + return longhorninventory.BlockLayoutReport{}, longhorninventory.RetainedLayoutIdentity{}, stageBlockLayout } - return layout, err + return layout, identity, err } blockReader, ok := reader.(nativeBlockLayoutReader) - if !ok { - return longhorninventory.BlockLayoutReport{}, stageClient + if !ok || closed { + return longhorninventory.BlockLayoutReport{}, longhorninventory.RetainedLayoutIdentity{}, stageClient } - return blockReader.ListBlockLayout(ctx) + layout, err := blockReader.ListBlockLayout(ctx) + return layout, longhorninventory.RetainedLayoutIdentity{}, err } func authorizeStorageRoot(getenv func(string) string, target string) error { @@ -154,7 +167,7 @@ func authorizeStorageRoot(getenv func(string) string, target string) error { } func runStorageRootInventory(ctx context.Context, target, source, runID string) error { - return runStorageInventoryScope(ctx, target, source, runID, false) + return runStorageInventoryScope(ctx, target, source, runID, storageRootOnly) } func authorizeStorageLayout(getenv func(string) string, target string) error { @@ -168,10 +181,10 @@ func runStorageLayoutInventory(ctx context.Context, target, source, runID string if !inventoryTemporarySelection.MatchString(target) { return stageProjectionBefore } - return runStorageInventoryScope(ctx, target, source, runID, true) + return runStorageInventoryScope(ctx, target, source, runID, storageLayout) } -func runStorageInventoryScope(ctx context.Context, target, source, runID string, includeBlocks bool) error { +func runStorageInventoryScope(ctx context.Context, target, source, runID string, scope storageObservationScope) error { c := client{run: inventoryReadCommand, context: "admin@prod"} if inventoryTemporarySelection.MatchString(target) { config, err := os.ReadFile("ksail.prod.yaml") @@ -186,7 +199,7 @@ func runStorageInventoryScope(ctx context.Context, target, source, runID string, if err := sourceCurrent(ctx, c.run, source); err != nil { return stageSourceBefore } - receipt, err := observeStorageRootScope(ctx, c, target, newInventoryReader, includeBlocks) + receipt, err := observeStorageRootScope(ctx, c, target, newInventoryReader, scope) if err != nil { return err } diff --git a/scripts/migrate-longhorn-v2/temporary_activation_prepared.go b/scripts/migrate-longhorn-v2/temporary_activation_prepared.go index e6805956ce..140d8cb4f5 100644 --- a/scripts/migrate-longhorn-v2/temporary_activation_prepared.go +++ b/scripts/migrate-longhorn-v2/temporary_activation_prepared.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "strings" - - "github.com/devantler-tech/platform/internal/strictjson" ) // preparedTemporaryBinding validates a completed unused-storage reset journal @@ -16,10 +14,8 @@ func preparedTemporaryBinding(cm object, binding temporaryV2Binding) (journal, e text(cm, "metadata", "uid") == "" || text(cm, "metadata", "resourceVersion") == "" || value(cm, "metadata", "deletionTimestamp") != nil { return journal{}, errTemporaryActivation } - data := []byte(text(cm, "data", "record")) - fields := map[string][]string{"": {"schema", "target", "source", "ownerRun", "phase", "nodeUID", "providerID", "machineUUID", "longhornUID", "ephemeralUUID", "originalDisks", "temporarySelection", "nativeEmptyDigest", "originalBootID", "rawVolumeAbsent", "stagingLayout", "retainedRawUUID", "retainedExtentDigest", "preparedRawUUID"}} - var record journal - if len(data) == 0 || len(data) > 32<<10 || strictjson.Validate(data, fields) != nil || json.Unmarshal(data, &record) != nil { + record, err := decodePreparationJournal([]byte(text(cm, "data", "record"))) + if err != nil { return journal{}, errTemporaryActivation } p := preparation{target: binding.Name, name: name, temporarySelection: binding.Selection} diff --git a/scripts/migrate-longhorn-v2/temporary_authorization.go b/scripts/migrate-longhorn-v2/temporary_authorization.go index 0252d82b76..14e621b469 100644 --- a/scripts/migrate-longhorn-v2/temporary_authorization.go +++ b/scripts/migrate-longhorn-v2/temporary_authorization.go @@ -10,3 +10,10 @@ func authorizeTemporaryPreparation(getenv func(string) string, selection string) } return authorizeWorkflow(getenv, "prepare-unused-longhorn-v2-"+selection) } + +func authorizeRetainedTemporaryPreparation(getenv func(string) string, selection string) error { + if selection != "temporary-v2-1" || getenv("LONGHORN_V2_NODE") != selection || getenv("LONGHORN_V2_OPERATION") != "prepare-retained-temporary-worker" { + return errors.New("the separate retaining temporary-worker preparation selection is required") + } + return authorizeWorkflow(getenv, "prepare-retained-longhorn-v2-temporary-v2-1") +} diff --git a/scripts/migrate-longhorn-v2/temporary_authorization_test.go b/scripts/migrate-longhorn-v2/temporary_authorization_test.go index 87e00d08e2..e2cdb32b41 100644 --- a/scripts/migrate-longhorn-v2/temporary_authorization_test.go +++ b/scripts/migrate-longhorn-v2/temporary_authorization_test.go @@ -33,3 +33,20 @@ func TestUnusedTemporaryPreparationRequiresItsSeparateExactDispatch(t *testing.T } } } + +func TestRetainingPreparationUsesOnlyItsDistinctProtectedSelection(t *testing.T) { + env := dispatchFixture() + env["LONGHORN_V2_OPERATION"] = "prepare-retained-temporary-worker" + env["LONGHORN_V2_NODE"] = "temporary-v2-1" + env["LONGHORN_V2_CONFIRM"] = "prepare-retained-longhorn-v2-temporary-v2-1" + getenv := func(key string) string { return env[key] } + if authorizeRetainedTemporaryPreparation(getenv, "temporary-v2-1") != nil || + authorizeTemporaryPreparation(getenv, "temporary-v2-1") == nil || + authorizeRetainedTemporaryPreparation(getenv, "temporary-v1-1") == nil { + t.Fatal("retaining preparation lost its distinct protected scope") + } + env["GITHUB_RUN_ATTEMPT"] = "2" + if authorizeRetainedTemporaryPreparation(getenv, "temporary-v2-1") == nil { + t.Fatal("a rerun could request retaining preparation") + } +} diff --git a/scripts/migrate-longhorn-v2/temporary_cordon.go b/scripts/migrate-longhorn-v2/temporary_cordon.go index e58fb03c77..7ec5a1db6c 100644 --- a/scripts/migrate-longhorn-v2/temporary_cordon.go +++ b/scripts/migrate-longhorn-v2/temporary_cordon.go @@ -10,7 +10,7 @@ func temporaryInventoryNodes(nodes []object, record *journal) ([]object, error) if record == nil { return nodes, nil } - if record.Schema != 2 || validateJournal(*record, record.Target) != nil || (record.Phase != "evacuating" && record.Phase != "draining") { + if (record.Schema != 2 && record.Schema != 3) || validateJournal(*record, record.Target) != nil || (record.Phase != "evacuating" && record.Phase != "draining") { return nil, errInventory } result := append([]object(nil), nodes...) diff --git a/scripts/migrate-longhorn-v2/temporary_prepare.go b/scripts/migrate-longhorn-v2/temporary_prepare.go index 7c8d3189b1..34625c66f1 100644 --- a/scripts/migrate-longhorn-v2/temporary_prepare.go +++ b/scripts/migrate-longhorn-v2/temporary_prepare.go @@ -54,6 +54,14 @@ func (p *temporaryPreparation) prepare(ctx context.Context) error { if err := waitReady(ctx, func(ctx context.Context) error { _, err := p.unusedPostState(ctx); return err }); err != nil { return err } + var stagedUUID string + if p.retainsRaw() { + var err error + stagedUUID, err = p.reopenRetained(ctx) + if err != nil { + return err + } + } n, err := p.unusedPostState(ctx) if err != nil { return err @@ -66,7 +74,7 @@ func (p *temporaryPreparation) prepare(ctx context.Context) error { } else if fence != "" || value(n, "spec", "unschedulable") == true { return errInventory } - if err = p.advance(ctx, "prepared"); err != nil { + if err = p.advanceWithStage(ctx, "prepared", stagedUUID); err != nil { return err } } @@ -187,7 +195,13 @@ func (p *temporaryPreparation) beginUnused(ctx context.Context) error { if err != nil { return err } - if err = p.desiredLayout(ctx, address); err != nil { + if p.retainsRaw() { + p.j.Schema = 3 + err = p.verifyRetainedLayoutConfig(ctx, address, true) + } else { + err = p.desiredLayout(ctx, address) + } + if err != nil { return err } eph, err := p.c.talosVolume(ctx, address, "EPHEMERAL") @@ -195,6 +209,16 @@ func (p *temporaryPreparation) beginUnused(ctx context.Context) error { return errInventory } p.j.EphemeralUUID = text(eph, "spec", "uuid") + if p.retainsRaw() { + receipt, err := p.observeRetainedPreparation(ctx, false, false) + if err != nil { + return err + } + p.j.NativeEmptyDigest = temporaryStorageDigest(receipt) + p.j.RetainedRawUUID = receipt.retainedLayout.PartitionUUID + p.j.RetainedExtentDigest = receipt.retainedLayout.ExtentDigest + return p.claimRecord(ctx, p.j) + } digest, err := p.observeUnused(ctx, false) if err != nil { return err @@ -207,9 +231,27 @@ func (p *temporaryPreparation) beginUnused(ctx context.Context) error { // and API identity checks, then commits the original encrypted partition and // cache-only parent directory evidence into the journal's digest. func (p temporaryPreparation) observeUnused(ctx context.Context, owned bool) (string, error) { - if err := sourceCurrent(ctx, p.c.run, p.source); err != nil { + receipt, err := p.observeTemporaryStorage(ctx, owned, storageLayout) + if err != nil { return "", err } + // The allocated original partition alone must cover the new 152 GiB + // reservation and 40 GiB root minimum, with 8 GiB for geometry overhead. + if p.staging && receipt.BlockLayout.EphemeralSizeBytes < 200<<30 { + return "", errors.New("native storage cannot fit the reviewed staging layout") + } + return temporaryStorageDigest(receipt), nil +} + +func temporaryStorageDigest(receipt storageRootReceipt) string { + digest := sha256.Sum256([]byte(receipt.APIProjectionDigest + ":" + receipt.Report.Digest + ":" + receipt.BlockLayout.Digest)) + return hex.EncodeToString(digest[:]) +} + +func (p temporaryPreparation) observeTemporaryStorage(ctx context.Context, owned bool, scope storageObservationScope) (storageRootReceipt, error) { + if err := sourceCurrent(ctx, p.c.run, p.source); err != nil { + return storageRootReceipt{}, err + } c := p.c c.inventoryPools = p.pools if owned { @@ -217,34 +259,32 @@ func (p temporaryPreparation) observeUnused(ctx context.Context, owned bool) (st } before, n, err := c.inventoryProjection(ctx, p.temporarySelection) if err != nil { - return "", err + return storageRootReceipt{}, err } if _, err = verifyNode(n, p.j, true); err != nil || before.Registered || len(before.Replicas) != 0 || before.BootID != p.j.OriginalBootID || before.StorageUID != p.j.LonghornUID { - return "", errInventory + return storageRootReceipt{}, errInventory } factory := p.reader if factory == nil { - factory = func(ctx context.Context, address string) (nativeReplicaReader, func() error, error) { - return openInventoryReader(ctx, address, p.j.EphemeralUUID) + if p.temporarySelection == "temporary-v2-1" { + factory = newInventoryReader + } else { + factory = func(ctx context.Context, address string) (nativeReplicaReader, func() error, error) { + return openInventoryReader(ctx, address, p.j.EphemeralUUID) + } } } - receipt, err := observeStorageLayout(ctx, c, p.temporarySelection, factory) + receipt, err := observeStorageRootScope(ctx, c, p.temporarySelection, factory, scope) if err != nil { - return "", errors.New(inventoryFailureMessage(err)) + return storageRootReceipt{}, errors.New(inventoryFailureMessage(err)) } if receipt.APIProjectionDigest != before.Digest || !unusedParentAbsent(receipt.Report) || receipt.BlockLayout == nil { - return "", errInventory - } - // The allocated original partition alone must cover the new 152 GiB - // reservation and 40 GiB root minimum, with 8 GiB for geometry overhead. - if p.staging && receipt.BlockLayout.EphemeralSizeBytes < 200<<30 { - return "", errors.New("native storage cannot fit the reviewed staging layout") + return storageRootReceipt{}, errInventory } if err = sourceCurrent(ctx, p.c.run, p.source); err != nil { - return "", err + return storageRootReceipt{}, err } - digest := sha256.Sum256([]byte(receipt.APIProjectionDigest + ":" + receipt.Report.Digest + ":" + receipt.BlockLayout.Digest)) - return hex.EncodeToString(digest[:]), nil + return receipt, nil } // Only recognized non-volume cache directories may remain on this fixed, @@ -347,6 +387,20 @@ func (p *temporaryPreparation) resetUnused(ctx context.Context) error { if err = requireBenchmarkBaseline(ctx, p.c, s.nodes); err != nil { return err } + if p.retainsRaw() { + if pass == 0 { + err = p.closeRetainedForReset(ctx, address) + } else { + _, err = p.observeRetainedPreparation(ctx, true, true) + if err == nil { + err = p.verifyRetainedLayoutConfig(ctx, address, false) + } + } + if err != nil { + return err + } + continue + } if _, err = p.observeUnused(ctx, true); err != nil { return err } @@ -387,6 +441,23 @@ func (p temporaryPreparation) unusedPostState(ctx context.Context) (node object, if !ready(status) || !hugepagesReady(n) || !inventoryBoot.MatchString(boot) || boot == p.j.OriginalBootID { return nil, errInventory } + if p.retainsRaw() { + if _, err = p.readRetainedRecovery(ctx, address, boot, false); err != nil && p.j.Phase == "reset-requested" { + _, err = p.readRetainedRecovery(ctx, address, boot, true) + } + if err != nil { + return nil, err + } + lh, err := p.storageNode(ctx) + if err != nil || !unusedStorageFenced(lh) { + return nil, errInventory + } + current, currentAddress, err := p.node(ctx, true) + if err != nil || currentAddress != address || text(current, "status", "nodeInfo", "bootID") != boot { + return nil, errInventory + } + return current, nil + } factory := p.reader if factory == nil { factory = newInventoryReader diff --git a/scripts/migrate-longhorn-v2/temporary_recovery.go b/scripts/migrate-longhorn-v2/temporary_recovery.go index e2a3dbda9e..376cb50d9f 100644 --- a/scripts/migrate-longhorn-v2/temporary_recovery.go +++ b/scripts/migrate-longhorn-v2/temporary_recovery.go @@ -3,7 +3,7 @@ package main // resolveTemporaryResetRecovery selects only the original journaled pool member // after durable reset intent, allowing observation while that member recovers. func resolveTemporaryResetRecovery(selection string, nodes []object, pools map[string]migrationPool, record journal) (string, error) { - if record.Schema != 2 || record.Phase != "reset-requested" || record.TemporarySelection != selection || validateJournal(record, record.Target) != nil { + if (record.Schema != 2 && record.Schema != 3) || record.Phase != "reset-requested" || record.TemporarySelection != selection || validateJournal(record, record.Target) != nil { return "", errInventory } target, _, err := resolveTemporarySelection(selection, nodes, pools, &record) @@ -17,7 +17,11 @@ func resolveTemporaryResetRecovery(selection string, nodes []object, pools map[s // recovery. It proves no readiness and grants no new reset. The state machine's // reset-requested phase can only inspect post-state, never replay destruction. func verifyTemporaryResetMember(n object, pool migrationPool, record journal) error { - if pool.role != "v1" || pool.name != "autoscale-longhorn-v1" || text(n, "metadata", "labels", migrationPoolLabel) != pool.role || + role := "v1" + if record.Schema == 3 { + role = "v2" + } + if pool.role != role || pool.name != "autoscale-longhorn-"+role || text(n, "metadata", "labels", migrationPoolLabel) != pool.role || text(n, "metadata", "labels", "ksail.io/autoscaled") != "true" || text(n, "metadata", "labels", "node.kubernetes.io/instance-type") != pool.serverType || text(n, "metadata", "resourceVersion") == "" { return errInventory }