diff --git a/.github/workflows/prepare-longhorn-v2-node.yaml b/.github/workflows/prepare-longhorn-v2-node.yaml index bae2544077..e4ec99d84f 100644 --- a/.github/workflows/prepare-longhorn-v2-node.yaml +++ b/.github/workflows/prepare-longhorn-v2-node.yaml @@ -199,9 +199,16 @@ jobs: HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} run: ./scripts/use-prod-stable-api-endpoint.sh - name: Run the confirmed migration preparation + if: ${{ inputs.operation != 'benchmark-v1' }} env: GH_TOKEN: ${{ github.token }} run: go run ./scripts/migrate-longhorn-v2 + - name: Run the confirmed V1 benchmark + if: ${{ inputs.operation == 'benchmark-v1' }} + env: + GH_TOKEN: ${{ github.token }} + HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }} + run: go run ./scripts/migrate-longhorn-v2 - name: Remove runner credentials if: ${{ always() }} shell: bash diff --git a/go.mod b/go.mod index bfb375c312..5c9b262757 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.47.5 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 github.com/google/go-github/v88 v88.0.0 + k8s.io/apimachinery v0.37.2 k8s.io/client-go v0.37.2 ) @@ -99,7 +100,6 @@ require ( golang.org/x/term v0.47.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.3 // indirect - k8s.io/apimachinery v0.37.2 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect diff --git a/internal/namespaceinventory/credentials.go b/internal/namespaceinventory/credentials.go index 13d4875a05..713efe1034 100644 --- a/internal/namespaceinventory/credentials.go +++ b/internal/namespaceinventory/credentials.go @@ -3,7 +3,9 @@ package namespaceinventory import ( "errors" "io" + "net" "net/http" + "net/netip" "net/url" "os" "path/filepath" @@ -16,6 +18,7 @@ import ( ) var credentialPath = regexp.MustCompile(`^/dev/shm/longhorn-namespace-inventory\.[A-Za-z0-9]+/kubeconfig$`) +var benchmarkCredentialPath = regexp.MustCompile(`^/dev/shm/longhorn-v2\.[A-Za-z0-9]+/kubeconfig$`) var sourcePattern = regexp.MustCompile(`^[0-9a-f]{40}$`) var runPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`) @@ -84,6 +87,38 @@ func LoadRetainedEmailClient(getenv func(string) string) (*http.Client, *url.URL return newProductionClient(config) } +// LoadBenchmarkAdmissionClient only admits the protected migration benchmark +// operation. The caller must first join the selected address to the serving +// Kubernetes process and independently verified Hetzner server. +func LoadBenchmarkAdmissionClient(getenv func(string) string, directIP netip.Addr) (*http.Client, *url.URL, error) { + for key, want := range map[string]string{ + "GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": "devantler-tech/platform", "GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted", + "GITHUB_WORKFLOW_REF": "devantler-tech/platform/.github/workflows/prepare-longhorn-v2-node.yaml@refs/heads/main", + "LONGHORN_V2_CONFIRM": "benchmark-longhorn-storage", + "LONGHORN_V2_OPERATION": "benchmark-v1", + } { + if getenv(key) != want { + return nil, nil, errors.New("protected benchmark admission dispatch is unverified") + } + } + path := getenv("KUBECONFIG") + if !sourcePattern.MatchString(getenv("GITHUB_SHA")) || !runPattern.MatchString(getenv("GITHUB_RUN_ID")) || + !benchmarkCredentialPath.MatchString(path) || filepath.Dir(path) != getenv("LONGHORN_V2_CREDENTIAL_DIR") || + !directIP.Is4() || !directIP.IsGlobalUnicast() || directIP.IsPrivate() { + return nil, nil, errors.New("fixed benchmark admission credentials and direct IPv4 peer are required") + } + raw, err := readPrivateKubeconfig(path) + if err != nil { + return nil, nil, err + } + config, err := clientcmd.Load(raw) + if err != nil { + return nil, nil, errors.New("protected benchmark kubeconfig could not be decoded") + } + return newProductionClientAt(config, directIP) +} + // AuthorizeRetainedEmailDispatch refuses unless the environment is the first attempt of the // confirmed, maintainer-dispatched disposal workflow on main in this repository, with a // well-formed source commit and run id. @@ -142,6 +177,10 @@ func readPrivateKubeconfig(path string) (raw []byte, err error) { } func newProductionClient(raw *clientcmdapi.Config) (*http.Client, *url.URL, error) { + return newProductionClientAt(raw, netip.Addr{}) +} + +func newProductionClientAt(raw *clientcmdapi.Config, directIP netip.Addr) (*http.Client, *url.URL, error) { if raw == nil || raw.Contexts["admin@prod"] == nil { return nil, nil, errors.New("fixed production context is missing") } @@ -173,10 +212,42 @@ func newProductionClient(raw *clientcmdapi.Config) (*http.Client, *url.URL, erro config.Timeout = 20 * time.Second config.DisableCompression = true config.Proxy = func(*http.Request) (*url.URL, error) { return nil, nil } + if directIP.IsValid() { + if !directIP.Is4() || !directIP.IsGlobalUnicast() || directIP.IsPrivate() { + return nil, nil, errors.New("benchmark direct API address is unverified") + } + endpoint.Host = net.JoinHostPort(directIP.String(), "6443") + config.Host, config.ServerName = endpoint.String(), directIP.String() + } client, err := rest.HTTPClientFor(config) if err != nil { return nil, nil, errors.New("verified production HTTP transport could not be constructed") } client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + if directIP.IsValid() { + client.Transport = directAPIPeer{host: endpoint.Host, transport: client.Transport} + } return client, endpoint, nil } + +type directAPIPeer struct { + host string + transport http.RoundTripper +} + +func (peer directAPIPeer) RoundTrip(req *http.Request) (*http.Response, error) { + if req.URL.Scheme != "https" || req.URL.Host != peer.host || req.URL.User != nil || req.URL.Opaque != "" || + (req.Host != "" && req.Host != peer.host) { + return nil, errors.New("direct API request addressed another peer") + } + if req.Method != http.MethodGet && (req.Method != http.MethodPatch || req.URL.RawQuery != "dryRun=All") { + return nil, errors.New("direct API client permits only observations and server dry-runs") + } + return peer.transport.RoundTrip(req) +} + +func (peer directAPIPeer) CloseIdleConnections() { + if transport, ok := peer.transport.(interface{ CloseIdleConnections() }); ok { + transport.CloseIdleConnections() + } +} diff --git a/internal/namespaceinventory/credentials_test.go b/internal/namespaceinventory/credentials_test.go index 4a103c28a5..cc9887de69 100644 --- a/internal/namespaceinventory/credentials_test.go +++ b/internal/namespaceinventory/credentials_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/http/httptest" + "net/netip" "strings" "testing" @@ -123,3 +124,42 @@ func TestCredentialLoaderRefusesOutsideProtectedDispatch(t *testing.T) { t.Fatal("credential loader admitted an unverified dispatch") } } + +func TestBenchmarkDirectClientPinsAddressAndNeverFollowsRedirect(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("the old endpoint must not receive direct-client credentials") + })) + t.Cleanup(server.Close) + raw := fixtureConfig(server) + raw.Clusters["prod"].TLSServerName = "old-dns-name.invalid" + direct := netip.MustParseAddr("203.0.113.10") + client, endpoint, err := newProductionClientAt(raw, direct) + if err != nil || endpoint.String() != "https://203.0.113.10:6443" { + t.Fatalf("direct endpoint is not pinned: %v", err) + } + if _, err := client.Get(server.URL + "/version"); err == nil { + t.Fatal("direct client accepted a request for a different peer") + } + if client.CheckRedirect(nil, nil) != http.ErrUseLastResponse { + t.Fatal("direct client permits a redirect") + } + for _, test := range []struct{ method, query string }{ + {http.MethodPatch, ""}, + {http.MethodPatch, "?dryRun=All&dryRun="}, + {http.MethodDelete, "?dryRun=All"}, + } { + req, err := http.NewRequest(test.method, endpoint.String()+"/api/v1/persistentvolumes/scratch"+test.query, nil) + if err != nil { + t.Fatal(err) + } + if _, err := client.Do(req); err == nil || !strings.Contains(err.Error(), "only observations and server dry-runs") { + t.Fatal("direct API client did not reject a write-capable request locally") + } + } +} + +func TestBenchmarkCredentialLoaderRefusesOutsideProtectedDispatch(t *testing.T) { + if _, _, err := LoadBenchmarkAdmissionClient(func(string) string { return "" }, netip.MustParseAddr("203.0.113.10")); err == nil { + t.Fatal("direct credential loader admitted an unverified dispatch") + } +} diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/kustomization.yaml b/k8s/providers/hetzner/infrastructure/controllers/longhorn/kustomization.yaml index ff1d681b2b..4bb6f03421 100644 --- a/k8s/providers/hetzner/infrastructure/controllers/longhorn/kustomization.yaml +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/kustomization.yaml @@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - namespace.yaml + - namespace-benchmark.yaml - helm-repository.yaml - helm-release.yaml # Default WaitForFirstConsumer StorageClass (see storage-class-longhorn-wffc.yaml); diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/namespace-benchmark.yaml b/k8s/providers/hetzner/infrastructure/controllers/longhorn/namespace-benchmark.yaml new file mode 100644 index 0000000000..c4f5ff3427 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/namespace-benchmark.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: longhorn-benchmark + labels: + pod-security.kubernetes.io/enforce: restricted + pod-security.kubernetes.io/enforce-version: latest + velero.io/exclude-from-backup: "true" + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/scripts/migrate-longhorn-v2/README.md b/scripts/migrate-longhorn-v2/README.md index 9cc1a04b89..24500a2acb 100644 --- a/scripts/migrate-longhorn-v2/README.md +++ b/scripts/migrate-longhorn-v2/README.md @@ -232,10 +232,11 @@ rebuilds, volume health and the recorded configuration are observed. Scratch obj from backups. Compare receipts knowing that, and avoid dispatching around the 02:17 UTC backup, whose load on the same disks is not measured. V2's necessary CPU reservation is reported alongside the V1 context; the comparison does not assume the engines have identical CPU costs. -A completed receipt requires healthy storage and -explicit cleanup of only the owned scratch Pod, claim, retained PV, backend, -class and namespace. An incomplete run, including one that reaches the Pod's 25 minute -deadline, holds with its journal and resources preserved for explicit recovery. +A completed receipt requires healthy storage and explicit cleanup of the recorded +scratch Pod, network policy, claim, retained PV, backend and class. Both phases use +the GitOps-managed `longhorn-benchmark` namespace. The workflow records its UID and +leaves the namespace in place. An incomplete run, including one that reaches the +Pod's 25 minute deadline, holds with its journal and resources preserved for explicit recovery. Run `benchmark-v2` only after every volume uses V2 and every replica is back on the same three permanent workers. It requires the baseline's hardware, OS, diff --git a/scripts/migrate-longhorn-v2/benchmark.go b/scripts/migrate-longhorn-v2/benchmark.go index bd36de9610..e1add2715b 100644 --- a/scripts/migrate-longhorn-v2/benchmark.go +++ b/scripts/migrate-longhorn-v2/benchmark.go @@ -8,8 +8,9 @@ import ( "errors" "fmt" "io" + "maps" "reflect" - "regexp" + "slices" "sort" "strings" "time" @@ -22,9 +23,13 @@ const benchmarkOwner = "storage.platform.devantler.tech/benchmark-owner" type benchmarkMachine struct{ ProviderID, SystemUUID, OSImage, Architecture string } type benchmarkResource struct{ Name, Namespace, UID string } type benchmarkRecord struct { - Recovery *benchmarkRecoveryOwner `json:",omitempty"` - PendingCreate string `json:",omitempty"` - TagNodes map[string]string `json:",omitempty"` + Recovery *benchmarkRecoveryOwner `json:",omitempty"` + V1Runtime *benchmarkV1Runtime `json:",omitempty"` + Fences map[string]benchmarkFenceReceipt `json:",omitempty"` + FenceCheckpoint string `json:",omitempty"` + FencePending string `json:",omitempty"` + PendingCreate string `json:",omitempty"` + TagNodes map[string]string `json:",omitempty"` Resources map[string]benchmarkResource Version int Phase, Source, Run, Image, Workload, Namespace, NamespaceUID, CompletedAt string @@ -94,8 +99,10 @@ func readBenchmark(ctx context.Context, c client, phase string) (object, benchma var record benchmarkRecord raw := text(cm, "data", "record") fields := map[string][]string{ - "": {"Recovery", "PendingCreate", "TagNodes", "Resources", "Version", "Phase", "Source", "Run", "Image", "Workload", "Namespace", "NamespaceUID", "CompletedAt", "Machines", "Measurements", "Cleaned", "CPU"}, - "Recovery": {"Source", "Run", "JournalUID", "Deletes"}, + "": {"Recovery", "V1Runtime", "Fences", "FenceCheckpoint", "FencePending", "PendingCreate", "TagNodes", "Resources", "Version", "Phase", "Source", "Run", "Image", "Workload", "Namespace", "NamespaceUID", "CompletedAt", "Machines", "Measurements", "Cleaned", "CPU"}, + "Recovery": {"Source", "Run", "JournalUID", "Deletes"}, + "V1Runtime": {"Image", "SettingUID", "SettingVersion", "Epochs"}, + "V1Runtime.Epochs[]": {"Pod", "PodUID", "ManagerUID", "Node", "ContainerID", "StartedAt"}, } for _, kind := range []string{"Namespace", "NetworkPolicy", "Pod", "PersistentVolumeClaim", "StorageClass", "PersistentVolume", "LonghornVolume"} { fields["Resources."+kind] = []string{"Name", "Namespace", "UID"} @@ -113,8 +120,8 @@ func readBenchmark(ctx context.Context, c client, phase string) (object, benchma return nil, record, errors.New("benchmark journal is malformed") } var extra any - if d.Decode(&extra) != io.EOF || record.Version != 2 || record.Phase != phase || record.Image != fioImage || record.Workload != benchmarkWorkload() || - !regexp.MustCompile(`^[0-9a-f]{40}$`).MatchString(record.Source) || !regexp.MustCompile(`^[1-9][0-9]*$`).MatchString(record.Run) || len(record.Machines) != 3 || record.Namespace != "longhorn-benchmark-"+phase+"-"+record.Run { + if d.Decode(&extra) != io.EOF || !benchmarkNamespaceMatches(record) || record.Phase != phase || record.Image != fioImage || record.Workload != benchmarkWorkload() || + !validBenchmarkOwner(record.Run, record.Source) || len(record.Machines) != 3 { return nil, record, errors.New("benchmark journal does not match this reviewed procedure") } for _, name := range []string{"prod-worker-1", "prod-worker-2", "prod-worker-3"} { @@ -129,6 +136,20 @@ func readBenchmark(ctx context.Context, c client, phase string) (object, benchma if err = validateBenchmarkResources(record); err != nil { return nil, record, err } + if err = validateBenchmarkFenceReceipts(record); err != nil { + return nil, record, err + } + if err = validateBenchmarkRuntimeReceipt(record); err != nil { + return nil, record, err + } + if len(record.Fences) != 0 { + for _, name := range benchmarkFenceNames(phase, record.Run) { + fields["Fences."+name] = []string{"PolicyUID", "BindingUID"} + } + if strictjson.Validate([]byte(raw), fields) != nil { + return nil, record, errors.New("benchmark fence receipts are ambiguous") + } + } if record.Recovery != nil && (!validBenchmarkOwner(record.Recovery.Run, record.Recovery.Source) || record.Recovery.Run == record.Run || record.Recovery.JournalUID != text(cm, "metadata", "uid") || record.Cleaned) { return nil, record, errors.New("benchmark recovery ownership is invalid") } @@ -303,6 +324,14 @@ func runBenchmark(ctx context.Context, c client, source, runID, phase string) er return err } } + var v1Runtime *benchmarkV1Runtime + if phase == "v1" { + runtime, captureErr := captureBenchmarkV1Runtime(ctx, c, slices.Sorted(maps.Keys(machines))) + if captureErr != nil { + return captureErr + } + v1Runtime = &runtime + } prior, _, err := readBenchmark(ctx, c, phase) if err != nil { return err @@ -310,16 +339,31 @@ func runBenchmark(ctx context.Context, c client, source, runID, phase string) er if prior != nil { return errors.New("benchmark journal already exists; inspect its completion and scratch-resource state before any recovery") } - tag := "longhorn-benchmark-" + phase + "-" + runID - objects, err := benchmarkResources(phase, runID, tag) + tag := benchmarkTag(phase, runID) + objects, err := benchmarkResources(phase, runID, benchmarkNamespace) if err != nil { return err } + namespace, err := observeBenchmarkWorkspace(ctx, c, "") + if err != nil { + return err + } + for _, asset := range objects { + existing, err := c.get(ctx, strings.ToLower(text(asset, "kind")), text(asset, "metadata", "namespace"), text(asset, "metadata", "name"), true) + if err != nil || existing != nil { + return errors.New("a prior or foreign benchmark asset already occupies the reviewed scope") + } + } tagNodes, err := benchmarkTagNodes(s.storageNodes) if err != nil { return err } - record := benchmarkRecord{Version: 2, Phase: phase, Source: source, Run: runID, Image: fioImage, Workload: benchmarkWorkload(), Namespace: tag, Machines: machines, TagNodes: tagNodes} + namespaceIdentity := benchmarkIdentity(namespace) + record := benchmarkRecord{Version: 3, Phase: phase, Source: source, Run: runID, Image: fioImage, Workload: benchmarkWorkload(), Namespace: benchmarkNamespace, NamespaceUID: namespaceIdentity.UID, + Resources: map[string]benchmarkResource{"Namespace": namespaceIdentity}, Machines: machines, TagNodes: tagNodes, V1Runtime: v1Runtime} + if err := validateBenchmarkRuntimeReceipt(record); err != nil { + return err + } body, _ := json.Marshal(record) cm, err := c.create(ctx, source, object{"apiVersion": "v1", "kind": "ConfigMap", "metadata": object{"name": "longhorn-v2-benchmark-" + phase, "namespace": "longhorn-system"}, "data": object{"record": string(body)}}) if err != nil { @@ -339,14 +383,14 @@ func runBenchmark(ctx context.Context, c client, source, runID, phase string) er pod := created["Pod"] var claim, pv, volume object if err = waitReady(ctx, func(ctx context.Context) error { - current, err := c.get(ctx, "pod", tag, "fio", false) + current, err := c.get(ctx, "pod", record.Namespace, "fio", false) if err != nil || text(current, "status", "phase") != "Running" { return errors.New("benchmark consumer is not running") } if err = verifyBenchmarkPod(current, text(pod, "metadata", "uid")); err != nil { return err } - claim, err = c.get(ctx, "persistentvolumeclaim", tag, "fio-data", false) + claim, err = c.get(ctx, "persistentvolumeclaim", record.Namespace, "fio-data", false) if err != nil || text(claim, "metadata", "uid") != text(created["PersistentVolumeClaim"], "metadata", "uid") || text(claim, "status", "phase") != "Bound" { return errors.New("benchmark claim has not bound") } @@ -369,7 +413,7 @@ func runBenchmark(ctx context.Context, c client, source, runID, phase string) er } record.Resources["PersistentVolume"] = benchmarkIdentity(pv) record.Resources["LonghornVolume"] = benchmarkIdentity(volume) - pod, err = c.get(ctx, "pod", tag, "fio", false) + pod, err = c.get(ctx, "pod", record.Namespace, "fio", false) if err != nil { return err } @@ -398,20 +442,20 @@ func runBenchmark(ctx context.Context, c client, source, runID, phase string) er if err = verifyTrialComparable(start, phase, machines, scratchName, replicas); err != nil { return err } - current, err := c.get(ctx, "pod", tag, "fio", false) + current, err := c.get(ctx, "pod", record.Namespace, "fio", false) if err != nil { return err } if err = verifyBenchmarkPod(current, text(pod, "metadata", "uid")); err != nil { return err } - if _, err = c.kubectl(ctx, nil, "exec", "--namespace="+tag, "fio", "--container=fio", "--", "/bin/sh", "-ec", "touch /tmp/benchmark-start"); err != nil { + if _, err = c.kubectl(ctx, nil, "exec", "--namespace="+record.Namespace, "fio", "--container=fio", "--", "/bin/sh", "-ec", "touch /tmp/benchmark-start"); err != nil { return err } // The Pod's own deadline ends the trials first, and it reports Failed when it does. sampled, unproven := time.Now(), 0 if err = waitReadyWithin(ctx, benchmarkCompletionLimit, func(ctx context.Context) error { - p, err := c.get(ctx, "pod", tag, "fio", false) + p, err := c.get(ctx, "pod", record.Namespace, "fio", false) if err != nil { return err } @@ -437,7 +481,7 @@ func runBenchmark(ctx context.Context, c client, source, runID, phase string) er }); err != nil { return err } - logs, err := c.kubectl(ctx, nil, "logs", "--namespace="+tag, "fio", "--container=fio") + logs, err := c.kubectl(ctx, nil, "logs", "--namespace="+record.Namespace, "fio", "--container=fio") if err != nil { return err } @@ -462,7 +506,7 @@ func runBenchmark(ctx context.Context, c client, source, runID, phase string) er }); err != nil { return err } - current, err = c.get(ctx, "pod", tag, "fio", false) + current, err = c.get(ctx, "pod", record.Namespace, "fio", false) if err != nil { return err } @@ -473,30 +517,22 @@ func runBenchmark(ctx context.Context, c client, source, runID, phase string) er if err != nil || !reflect.DeepEqual(endingCPU, record.CPU) { return errors.New("admitted client or storage CPU context changed during the benchmark") } - if err = cleanupBenchmark(ctx, c, source, tag, created, pv, volume); err != nil { + if err = cleanupBenchmark(ctx, c, &record, created, claim, pv, volume); err != nil { return err } if err = benchmarkTags(ctx, c, source, tag, record.TagNodes, false); err != nil { return err } - record.NamespaceUID, record.Cleaned, record.CompletedAt = text(created["Namespace"], "metadata", "uid"), true, time.Now().UTC().Format(time.RFC3339) - body, _ = json.Marshal(record) - cm, err = c.get(ctx, "configmap", "longhorn-system", text(cm, "metadata", "name"), false) - if err != nil { - return err - } - var original benchmarkRecord - if json.Unmarshal([]byte(text(cm, "data", "record")), &original) != nil || original.Run != runID || original.Source != source || original.Cleaned { + currentJournal, original, err := readBenchmark(ctx, c, phase) + if err != nil || text(currentJournal, "metadata", "uid") != text(cm, "metadata", "uid") || !reflect.DeepEqual(original, record) { return errors.New("benchmark journal ownership changed") } - if err = sourceCurrent(ctx, c.run, source); err != nil { - return err - } - if err = c.patch(ctx, "configmap", "longhorn-system", cm, object{"op": "replace", "path": "/data/record", "value": string(body)}); err != nil { + record.Cleaned, record.CompletedAt = true, time.Now().UTC().Format(time.RFC3339) + if err = writeBenchmarkProgress(ctx, c, currentJournal, source, record); err != nil { return err } - _, checked, err := readBenchmark(ctx, c, phase) - if err != nil || !reflect.DeepEqual(checked, record) { + checkedJournal, checked, err := readBenchmark(ctx, c, phase) + if err != nil || text(checkedJournal, "metadata", "uid") != text(cm, "metadata", "uid") || !reflect.DeepEqual(checked, record) { return errors.New("completed benchmark receipt could not be read back") } fmt.Printf("Longhorn %s: completed eighteen controlled fio trials; scratch resources removed.\n", phase) @@ -593,6 +629,9 @@ func validateBenchmarkResources(record benchmarkRecord) error { if ns, present := record.Resources["Namespace"]; present && ns.UID != record.NamespaceUID { return errors.New("benchmark namespace identity disagrees with its journal") } + if record.Version == 3 && (record.Resources["Namespace"].UID == "" || record.PendingCreate == "Namespace") { + return errors.New("the GitOps benchmark namespace must have a prior UID observation") + } return nil } @@ -607,11 +646,21 @@ func saveBenchmarkProgress(ctx context.Context, c client, source, runID string, if !reflect.DeepEqual(previous.TagNodes, record.TagNodes) { return errors.New("benchmark tag node identities changed") } + if !reflect.DeepEqual(previous.Fences, record.Fences) || previous.FenceCheckpoint != record.FenceCheckpoint || previous.FencePending != record.FencePending { + return errors.New("benchmark fence receipts changed outside their journal transition") + } + if !reflect.DeepEqual(previous.V1Runtime, record.V1Runtime) { + return errors.New("benchmark retirement runtime receipts changed") + } for kind, old := range previous.Resources { if record.Resources[kind] != old { return errors.New("benchmark scratch identity changed") } } + return writeBenchmarkProgress(ctx, c, cm, source, record) +} + +func writeBenchmarkProgress(ctx context.Context, c client, cm object, source string, record benchmarkRecord) error { body, err := json.Marshal(record) if err != nil { return err diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_install.go b/scripts/migrate-longhorn-v2/benchmark_admission_install.go new file mode 100644 index 0000000000..ea081cd35b --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_install.go @@ -0,0 +1,244 @@ +package main + +import ( + "context" + "errors" + "maps" + "os" + "reflect" + "slices" + "strings" +) + +const benchmarkAdmissionAPI = "admissionregistration.k8s.io/v1" + +type benchmarkFenceReceipt struct{ PolicyUID, BindingUID string } + +func benchmarkFenceNames(phase, run string) []string { + suffix := "-" + phase + "-" + run + return []string{"lh-benchmark-retire-pv" + suffix, "lh-benchmark-retire-backend" + suffix, "lh-benchmark-retire-instances" + suffix} +} + +func validateBenchmarkFenceReceipts(record benchmarkRecord) error { + if record.Version == 3 && record.Phase == "v1" && record.Cleaned && len(record.Fences) != len(benchmarkFenceNames(record.Phase, record.Run)) { + return errors.New("a completed V1 benchmark needs all original retirement fence receipts") + } + if record.FencePending == "" && record.FenceCheckpoint == "" && len(record.Fences) == 0 { + return nil + } + original := record + original.Cleaned = false + if record.Phase != "v1" || !validBenchmarkOwner(record.Run, record.Source) || verifyRetirementReceipts(original) != nil { + return errors.New("benchmark fence journal has no verified original scratch") + } + names := benchmarkFenceNames(record.Phase, record.Run) + if record.FenceCheckpoint == "" && (record.Fences[names[2]].PolicyUID != "" || strings.HasPrefix(record.FencePending, names[2]+"/")) || + record.FenceCheckpoint != "" && (!benchmarkCheckpointPattern.MatchString(record.FenceCheckpoint) || !strings.HasPrefix(record.FenceCheckpoint, record.Run+"-")) { + return errors.New("benchmark instance checkpoint is invalid") + } + seen := map[string]bool{} + for _, resource := range record.Resources { + seen[resource.UID] = true + } + for name, receipt := range record.Fences { + if !slices.Contains(names, name) || receipt.PolicyUID == "" || receipt.PolicyUID == receipt.BindingUID || + seen[receipt.PolicyUID] || seen[receipt.BindingUID] || record.Cleaned && receipt.BindingUID == "" { + return errors.New("benchmark fence UID receipt is invalid") + } + seen[receipt.PolicyUID] = true + if receipt.BindingUID != "" { + seen[receipt.BindingUID] = true + } + } + if record.FencePending == "" { + return nil + } + name, kind, ok := strings.Cut(record.FencePending, "/") + receipt := record.Fences[name] + expectedKind := "policy" + if receipt.PolicyUID != "" { + expectedKind = "binding" + } + if record.Cleaned || !ok || !slices.Contains(names, name) || receipt.BindingUID != "" || kind != expectedKind { + return errors.New("benchmark fence creation intent is invalid") + } + return nil +} + +func benchmarkDenyBinding(name, policyUID string) object { + return object{ + "apiVersion": benchmarkAdmissionAPI, "kind": "ValidatingAdmissionPolicyBinding", + "metadata": object{ + "name": name, + "ownerReferences": []any{object{ + "apiVersion": benchmarkAdmissionAPI, "kind": "ValidatingAdmissionPolicy", "name": name, "uid": policyUID, + }}, + }, + "spec": object{"policyName": name, "validationActions": []any{"Deny"}}, + } +} + +func matchesOwnedBenchmarkPolicy(policy, desired object, uid string) bool { + return uid != "" && text(policy, "metadata", "uid") == uid && + text(policy, "metadata", "resourceVersion") != "" && value(policy, "metadata", "deletionTimestamp") == nil && + text(policy, "apiVersion") == benchmarkAdmissionAPI && text(policy, "kind") == "ValidatingAdmissionPolicy" && + text(policy, "metadata", "name") == text(desired, "metadata", "name") && + reflect.DeepEqual(value(policy, "spec"), value(desired, "spec")) +} + +func matchesOwnedBenchmarkFence(policy, binding, desired object, receipt benchmarkFenceReceipt) bool { + expected := benchmarkDenyBinding(text(desired, "metadata", "name"), receipt.PolicyUID) + return matchesOwnedBenchmarkPolicy(policy, desired, receipt.PolicyUID) && + receipt.BindingUID != "" && receipt.BindingUID != receipt.PolicyUID && text(binding, "metadata", "uid") == receipt.BindingUID && + text(binding, "metadata", "resourceVersion") != "" && value(binding, "metadata", "deletionTimestamp") == nil && + text(binding, "apiVersion") == benchmarkAdmissionAPI && text(binding, "kind") == "ValidatingAdmissionPolicyBinding" && + text(binding, "metadata", "name") == text(expected, "metadata", "name") && + reflect.DeepEqual(value(binding, "metadata", "ownerReferences"), value(expected, "metadata", "ownerReferences")) && + reflect.DeepEqual(value(binding, "spec"), value(expected, "spec")) +} + +// A failed or ambiguous CREATE leaves durable intent. A new object with the +// right name or body is never adopted without its original API UID receipt. +func installBenchmarkReferenceFence(ctx context.Context, c client, record *benchmarkRecord, desired object) error { + policies, err := benchmarkRetirementFences(*record) + if err != nil { + return err + } + if !reflect.DeepEqual(desired, policies[0]) && !reflect.DeepEqual(desired, policies[1]) { + return errors.New("benchmark fence is outside the reviewed reference scope") + } + return installBenchmarkFence(ctx, c, record, desired) +} + +func activeBenchmarkV1FenceOwner(c client, record benchmarkRecord) bool { + return c.context == "admin@prod" && authorizeBenchmark(os.Getenv, "v1") == nil && + record.Phase == "v1" && record.Source == os.Getenv("GITHUB_SHA") && record.Run == os.Getenv("GITHUB_RUN_ID") && + record.Recovery == nil && !record.Cleaned && record.FencePending == "" +} + +func installBenchmarkFence(ctx context.Context, c client, record *benchmarkRecord, desired object) error { + name := text(desired, "metadata", "name") + if !activeBenchmarkV1FenceOwner(c, *record) || !slices.Contains(benchmarkFenceNames(record.Phase, record.Run), name) { + return errors.New("benchmark fence installation has no verified workflow owner or target") + } + receipt := record.Fences[name] + if receipt.BindingUID != "" { + _, err := observeBenchmarkReferenceFence(ctx, c, desired, receipt) + return err + } + if receipt.PolicyUID == "" { + next := *record + next.FencePending = name + "/policy" + if err := advanceBenchmarkFenceJournal(ctx, c, record, next); err != nil { + return err + } + policy, err := c.create(ctx, record.Source, desired) + if err != nil { + return err + } + receipt.PolicyUID = text(policy, "metadata", "uid") + if !matchesOwnedBenchmarkPolicy(policy, desired, receipt.PolicyUID) { + return errors.New("benchmark policy creation readback differs") + } + if err := saveBenchmarkFenceReceipt(ctx, c, record, name, receipt); err != nil { + return err + } + } + policy, err := c.get(ctx, "validatingadmissionpolicies.admissionregistration.k8s.io", "", name, false) + if err != nil || !matchesOwnedBenchmarkPolicy(policy, desired, receipt.PolicyUID) { + return errors.New("original benchmark fence policy is unverified") + } + next := *record + next.FencePending = name + "/binding" + if err := advanceBenchmarkFenceJournal(ctx, c, record, next); err != nil { + return err + } + binding, err := c.create(ctx, record.Source, benchmarkDenyBinding(name, receipt.PolicyUID)) + if err != nil { + return err + } + receipt.BindingUID = text(binding, "metadata", "uid") + if !matchesOwnedBenchmarkFence(policy, binding, desired, receipt) { + return errors.New("benchmark binding creation readback differs") + } + if err := saveBenchmarkFenceReceipt(ctx, c, record, name, receipt); err != nil { + return err + } + _, err = observeBenchmarkReferenceFence(ctx, c, desired, receipt) + return err +} + +func saveBenchmarkFenceReceipt(ctx context.Context, c client, record *benchmarkRecord, name string, receipt benchmarkFenceReceipt) error { + next := *record + next.Fences = maps.Clone(record.Fences) + if next.Fences == nil { + next.Fences = map[string]benchmarkFenceReceipt{} + } + next.Fences[name], next.FencePending = receipt, "" + return advanceBenchmarkFenceJournal(ctx, c, record, next) +} + +func advanceBenchmarkFenceJournal(ctx context.Context, c client, record *benchmarkRecord, next benchmarkRecord) error { + cm, actual, err := readBenchmark(ctx, c, record.Phase) + if err != nil || cm == nil || !reflect.DeepEqual(actual, *record) { + return errors.New("benchmark journal changed before recording its fence") + } + if err := validateBenchmarkFenceReceipts(next); err != nil { + return err + } + if err := writeBenchmarkProgress(ctx, c, cm, record.Source, next); err != nil { + return err + } + checked, actual, err := readBenchmark(ctx, c, record.Phase) + if err != nil || text(checked, "metadata", "uid") != text(cm, "metadata", "uid") || !reflect.DeepEqual(actual, next) { + return errors.New("benchmark fence progress readback changed") + } + *record = next + return nil +} + +func removeCompletedBenchmarkFences(ctx context.Context, c client, record benchmarkRecord) error { + if !activeBenchmarkV1FenceOwner(c, record) || len(record.Fences) != len(benchmarkFenceNames(record.Phase, record.Run)) { + return errors.New("benchmark fence cleanup has no complete original receipts") + } + groups, _, err := readBenchmarkInstanceGroups(ctx, c) + if err != nil { + return err + } + children, err := qualifyRetirementInstances(record, groups) + if err != nil || len(children) != 0 { + return errors.New("native V1 benchmark children still depend on their retirement fence") + } + const bindingResource = "validatingadmissionpolicybindings.admissionregistration.k8s.io" + const policyResource = "validatingadmissionpolicies.admissionregistration.k8s.io" + bindings, err := c.list(ctx, bindingResource, "") + if err != nil { + return err + } + for _, binding := range bindings { + name := text(binding, "spec", "policyName") + if receipt, managed := record.Fences[name]; managed && text(binding, "metadata", "uid") != receipt.BindingUID { + return errors.New("a foreign binding references an original benchmark policy") + } + } + for _, name := range benchmarkFenceNames(record.Phase, record.Run) { + receipt := record.Fences[name] + binding, err := c.get(ctx, bindingResource, "", name, false) + if err != nil || text(binding, "metadata", "uid") != receipt.BindingUID || receipt.BindingUID == "" || + text(binding, "spec", "policyName") != name { + return errors.New("the original benchmark policy binding changed before retirement") + } + policy, err := c.get(ctx, policyResource, "", name, false) + if err != nil || text(policy, "metadata", "uid") != receipt.PolicyUID || receipt.PolicyUID == "" { + return errors.New("the original benchmark policy changed before retirement") + } + base := "/apis/" + benchmarkAdmissionAPI + "/" + if err := removeScratch(ctx, c, record.Source, bindingResource, "", base+"validatingadmissionpolicybindings/"+name, binding); err != nil { + return err + } + if err := removeScratch(ctx, c, record.Source, policyResource, "", base+"validatingadmissionpolicies/"+name, policy); err != nil { + return err + } + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_install_test.go b/scripts/migrate-longhorn-v2/benchmark_admission_install_test.go new file mode 100644 index 0000000000..6d73778566 --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_install_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "maps" + "testing" +) + +func TestBenchmarkReferenceFenceRejectsUnreceiptedPolicy(t *testing.T) { + const name = "lh-benchmark-retire-pv-v1-102" + desired := object{ + "apiVersion": "admissionregistration.k8s.io/v1", + "kind": "ValidatingAdmissionPolicy", + "metadata": object{"name": name}, + "spec": object{"failurePolicy": "Fail"}, + } + policy := object{ + "apiVersion": desired["apiVersion"], "kind": desired["kind"], "spec": desired["spec"], + "metadata": object{"name": name, "uid": "policy-uid", "resourceVersion": "17"}, + } + binding := benchmarkDenyBinding(name, "policy-uid") + bindingMetadata := value(binding, "metadata").(object) + bindingMetadata["uid"], bindingMetadata["resourceVersion"] = "binding-uid", "25" + receipts := benchmarkFenceReceipt{PolicyUID: "policy-uid", BindingUID: "binding-uid"} + if !matchesOwnedBenchmarkFence(policy, binding, desired, receipts) { + t.Fatal("the exact journaled policy and binding must qualify") + } + if matchesOwnedBenchmarkFence(policy, binding, desired, benchmarkFenceReceipt{}) { + t.Fatal("an identical name and spec cannot establish ownership") + } + bindingMetadata["uid"] = "replacement-binding-uid" + if matchesOwnedBenchmarkFence(policy, binding, desired, receipts) { + t.Fatal("the replacement binding must not qualify") + } + bindingMetadata["uid"] = "binding-uid" + policyMetadata := value(policy, "metadata").(object) + policyMetadata["uid"] = "replacement-policy-uid" + if matchesOwnedBenchmarkFence(policy, binding, desired, receipts) { + t.Fatal("the replacement policy must not qualify") + } +} + +func TestBenchmarkFenceJournalRejectsAmbiguousOwnership(t *testing.T) { + record := retirementFenceFixture(t) + record.Source = "1234567890abcdef1234567890abcdef12345678" + names := benchmarkFenceNames(record.Phase, record.Run) + record.Fences = map[string]benchmarkFenceReceipt{ + names[0]: {PolicyUID: "pv-policy", BindingUID: "pv-binding"}, + } + record.FencePending = names[1] + "/policy" + if err := validateBenchmarkFenceReceipts(record); err != nil { + t.Fatalf("verified intent and existing receipt should coexist: %v", err) + } + for _, test := range []struct { + name string + mutate func(*benchmarkRecord) + }{ + {"unknown fence", func(r *benchmarkRecord) { r.Fences["foreign"] = benchmarkFenceReceipt{PolicyUID: "unknown"} }}, + {"duplicate UID", func(r *benchmarkRecord) { r.Fences[names[1]] = benchmarkFenceReceipt{PolicyUID: "pv-policy"} }}, + {"borrowed scratch UID", func(r *benchmarkRecord) { r.Fences[names[1]] = benchmarkFenceReceipt{PolicyUID: r.NamespaceUID} }}, + {"intent over existing policy", func(r *benchmarkRecord) { r.FencePending = names[0] + "/policy" }}, + {"binding intent without policy receipt", func(r *benchmarkRecord) { r.FencePending = names[1] + "/binding" }}, + {"cleaned despite intent", func(r *benchmarkRecord) { r.Cleaned = true }}, + } { + t.Run(test.name, func(t *testing.T) { + changed := record + changed.Fences = maps.Clone(record.Fences) + test.mutate(&changed) + if err := validateBenchmarkFenceReceipts(changed); err == nil { + t.Fatal("ambiguous benchmark fence ownership was accepted") + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_instance.go b/scripts/migrate-longhorn-v2/benchmark_admission_instance.go new file mode 100644 index 0000000000..cbfafa904e --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_instance.go @@ -0,0 +1,180 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "maps" + "net/http" + "net/url" + "reflect" + "slices" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +const benchmarkInstanceFenceMessage = "Benchmark instance creation and attachment are fenced for retirement" + +var benchmarkInstanceKinds = map[string]string{ + "engines": "Engine", "replicas": "Replica", "enginefrontends": "EngineFrontend", "volumeattachments": "VolumeAttachment", "snapshots": "Snapshot", +} + +func readBenchmarkInstanceGroups(ctx context.Context, c client) (map[string][]object, map[string]object, error) { + const version = "longhorn.io/v1beta2" + groups := map[string][]object{} + observations := map[string]object{} + for _, resource := range slices.Sorted(maps.Keys(benchmarkInstanceKinds)) { + path := "/apis/" + version + "/namespaces/longhorn-system/" + resource + raw, err := c.kubectl(ctx, nil, "get", "--raw="+path) + if err != nil { + return nil, nil, errors.New("native benchmark instance inventory is unavailable") + } + items, err := decodeBenchmarkAdmissionList(raw, version, benchmarkInstanceKinds[resource]+"List") + if err != nil { + return nil, nil, err + } + groups[resource] = items + for _, item := range items { + observations[text(item, "metadata", "uid")] = item + } + } + return groups, observations, nil +} + +func proveAndCheckpointBenchmarkInstances(ctx context.Context, c client, record *benchmarkRecord) error { + if !activeBenchmarkV1FenceOwner(c, *record) || record.FenceCheckpoint != "" { + return errors.New("benchmark instance checkpoint has no active workflow owner or is already recorded") + } + groups, observations, err := readBenchmarkInstanceGroups(ctx, c) + if err != nil { + return err + } + children, err := qualifyRetirementInstances(*record, groups) + if err != nil { + return err + } + counts := map[string]int{} + for _, child := range children { + counts[child.Resource]++ + } + if counts["engines"] != 1 || counts["replicas"] != 3 || counts["volumeattachments"] != 1 || counts["enginefrontends"] != 0 { + return errors.New("benchmark V1 instance roster differs from one engine, three replicas and one attachment") + } + var nonce [16]byte + if _, err := rand.Read(nonce[:]); err != nil { + return err + } + next := *record + next.FenceCheckpoint = record.Run + "-" + hex.EncodeToString(nonce[:]) + policy, err := benchmarkRetirementInstanceFence(next, groups, next.FenceCheckpoint) + if err != nil { + return err + } + if err := advanceBenchmarkFenceJournal(ctx, c, record, next); err != nil { + return err + } + if err := installBenchmarkFence(ctx, c, record, policy); err != nil { + return err + } + name := text(policy, "metadata", "name") + receipt := record.Fences[name] + if err := proveBenchmarkFenceOnServingAPIs(ctx, c, record.Source, policy, receipt, func(ctx context.Context, client *http.Client, endpoint *url.URL) error { + for _, child := range children { + if err := probeBenchmarkInstanceAdmission(ctx, client, endpoint, child, observations[child.Identity.UID], name, record.FenceCheckpoint); err != nil { + return err + } + } + return nil + }); err != nil { + return err + } + afterActivation, _, err := readBenchmarkInstanceGroups(ctx, c) + if err != nil { + return err + } + qualified, err := qualifyRetirementInstances(*record, afterActivation) + if err != nil || !reflect.DeepEqual(children, qualified) { + return errors.New("benchmark instances changed while their admission fence activated") + } + for _, child := range children { + original := observations[child.Identity.UID] + kind := child.Resource + ".longhorn.io" + if text(original, "metadata", "annotations", retirementCheckpointAnnotation) == record.FenceCheckpoint { + return errors.New("benchmark instance checkpoint is already on an unqualified original") + } + if err := sourceCurrent(ctx, c.run, record.Source); err != nil { + return err + } + if err := c.patch(ctx, kind, "longhorn-system", original, metadataChanges(original, "annotations", map[string]string{retirementCheckpointAnnotation: record.FenceCheckpoint})...); err != nil { + return err + } + current, err := c.get(ctx, kind, "longhorn-system", child.Identity.Name, false) + if err != nil || benchmarkIdentity(current) != child.Identity || + text(current, "metadata", "resourceVersion") == child.ResourceVersion || + text(current, "metadata", "annotations", retirementCheckpointAnnotation) != record.FenceCheckpoint || + !reflect.DeepEqual(value(current, "spec"), value(original, "spec")) { + return errors.New("benchmark instance checkpoint was not confirmed") + } + } + afterCheckpoints, current, err := readBenchmarkInstanceGroups(ctx, c) + if err != nil { + return err + } + qualified, err = qualifyRetirementInstances(*record, afterCheckpoints) + if err != nil || len(qualified) != len(children) { + return errors.New("the checkpointed benchmark instance roster changed") + } + for index, child := range qualified { + old := children[index] + if child.Identity != old.Identity || child.Resource != old.Resource || + text(current[child.Identity.UID], "metadata", "annotations", retirementCheckpointAnnotation) != record.FenceCheckpoint || + !reflect.DeepEqual(value(current[child.Identity.UID], "spec"), value(observations[old.Identity.UID], "spec")) { + return errors.New("a native benchmark checkpoint identity or spec changed") + } + } + _, err = observeBenchmarkReferenceFence(ctx, c, policy, receipt) + return err +} + +func probeBenchmarkInstanceAdmission(ctx context.Context, client *http.Client, endpoint *url.URL, child retirementChild, actual object, policy, checkpoint string) error { + unverified := errors.New("the original instance retirement fence is unverified on this API server") + const group = "longhorn.io" + const version = group + "/v1beta2" + kind, known := benchmarkInstanceKinds[child.Resource] + if !known { + return unverified + } + name := child.Identity.Name + if child.Identity.Namespace != "longhorn-system" || !kubeName.MatchString(name) || len(name) > 253 || + child.Identity.UID == "" || child.ResourceVersion == "" || !benchmarkCheckpointPattern.MatchString(checkpoint) || + benchmarkIdentity(actual) != child.Identity || text(actual, "metadata", "resourceVersion") != child.ResourceVersion || + text(actual, "kind") != kind || text(actual, "apiVersion") != version || + value(actual, "metadata", "deletionTimestamp") != nil { + return unverified + } + path := "/apis/" + version + "/namespaces/longhorn-system/" + child.Resource + "/" + name + tested := []object{ + {"op": "test", "path": "/metadata/uid", "value": child.Identity.UID}, + {"op": "test", "path": "/metadata/resourceVersion", "value": child.ResourceVersion}, + } + positive := append(tested, metadataChanges(actual, "annotations", map[string]string{retirementCheckpointAnnotation: checkpoint})...) + code, raw, err := benchmarkAdmissionDryRun(ctx, client, endpoint, path, positive) + var admitted object + if err != nil || code != http.StatusOK || strictjson.Validate(raw, nil) != nil || json.Unmarshal(raw, &admitted) != nil || + benchmarkIdentity(admitted) != child.Identity || text(admitted, "apiVersion") != version || text(admitted, "kind") != kind || + text(admitted, "metadata", "annotations", retirementCheckpointAnnotation) != checkpoint || + !reflect.DeepEqual(value(admitted, "spec"), value(actual, "spec")) || + !reflect.DeepEqual(value(admitted, "status"), value(actual, "status")) { + return unverified + } + // A different metadata checkpoint must be rejected, including after a + // legitimate earlier write already installed this checkpoint on the child. + negative := append(tested, metadataChanges(actual, "annotations", map[string]string{retirementCheckpointAnnotation: "unverified"})...) + code, raw, err = benchmarkAdmissionDryRun(ctx, client, endpoint, path, negative) + if err != nil || !benchmarkDenialMatches(code, raw, group, child.Resource, name, policy, policy, benchmarkInstanceFenceMessage) { + return unverified + } + return nil +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_instance_test.go b/scripts/migrate-longhorn-v2/benchmark_admission_instance_test.go new file mode 100644 index 0000000000..83e7d4871d --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_instance_test.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "strings" + "sync/atomic" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBenchmarkInstanceAdmissionRequiresExactNativeDenial(t *testing.T) { + for _, fault := range []string{"", "changed child", "stale checkpoint", "generic denial", "negative accepted"} { + t.Run(fault, func(t *testing.T) { + record, groups := retirementInstanceFixture(t) + children, err := qualifyRetirementInstances(record, groups) + if err != nil { + t.Fatal(err) + } + child := children[slices.IndexFunc(children, func(child retirementChild) bool { return child.Resource == "engines" })] + actual := groups["engines"][0] + actual["kind"], actual["apiVersion"] = "Engine", "longhorn.io/v1beta2" + checkpoint := record.Run + "-" + strings.Repeat("a", 32) + policy := "lh-benchmark-retire-instances-" + record.Phase + "-" + record.Run + path := "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/engines/" + child.Identity.Name + var calls atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodPatch || req.URL.Path != path || req.URL.RawQuery != "dryRun=All" { + t.Error("unexpected native admission request") + w.WriteHeader(http.StatusBadRequest) + return + } + var ops []object + if err := json.NewDecoder(req.Body).Decode(&ops); err != nil || len(ops) != 4 || + text(ops[0], "value") != child.Identity.UID || text(ops[1], "value") != child.ResourceVersion { + t.Error("admission request was not scoped to the child UID and revision") + w.WriteHeader(http.StatusBadRequest) + return + } + current := calls.Add(1) + if current == 1 || fault == "negative accepted" { + out := cloneBenchmarkProbeObject(t, actual) + annotations := object{retirementCheckpointAnnotation: text(ops[3], "value")} + valueObject(out, "metadata")["annotations"] = annotations + if fault == "stale checkpoint" { + annotations[retirementCheckpointAnnotation] = "old" + } + _ = json.NewEncoder(w).Encode(out) + return + } + cause := fmt.Sprintf("ValidatingAdmissionPolicy '%s' with binding '%s' denied request: %s", policy, policy, benchmarkInstanceFenceMessage) + if fault == "generic denial" { + cause = "denied by unrelated admission" + } + status := metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: "Failure", Reason: metav1.StatusReasonForbidden, Code: http.StatusForbidden, + Message: fmt.Sprintf("engines.longhorn.io %q is forbidden: %s", child.Identity.Name, cause), + Details: &metav1.StatusDetails{Name: child.Identity.Name, Group: "longhorn.io", Kind: "engines", Causes: []metav1.StatusCause{{Message: cause}}}, + } + w.WriteHeader(http.StatusForbidden) + _ = json.NewEncoder(w).Encode(status) + })) + defer server.Close() + if fault == "changed child" { + valueObject(actual, "metadata")["uid"] = "foreign" + } + endpoint, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + err = probeBenchmarkInstanceAdmission(context.Background(), server.Client(), endpoint, child, actual, policy, checkpoint) + if (err == nil) != (fault == "") { + t.Fatalf("native proof result was %v for fault %q", err, fault) + } + want := int32(2) + if fault == "changed child" { + want = 0 + } + if fault == "stale checkpoint" { + want = 1 + } + if calls.Load() != want { + t.Fatalf("got %d probes, want %d", calls.Load(), want) + } + }) + } +} + +func cloneBenchmarkProbeObject(t *testing.T, original object) object { + t.Helper() + body, err := json.Marshal(original) + if err != nil { + t.Error(err) + return nil + } + var result object + if err := json.Unmarshal(body, &result); err != nil { + t.Error(err) + } + return result +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_probe.go b/scripts/migrate-longhorn-v2/benchmark_admission_probe.go new file mode 100644 index 0000000000..6cb11600f1 --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_probe.go @@ -0,0 +1,209 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "os" + "reflect" + "slices" + + "github.com/devantler-tech/platform/internal/namespaceinventory" + "github.com/devantler-tech/platform/internal/strictjson" +) + +const benchmarkReferenceFenceMessage = "Benchmark backend references are fenced for retirement" + +type benchmarkAdmissionEpoch struct { + PolicyUID, PolicyVersion, BindingUID, BindingVersion string +} + +// Prove the fence for the supplied original reference. Instance and namespace +// fences and the journal still require separate evidence before any delete. +func proveBenchmarkReferenceFenceOnServingAPIs(ctx context.Context, c client, record benchmarkRecord, observed object) error { + if err := authorizeBenchmark(os.Getenv, "v1"); err != nil { + return err + } + if record.Source != os.Getenv("GITHUB_SHA") || record.Run != os.Getenv("GITHUB_RUN_ID") { + return errors.New("the active V1 benchmark does not own this proof") + } + policies, err := benchmarkRetirementFences(record) + if err != nil || record.Phase != "v1" { + return errors.New("original V1 benchmark reference fence is unverified") + } + var policy object + var probe func(context.Context, *http.Client, *url.URL, benchmarkRecord, object) error + switch text(observed, "kind") { + case "PersistentVolume": + policy, probe = policies[0], probeBenchmarkRetainedPVAdmission + case "Volume": + policy, probe = policies[1], probeBenchmarkDetachedBackendAdmission + default: + return errors.New("benchmark reference fence received an unsupported resource") + } + receipt := record.Fences[text(policy, "metadata", "name")] + return proveBenchmarkFenceOnServingAPIs(ctx, c, record.Source, policy, receipt, func(ctx context.Context, direct *http.Client, address *url.URL) error { + return probe(ctx, direct, address, record, observed) + }) +} + +func proveBenchmarkFenceOnServingAPIs(ctx context.Context, c client, source string, policy object, receipt benchmarkFenceReceipt, probe func(context.Context, *http.Client, *url.URL) error) error { + if err := sourceCurrent(ctx, c.run, source); err != nil { + return err + } + before, err := observeBenchmarkReferenceFence(ctx, c, policy, receipt) + if err != nil { + return err + } + roster, err := observeBenchmarkAPIServers(ctx, c) + if err != nil { + return err + } + if err := verifyBenchmarkProviderEndpoints(ctx, os.Getenv("HCLOUD_TOKEN"), roster); err != nil { + return err + } + for _, endpoint := range roster { + direct, address, err := namespaceinventory.LoadBenchmarkAdmissionClient(os.Getenv, endpoint.ExternalIP) + if err != nil { + return err + } + proof := probe(ctx, direct, address) + direct.CloseIdleConnections() + if proof != nil { + return proof + } + } + after, err := observeBenchmarkAPIServers(ctx, c) + if err != nil || !slices.Equal(roster, after) { + return errors.New("serving API processes changed during benchmark admission proof") + } + bound, err := observeBenchmarkReferenceFence(ctx, c, policy, receipt) + if err != nil || before != bound { + return errors.New("the benchmark reference fence changed during native admission proof") + } + return sourceCurrent(ctx, c.run, source) +} + +func observeBenchmarkReferenceFence(ctx context.Context, c client, desired object, receipt benchmarkFenceReceipt) (benchmarkAdmissionEpoch, error) { + unverified := errors.New("the exact Deny policy and binding for the original reference are unverified") + name := text(desired, "metadata", "name") + policy, err := c.get(ctx, "validatingadmissionpolicies.admissionregistration.k8s.io", "", name, false) + if err != nil || !matchesOwnedBenchmarkPolicy(policy, desired, receipt.PolicyUID) { + return benchmarkAdmissionEpoch{}, unverified + } + binding, err := c.get(ctx, "validatingadmissionpolicybindings.admissionregistration.k8s.io", "", name, false) + if err != nil || !matchesOwnedBenchmarkFence(policy, binding, desired, receipt) { + return benchmarkAdmissionEpoch{}, unverified + } + return benchmarkAdmissionEpoch{ + PolicyUID: text(policy, "metadata", "uid"), PolicyVersion: text(policy, "metadata", "resourceVersion"), + BindingUID: text(binding, "metadata", "uid"), BindingVersion: text(binding, "metadata", "resourceVersion"), + }, nil +} + +func probeBenchmarkRetainedPVAdmission(ctx context.Context, client *http.Client, endpoint *url.URL, record benchmarkRecord, pv object) error { + unverified := errors.New("the original Retain PV fence is unverified on this API server") + identity := record.Resources["PersistentVolume"] + version := text(pv, "metadata", "resourceVersion") + if verifyRetirementReceipts(record) != nil || benchmarkIdentity(pv) != identity || version == "" || + text(pv, "apiVersion") != "v1" || text(pv, "kind") != "PersistentVolume" || + text(pv, "spec", "csi", "driver") != "driver.longhorn.io" || + text(pv, "spec", "csi", "volumeHandle") != record.Resources["LonghornVolume"].Name || + text(pv, "spec", "claimRef", "uid") != record.Resources["PersistentVolumeClaim"].UID || + text(pv, "spec", "claimRef", "name") != "fio-data" || + text(pv, "spec", "claimRef", "namespace") != record.Namespace || + text(pv, "spec", "storageClassName") != record.Resources["StorageClass"].Name || + text(pv, "spec", "persistentVolumeReclaimPolicy") != "Retain" { + return unverified + } + + path := "/api/v1/persistentvolumes/" + identity.Name + tested := []object{{"op": "test", "path": "/metadata/uid", "value": identity.UID}, + {"op": "test", "path": "/metadata/resourceVersion", "value": version}, + {"op": "replace", "path": "/spec/persistentVolumeReclaimPolicy", "value": "Retain"}} + code, raw, err := benchmarkAdmissionDryRun(ctx, client, endpoint, path, tested) + var admitted object + if err != nil || code != http.StatusOK || strictjson.Validate(raw, nil) != nil || json.Unmarshal(raw, &admitted) != nil || + benchmarkIdentity(admitted) != identity || text(admitted, "apiVersion") != "v1" || + text(admitted, "kind") != "PersistentVolume" || !reflect.DeepEqual(value(admitted, "spec"), value(pv, "spec")) { + return unverified + } + tested[2]["value"] = "Delete" + code, raw, err = benchmarkAdmissionDryRun(ctx, client, endpoint, path, tested) + name := "lh-benchmark-retire-pv-" + record.Phase + "-" + record.Run + if err != nil || !benchmarkDenialMatches(code, raw, "", "persistentvolumes", identity.Name, name, name, benchmarkReferenceFenceMessage) { + return unverified + } + return nil +} + +func probeBenchmarkDetachedBackendAdmission(ctx context.Context, client *http.Client, endpoint *url.URL, record benchmarkRecord, backend object) error { + unverified := errors.New("the detached backend fence is unverified on this API server") + identity := record.Resources["LonghornVolume"] + const group = "longhorn.io" + const version = group + "/v1beta2" + resourceVersion := text(backend, "metadata", "resourceVersion") + if verifyRetirementReceipts(record) != nil || benchmarkIdentity(backend) != identity || resourceVersion == "" || + text(backend, "apiVersion") != version || text(backend, "kind") != "Volume" || + text(backend, "spec", "dataEngine") != record.Phase || + verifyBenchmarkDetachedBackendState(backend) != nil || + verifyRecoveredBackendCascadeCleared(backend) != nil { + return unverified + } + path := "/apis/" + version + "/namespaces/longhorn-system/volumes/" + identity.Name + tested := []object{ + {"op": "test", "path": "/metadata/uid", "value": identity.UID}, + {"op": "test", "path": "/metadata/resourceVersion", "value": resourceVersion}, + {"op": "replace", "path": "/spec/dataEngine", "value": record.Phase}, + } + code, raw, err := benchmarkAdmissionDryRun(ctx, client, endpoint, path, tested) + var admitted object + if err != nil || code != http.StatusOK || strictjson.Validate(raw, nil) != nil || json.Unmarshal(raw, &admitted) != nil || + benchmarkIdentity(admitted) != identity || text(admitted, "apiVersion") != version || text(admitted, "kind") != "Volume" || + !reflect.DeepEqual(value(admitted, "spec"), value(backend, "spec")) || + verifyBenchmarkDetachedBackendState(admitted) != nil || verifyRecoveredBackendCascadeCleared(admitted) != nil { + return unverified + } + tested[2] = object{"op": "add", "path": "/spec/nodeID", "value": "prod-worker-3"} + code, raw, err = benchmarkAdmissionDryRun(ctx, client, endpoint, path, tested) + name := "lh-benchmark-retire-backend-" + record.Phase + "-" + record.Run + if err != nil || !benchmarkDenialMatches(code, raw, group, "volumes", identity.Name, name, name, benchmarkReferenceFenceMessage) { + return unverified + } + return nil +} + +// Keep the native response in bounded memory. The caller accepts only its +// fixed object or admission-policy status and never emits the response. +func benchmarkAdmissionDryRun(ctx context.Context, client *http.Client, endpoint *url.URL, path string, operations []object) (int, []byte, error) { + const limit = 2 << 20 + unverified := errors.New("native admission dry-run did not return a bounded response") + if client == nil || endpoint == nil || endpoint.Scheme != "https" || endpoint.Host == "" || endpoint.User != nil { + return 0, nil, unverified + } + destination := url.URL{Scheme: "https", Host: endpoint.Host, Path: path, RawQuery: "dryRun=All"} + payload, err := json.Marshal(operations) + if err != nil { + return 0, nil, unverified + } + req, err := http.NewRequestWithContext(ctx, http.MethodPatch, destination.String(), bytes.NewReader(payload)) + if err != nil { + return 0, nil, unverified + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json-patch+json") + res, err := client.Do(req) + if err != nil { + return 0, nil, unverified + } + raw, readErr := io.ReadAll(io.LimitReader(res.Body, limit+1)) + closeErr := res.Body.Close() + if readErr != nil || closeErr != nil || len(raw) > limit { + return 0, nil, unverified + } + return res.StatusCode, raw, nil +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_probe_test.go b/scripts/migrate-longhorn-v2/benchmark_admission_probe_test.go new file mode 100644 index 0000000000..5d0e0b2389 --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_probe_test.go @@ -0,0 +1,218 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "maps" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "slices" + "strconv" + "sync/atomic" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBenchmarkPVFenceRequiresExactPolicyAndDenyBinding(t *testing.T) { + for _, fault := range []string{"", "policy changed", "policy terminating", "binding narrowed", "binding warns", "different policy"} { + t.Run(fault, func(t *testing.T) { + policies, err := benchmarkRetirementFences(retirementFenceFixture(t)) + if err != nil { + t.Fatal(err) + } + desired := policies[0] + name := text(desired, "metadata", "name") + policy := object{"apiVersion": benchmarkAdmissionAPI, "kind": "ValidatingAdmissionPolicy", "metadata": object{"name": name, "uid": "original-policy", "resourceVersion": "100"}, "spec": maps.Clone(valueObject(desired, "spec"))} + binding := benchmarkDenyBinding(name, "original-policy") + valueObject(binding, "metadata")["uid"] = "original-binding" + valueObject(binding, "metadata")["resourceVersion"] = "101" + switch fault { + case "policy changed": + valueObject(policy, "spec")["failurePolicy"] = "Ignore" + case "policy terminating": + valueObject(policy, "metadata")["deletionTimestamp"] = "2026-01-01T00:00:00Z" + case "binding narrowed": + valueObject(binding, "spec")["matchResources"] = object{"namespaceSelector": object{"matchLabels": object{"missing": "true"}}} + case "binding warns": + valueObject(binding, "spec")["validationActions"] = []any{"Warn"} + case "different policy": + valueObject(binding, "spec")["policyName"] = "unrelated" + } + c := client{context: "admin@prod", run: func(_ context.Context, data []byte, program string, args ...string) ([]byte, error) { + if data != nil || program != "kubectl" || len(args) != 6 || + !slices.Equal(args[:3], []string{"--context=admin@prod", "--request-timeout=20s", "get"}) || + !slices.Equal(args[4:], []string{name, "--output=json"}) { + return nil, errors.New("unexpected benchmark fence read") + } + switch args[3] { + case "validatingadmissionpolicies.admissionregistration.k8s.io": + return json.Marshal(policy) + case "validatingadmissionpolicybindings.admissionregistration.k8s.io": + return json.Marshal(binding) + } + return nil, errors.New("unexpected benchmark fence kind") + }} + got, err := observeBenchmarkReferenceFence(context.Background(), c, desired, benchmarkFenceReceipt{PolicyUID: "original-policy", BindingUID: "original-binding"}) + if fault != "" && err == nil || fault == "" && (err != nil || got != (benchmarkAdmissionEpoch{"original-policy", "100", "original-binding", "101"})) { + t.Fatalf("policy and binding readback mismatch: %v", err) + } + }) + } +} + +func TestBenchmarkRetainedPVAdmissionRequiresBothDryRuns(t *testing.T) { + for _, test := range []struct { + name, fault string + proven bool + }{ + {name: "both exact dry-runs", proven: true}, + {name: "identity changes on allowed response", fault: "positive-uid"}, + {name: "generic rejection does not prove fence", fault: "generic-denial"}, + {name: "unfenced server accepts the negative", fault: "negative-allowed"}, + {name: "changed original fails before either request", fault: "original-uid"}, + } { + t.Run(test.name, func(t *testing.T) { + record := retirementFenceFixture(t) + pv := record.Resources["PersistentVolume"] + current := object{ + "apiVersion": "v1", "kind": "PersistentVolume", + "metadata": object{"name": pv.Name, "uid": pv.UID, "resourceVersion": "100"}, + "spec": object{ + "persistentVolumeReclaimPolicy": "Retain", "storageClassName": record.Resources["StorageClass"].Name, + "csi": object{"driver": "driver.longhorn.io", "volumeHandle": record.Resources["LonghornVolume"].Name}, + "claimRef": object{"uid": record.Resources["PersistentVolumeClaim"].UID, "name": "fio-data", "namespace": record.Namespace}, + }, + } + var calls atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempt := calls.Add(1) + if r.Method != http.MethodPatch || r.URL.Path != "/api/v1/persistentvolumes/"+pv.Name || r.URL.RawQuery != "dryRun=All" || + r.Header.Get("Content-Type") != "application/json-patch+json" { + t.Error("probe did not use the fixed server-side dry-run endpoint") + } + var changes []object + if err := json.NewDecoder(r.Body).Decode(&changes); err != nil || len(changes) != 3 || + !reflect.DeepEqual(changes[:2], []object{ + {"op": "test", "path": "/metadata/uid", "value": pv.UID}, + {"op": "test", "path": "/metadata/resourceVersion", "value": "100"}, + }) || text(changes[2], "op") != "replace" || text(changes[2], "path") != "/spec/persistentVolumeReclaimPolicy" { + t.Error("probe lacks the original UID and resource-version tests") + w.WriteHeader(http.StatusBadRequest) + return + } + if attempt == 1 { + if text(changes[2], "value") != "Retain" { + t.Error("positive request changed the Retain policy") + } + reply := maps.Clone(current) + if test.fault == "positive-uid" { + metadata := maps.Clone(valueObject(current, "metadata")) + metadata["uid"] = "replacement" + reply["metadata"] = metadata + } + _ = json.NewEncoder(w).Encode(reply) + return + } + if text(changes[2], "value") != "Delete" { + t.Error("negative request does not challenge the fence") + } + if test.fault == "negative-allowed" { + _ = json.NewEncoder(w).Encode(current) + return + } + policy := "lh-benchmark-retire-pv-" + record.Phase + "-" + record.Run + cause := "ValidatingAdmissionPolicy '" + policy + "' with binding '" + policy + "' denied request: Benchmark backend references are fenced for retirement" + status := metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: metav1.StatusFailure, Reason: metav1.StatusReasonForbidden, Code: http.StatusForbidden, + Message: "persistentvolumes " + strconv.Quote(pv.Name) + " is forbidden: " + cause, + Details: &metav1.StatusDetails{Name: pv.Name, Kind: "persistentvolumes", Causes: []metav1.StatusCause{{Message: cause}}}, + } + if test.fault == "generic-denial" { + status.Details.Causes = nil + } + w.WriteHeader(http.StatusForbidden) + _ = json.NewEncoder(w).Encode(status) + })) + t.Cleanup(server.Close) + endpoint, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + if test.fault == "original-uid" { + valueObject(current, "metadata")["uid"] = "replacement" + } + err = probeBenchmarkRetainedPVAdmission(context.Background(), server.Client(), endpoint, record, current) + if (err == nil) != test.proven || test.proven && calls.Load() != 2 || test.fault == "original-uid" && calls.Load() != 0 { + t.Fatalf("proven = %t, calls = %d, error = %v", test.proven, calls.Load(), err) + } + }) + } +} + +func TestBenchmarkBackendAdmissionProbesOnlyNaturallyClearedBackend(t *testing.T) { + for _, fault := range []string{"", "still references PV", "deny from other policy"} { + t.Run(fault, func(t *testing.T) { + record := retirementFenceFixture(t) + identity := record.Resources["LonghornVolume"] + backend := object{ + "apiVersion": "longhorn.io/v1beta2", "kind": "Volume", + "metadata": object{"name": identity.Name, "namespace": identity.Namespace, "uid": identity.UID, "resourceVersion": "100"}, + "spec": object{"dataEngine": record.Phase}, + "status": object{"state": "detached", "kubernetesStatus": object{}}, + } + if fault == "still references PV" { + valueObject(backend, "status", "kubernetesStatus")["pvName"] = record.Resources["PersistentVolume"].Name + } + var calls atomic.Int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + attempt := calls.Add(1) + var changes []object + if req.Method != http.MethodPatch || req.URL.RawQuery != "dryRun=All" || + req.URL.Path != "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/"+identity.Name || + json.NewDecoder(req.Body).Decode(&changes) != nil || len(changes) != 3 || + text(changes[0], "value") != identity.UID || text(changes[1], "value") != "100" { + t.Error("unexpected backend probe") + w.WriteHeader(http.StatusBadRequest) + return + } + if attempt == 1 { + if text(changes[2], "path") != "/spec/dataEngine" || text(changes[2], "value") != record.Phase { + t.Error("allowed probe attempted a changed backend") + } + _ = json.NewEncoder(w).Encode(backend) + return + } + if text(changes[2], "path") != "/spec/nodeID" || text(changes[2], "value") != "prod-worker-3" { + t.Error("negative probe did not challenge the detached fence") + } + policy := "lh-benchmark-retire-backend-" + record.Phase + "-" + record.Run + if fault == "deny from other policy" { + policy = "different-policy" + } + cause := "ValidatingAdmissionPolicy '" + policy + "' with binding '" + policy + "' denied request: " + benchmarkReferenceFenceMessage + status := metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: metav1.StatusFailure, Reason: metav1.StatusReasonForbidden, Code: http.StatusForbidden, + Message: "volumes.longhorn.io " + strconv.Quote(identity.Name) + " is forbidden: " + cause, + Details: &metav1.StatusDetails{Group: "longhorn.io", Name: identity.Name, Kind: "volumes", Causes: []metav1.StatusCause{{Message: cause}}}, + } + w.WriteHeader(http.StatusForbidden) + _ = json.NewEncoder(w).Encode(status) + })) + t.Cleanup(server.Close) + address, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + err = probeBenchmarkDetachedBackendAdmission(context.Background(), server.Client(), address, record, backend) + if (fault == "") != (err == nil) || fault == "" && calls.Load() != 2 || fault == "still references PV" && calls.Load() != 0 { + t.Fatalf("unexpected backend fence verdict: requests=%d error=%v", calls.Load(), err) + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_provider.go b/scripts/migrate-longhorn-v2/benchmark_admission_provider.go new file mode 100644 index 0000000000..80cf721e7d --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_provider.go @@ -0,0 +1,88 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +const benchmarkProviderReplyLimit = 128 << 10 + +// Verify the independently owned Hetzner address before any production API +// credential can be used against an address advertised by the Kubernetes Node. +func verifyBenchmarkProviderEndpoints(ctx context.Context, token string, roster []benchmarkAPIServer) error { + unverified := errors.New("direct serving API endpoints do not match the provider") + if token == "" || len(roster) != benchmarkControlPlaneCount { + return unverified + } + transport := &http.Transport{DisableCompression: true} + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, Timeout: 15 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + for _, server := range roster { + if !benchmarkControlPlaneProvider.MatchString(server.ProviderID) { + return unverified + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.hetzner.cloud/v1/servers/"+strings.TrimPrefix(server.ProviderID, "hcloud://"), nil) + if err != nil { + return unverified + } + req.Header.Set("Authorization", "Bearer "+token) + res, err := client.Do(req) + if err != nil { + return unverified + } + raw, readErr := io.ReadAll(io.LimitReader(res.Body, benchmarkProviderReplyLimit+1)) + closeErr := res.Body.Close() + if res.StatusCode != http.StatusOK || readErr != nil || closeErr != nil || !benchmarkProviderServerMatches(raw, server) { + return unverified + } + } + return nil +} + +func benchmarkProviderServerMatches(raw []byte, endpoint benchmarkAPIServer) bool { + if len(raw) > benchmarkProviderReplyLimit || strictjson.Validate(raw, map[string][]string{ + "": {"server"}, + "server": {"id", "name", "status", "labels", "public_net", "private_net"}, + "server.public_net": {"ipv4"}, + "server.public_net.ipv4": {"ip", "blocked"}, + "server.private_net[]": {"ip"}, + }) != nil { + return false + } + var response struct { + Server struct { + ID int64 `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Labels map[string]string `json:"labels"` + Public struct { + IPv4 struct { + IP string `json:"ip"` + Blocked bool `json:"blocked"` + } `json:"ipv4"` + } `json:"public_net"` + Private []struct { + IP string `json:"ip"` + } `json:"private_net"` + } `json:"server"` + } + if json.Unmarshal(raw, &response) != nil || !endpoint.InternalIP.Is4() || !endpoint.ExternalIP.Is4() || + !strings.HasPrefix(endpoint.Node, "prod-control-plane-") || !benchmarkControlPlaneProvider.MatchString(endpoint.ProviderID) { + return false + } + s := response.Server + return "hcloud://"+strconv.FormatInt(s.ID, 10) == endpoint.ProviderID && s.Name == endpoint.Node && + s.Status == "running" && s.Labels["ksail.owned"] == "true" && s.Labels["ksail.cluster.name"] == "prod" && + s.Labels["ksail.node.type"] == "control-plane" && s.Labels["ksail.node.index"] == strings.TrimPrefix(endpoint.Node, "prod-control-plane-") && + s.Labels["hcloud/node-group"] == "" && !s.Public.IPv4.Blocked && s.Public.IPv4.IP == endpoint.ExternalIP.String() && + len(s.Private) == 1 && s.Private[0].IP == endpoint.InternalIP.String() +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_provider_test.go b/scripts/migrate-longhorn-v2/benchmark_admission_provider_test.go new file mode 100644 index 0000000000..512eecf94b --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_provider_test.go @@ -0,0 +1,51 @@ +package main + +import ( + "encoding/json" + "net/netip" + "testing" +) + +func TestBenchmarkDirectEndpointRequiresMatchingProviderServer(t *testing.T) { + endpoint := benchmarkAPIServer{ + Node: "prod-control-plane-1", ProviderID: "hcloud://1001", + InternalIP: netip.MustParseAddr("10.0.0.1"), ExternalIP: netip.MustParseAddr("203.0.113.1"), + } + for _, test := range []struct { + name string + edit func(object) + proven bool + }{ + {"exact running KSail server", func(object) {}, true}, + {"different provider ID", func(s object) { s["id"] = 1002 }, false}, + {"different server name", func(s object) { s["name"] = "other" }, false}, + {"not running", func(s object) { s["status"] = "off" }, false}, + {"unowned", func(s object) { delete(valueObject(s, "labels"), "ksail.owned") }, false}, + {"different cluster", func(s object) { valueObject(s, "labels")["ksail.cluster.name"] = "dev" }, false}, + {"different role", func(s object) { valueObject(s, "labels")["ksail.node.type"] = "worker" }, false}, + {"different ordinal", func(s object) { valueObject(s, "labels")["ksail.node.index"] = "2" }, false}, + {"blocked public address", func(s object) { valueObject(s, "public_net", "ipv4")["blocked"] = true }, false}, + {"different public IPv4", func(s object) { valueObject(s, "public_net", "ipv4")["ip"] = "203.0.113.2" }, false}, + {"different private IPv4", func(s object) { s["private_net"] = []object{{"ip": "10.0.0.2"}} }, false}, + {"multiple provider private networks", func(s object) { + s["private_net"] = []object{{"ip": "10.0.0.1"}, {"ip": "10.0.0.1"}} + }, false}, + } { + t.Run(test.name, func(t *testing.T) { + server := object{ + "id": 1001, "name": endpoint.Node, "status": "running", + "labels": object{"ksail.owned": "true", "ksail.cluster.name": "prod", "ksail.node.type": "control-plane", "ksail.node.index": "1"}, + "public_net": object{"ipv4": object{"ip": "203.0.113.1"}}, + "private_net": []object{{"ip": "10.0.0.1"}}, + } + test.edit(server) + body, err := json.Marshal(object{"server": server}) + if err != nil { + t.Fatal(err) + } + if got := benchmarkProviderServerMatches(body, endpoint); got != test.proven { + t.Fatalf("provider endpoint proven = %t, want %t", got, test.proven) + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_roster.go b/scripts/migrate-longhorn-v2/benchmark_admission_roster.go new file mode 100644 index 0000000000..9c9fd1c879 --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_roster.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "net/netip" + "net/url" + "regexp" + "sort" + "time" + + "github.com/devantler-tech/platform/internal/strictjson" +) + +const benchmarkControlPlaneCount = 3 + +var benchmarkControlPlaneProvider = regexp.MustCompile(`^hcloud://[1-9][0-9]{0,18}$`) + +type benchmarkAPIServer struct { + Node, NodeUID, ProviderID, BootID, PodUID, ContainerID string + InternalIP, ExternalIP netip.Addr + StartedAt time.Time +} + +func observeBenchmarkAPIServers(ctx context.Context, c client) ([]benchmarkAPIServer, error) { + if c.context != "admin@prod" { + return nil, errors.New("the protected production API roster is required") + } + lists := [3][]object{} + for index, query := range []struct { + path, selector, apiVersion, kind string + }{ + {"/api/v1/nodes", "node-role.kubernetes.io/control-plane", "v1", "NodeList"}, + {"/api/v1/namespaces/kube-system/pods", "component=kube-apiserver,tier=control-plane", "v1", "PodList"}, + {"/apis/discovery.k8s.io/v1/namespaces/default/endpointslices", "kubernetes.io/service-name=kubernetes", "discovery.k8s.io/v1", "EndpointSliceList"}, + } { + path := query.path + "?" + url.Values{"labelSelector": {query.selector}}.Encode() + raw, err := c.kubectl(ctx, nil, "get", "--raw="+path) + if err != nil { + return nil, errors.New("serving API server inventory is unavailable") + } + lists[index], err = decodeBenchmarkAdmissionList(raw, query.apiVersion, query.kind) + if err != nil { + return nil, errors.New("serving API server inventory is incomplete") + } + } + return qualifyBenchmarkAPIServers(lists[0], lists[1], lists[2]) +} + +func decodeBenchmarkAdmissionList(raw []byte, apiVersion, kind string) ([]object, error) { + var envelope struct { + APIVersion string `json:"apiVersion"` + Kind string `json:"kind"` + Metadata struct { + Version string `json:"resourceVersion"` + Remaining int64 `json:"remainingItemCount"` + } `json:"metadata"` + } + if strictjson.Validate(raw, map[string][]string{ + "": {"apiVersion", "kind", "metadata", "items"}, + "metadata": {"resourceVersion", "remainingItemCount", "continue"}, + }) != nil || json.Unmarshal(raw, &envelope) != nil || envelope.APIVersion != apiVersion || + envelope.Kind != kind || envelope.Metadata.Version == "" || envelope.Metadata.Remaining != 0 { + return nil, errors.New("native API server inventory did not identify a complete revision") + } + return decodeList(raw) +} + +// The service publishes internal addresses. Each address must match the mirror +// Pod on a Ready control-plane host, whose direct external route can be verified +// separately against the provider. A service or floating IP alone cannot prove +// that a policy is active on each API server. +func qualifyBenchmarkAPIServers(nodes, pods, slices []object) ([]benchmarkAPIServer, error) { + incomplete := errors.New("complete native serving API server roster is required") + if len(nodes) != benchmarkControlPlaneCount || len(pods) != len(nodes) || len(slices) != 1 { + return nil, incomplete + } + nodeByName, err := indexed(nodes) + if err != nil { + return nil, incomplete + } + podByName, err := indexed(pods) + if err != nil { + return nil, incomplete + } + served, err := benchmarkServiceAddresses(slices[0]) + if err != nil { + return nil, err + } + + roster := make([]benchmarkAPIServer, 0, len(nodes)) + direct := map[netip.Addr]bool{} + providers := map[string]bool{} + for name, node := range nodeByName { + pod := podByName["kube-apiserver-"+name] + endpoint, err := benchmarkControlPlaneEndpoint(node, pod) + if err != nil || !served[endpoint.InternalIP] || direct[endpoint.ExternalIP] || providers[endpoint.ProviderID] { + return nil, incomplete + } + delete(served, endpoint.InternalIP) + direct[endpoint.ExternalIP], providers[endpoint.ProviderID] = true, true + roster = append(roster, endpoint) + } + sort.Slice(roster, func(i, j int) bool { return roster[i].Node < roster[j].Node }) + return roster, nil +} + +func benchmarkServiceAddresses(slice object) (map[netip.Addr]bool, error) { + incomplete := errors.New("default Kubernetes API service endpoints are unverified") + ports, _ := value(slice, "ports").([]any) + endpoints, _ := value(slice, "endpoints").([]any) + if text(slice, "metadata", "namespace") != "default" || text(slice, "metadata", "uid") == "" || + text(slice, "metadata", "labels", "kubernetes.io/service-name") != "kubernetes" || + text(slice, "addressType") != "IPv4" || len(ports) != 1 || len(endpoints) != benchmarkControlPlaneCount { + return nil, incomplete + } + port, _ := ports[0].(object) + if text(port, "name") != "https" || text(port, "protocol") != "TCP" || value(port, "port") != float64(6443) { + return nil, incomplete + } + served := make(map[netip.Addr]bool, len(endpoints)) + for _, item := range endpoints { + endpoint, _ := item.(object) + addresses, _ := value(endpoint, "addresses").([]any) + if len(addresses) != 1 || value(endpoint, "conditions", "ready") != true || + value(endpoint, "conditions", "terminating") == true { + return nil, incomplete + } + raw, _ := addresses[0].(string) + ip, err := netip.ParseAddr(raw) + if err != nil || !ip.Is4() || !ip.IsGlobalUnicast() || served[ip] { + return nil, incomplete + } + served[ip] = true + } + return served, nil +} + +func benchmarkControlPlaneEndpoint(node, pod object) (benchmarkAPIServer, error) { + incomplete := errors.New("control-plane Pod is not bound to its observed host") + endpoint := benchmarkAPIServer{ + Node: text(node, "metadata", "name"), NodeUID: text(node, "metadata", "uid"), + ProviderID: text(node, "spec", "providerID"), BootID: text(node, "status", "nodeInfo", "bootID"), + PodUID: text(pod, "metadata", "uid"), + } + _, controlPlane := valueObject(node, "metadata", "labels")["node-role.kubernetes.io/control-plane"] + if !controlPlane || !kubeNamespace.MatchString(endpoint.Node) || + !benchmarkControlPlaneProvider.MatchString(endpoint.ProviderID) || endpoint.BootID == "" || + !ready(valueObject(node, "status")) || endpoint.PodUID == "" || + text(pod, "metadata", "namespace") != "kube-system" || text(pod, "spec", "nodeName") != endpoint.Node || + text(pod, "metadata", "annotations", "kubernetes.io/config.mirror") == "" || + value(pod, "spec", "hostNetwork") != true || text(pod, "status", "phase") != "Running" || + !ready(valueObject(pod, "status")) { + return benchmarkAPIServer{}, incomplete + } + owners, _ := value(pod, "metadata", "ownerReferences").([]any) + if len(owners) != 1 { + return benchmarkAPIServer{}, incomplete + } + owner, _ := owners[0].(object) + if text(owner, "kind") != "Node" || text(owner, "apiVersion") != "v1" || value(owner, "controller") != true || + text(owner, "name") != endpoint.Node || text(owner, "uid") != endpoint.NodeUID { + return benchmarkAPIServer{}, incomplete + } + var err error + endpoint.InternalIP, endpoint.ExternalIP, err = benchmarkControlPlaneAddresses(node) + if err != nil || text(pod, "status", "hostIP") != endpoint.InternalIP.String() || + text(pod, "status", "podIP") != endpoint.InternalIP.String() { + return benchmarkAPIServer{}, incomplete + } + containers, _ := value(pod, "status", "containerStatuses").([]any) + if len(containers) != 1 { + return benchmarkAPIServer{}, incomplete + } + running, _ := containers[0].(object) + endpoint.ContainerID = text(running, "containerID") + endpoint.StartedAt, err = time.Parse(time.RFC3339Nano, text(running, "state", "running", "startedAt")) + if text(running, "name") != "kube-apiserver" || value(running, "ready") != true || + value(running, "started") != true || !benchmarkContainerID.MatchString(endpoint.ContainerID) || + err != nil || endpoint.StartedAt.IsZero() { + return benchmarkAPIServer{}, incomplete + } + return endpoint, nil +} + +func benchmarkControlPlaneAddresses(node object) (internal, external netip.Addr, err error) { + incomplete := errors.New("control-plane address pairs are incomplete") + addresses, _ := value(node, "status", "addresses").([]any) + for _, item := range addresses { + address, _ := item.(object) + var target *netip.Addr + switch text(address, "type") { + case "InternalIP": + target = &internal + case "ExternalIP": + target = &external + default: + continue + } + ip, parseErr := netip.ParseAddr(text(address, "address")) + if parseErr != nil || !ip.Is4() || !ip.IsGlobalUnicast() || target.IsValid() { + return netip.Addr{}, netip.Addr{}, incomplete + } + *target = ip + } + if !internal.IsPrivate() || !external.IsValid() || external.IsPrivate() { + return netip.Addr{}, netip.Addr{}, incomplete + } + return internal, external, nil +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_roster_test.go b/scripts/migrate-longhorn-v2/benchmark_admission_roster_test.go new file mode 100644 index 0000000000..17fe71df1e --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_roster_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" +) + +func TestBenchmarkObservesNativeAdmissionListRevisions(t *testing.T) { + nodes, pods, endpoints := benchmarkAdmissionFixture() + for _, corrupt := range []string{"", "wrong kind", "missing revision", "continued"} { + t.Run(corrupt, func(t *testing.T) { + calls := 0 + paths := []string{ + "/api/v1/nodes?labelSelector=node-role.kubernetes.io%2Fcontrol-plane", + "/api/v1/namespaces/kube-system/pods?labelSelector=component%3Dkube-apiserver%2Ctier%3Dcontrol-plane", + "/apis/discovery.k8s.io/v1/namespaces/default/endpointslices?labelSelector=kubernetes.io%2Fservice-name%3Dkubernetes", + } + c := client{context: "admin@prod", run: func(_ context.Context, data []byte, program string, args ...string) ([]byte, error) { + if data != nil || program != "kubectl" || calls >= len(paths) || !slices.Equal(args, []string{ + "--context=admin@prod", "--request-timeout=20s", "get", "--raw=" + paths[calls], + }) { + t.Fatal("API server inventory issued an unexpected command") + } + kind := []string{"NodeList", "PodList", "EndpointSliceList"}[calls] + apiVersion := []string{"v1", "v1", "discovery.k8s.io/v1"}[calls] + items := [][]object{nodes, pods, endpoints}[calls] + metadata := object{"resourceVersion": "100"} + calls++ + if calls == 3 { + switch corrupt { + case "wrong kind": + kind = "List" + case "missing revision": + delete(metadata, "resourceVersion") + case "continued": + metadata["continue"] = "another-page" + } + } + return json.Marshal(object{"apiVersion": apiVersion, "kind": kind, "metadata": metadata, "items": items}) + }} + roster, err := observeBenchmarkAPIServers(context.Background(), c) + if calls != 3 || (corrupt == "") != (err == nil) || err == nil && len(roster) != 3 { + t.Fatalf("native inventory completion mismatch: calls=%d error=%v", calls, err) + } + }) + } +} + +func TestBenchmarkAdmissionRequiresEveryServingAPIServer(t *testing.T) { + for _, tc := range []struct { + name string + change func(nodes, pods, slices []object) + ok bool + }{ + {name: "all three kube-apiserver mirrors and endpoints", ok: true}, + {name: "stale service roster", change: func(_, _ []object, slices []object) { + slices[0]["endpoints"] = value(slices[0], "endpoints").([]any)[:2] + }}, + {name: "wrong Kubernetes node epoch", change: func(_, pods, _ []object) { + value(pods[0], "metadata", "ownerReferences").([]any)[0].(object)["uid"] = "replacement-node" + }}, + {name: "non-host API process", change: func(_, pods, _ []object) { + valueObject(pods[0], "spec")["hostNetwork"] = false + }}, + {name: "API process epoch unavailable", change: func(_, pods, _ []object) { + delete(valueObject(pods[0], "status"), "containerStatuses") + }}, + {name: "duplicate direct transport", change: func(nodes, _, _ []object) { + value(nodes[0], "status", "addresses").([]any)[1].(object)["address"] = "203.0.113.2" + }}, + {name: "unready endpoint", change: func(_, _ []object, slices []object) { + valueObject(value(slices[0], "endpoints").([]any)[0].(object), "conditions")["ready"] = false + }}, + } { + t.Run(tc.name, func(t *testing.T) { + nodes, pods, slices := benchmarkAdmissionFixture() + if tc.change != nil { + tc.change(nodes, pods, slices) + } + roster, err := qualifyBenchmarkAPIServers(nodes, pods, slices) + if tc.ok { + if err != nil || len(roster) != 3 || roster[0].Node != "prod-control-plane-1" || + roster[0].ExternalIP.String() != "203.0.113.1" { + t.Fatalf("unexpected serving API roster: %v", err) + } + } else if err == nil { + t.Fatal("admission proof accepted an incomplete or ambiguous serving API roster") + } + }) + } +} + +func benchmarkAdmissionFixture() (nodes, pods, slices []object) { + endpoints := make([]any, 0, 3) + for i := 1; i <= 3; i++ { + name, uid := fmt.Sprintf("prod-control-plane-%d", i), fmt.Sprintf("node-uid-%d", i) + internal, external := fmt.Sprintf("10.0.0.%d", i), fmt.Sprintf("203.0.113.%d", i) + nodes = append(nodes, object{ + "metadata": object{"name": name, "uid": uid, "labels": object{"node-role.kubernetes.io/control-plane": ""}}, + "spec": object{"providerID": fmt.Sprintf("hcloud://%d", 1000+i)}, + "status": object{ + "nodeInfo": object{"bootID": fmt.Sprintf("boot-%d", i)}, + "conditions": readyConditions(), + "addresses": []any{ + object{"type": "InternalIP", "address": internal}, + object{"type": "ExternalIP", "address": external}, + }, + }, + }) + pods = append(pods, object{ + "metadata": object{ + "name": "kube-apiserver-" + name, "namespace": "kube-system", "uid": "pod-" + uid, + "annotations": object{"kubernetes.io/config.mirror": "native"}, + "ownerReferences": []any{object{ + "apiVersion": "v1", "kind": "Node", "name": name, "uid": uid, "controller": true, + }}, + }, + "spec": object{"nodeName": name, "hostNetwork": true}, + "status": object{ + "phase": "Running", "conditions": readyConditions(), "hostIP": internal, "podIP": internal, + "containerStatuses": []any{object{ + "name": "kube-apiserver", "ready": true, "started": true, + "containerID": "containerd://" + strings.Repeat(fmt.Sprint(i), 64), + "state": object{"running": object{"startedAt": "2026-11-10T10:00:00Z"}}, + }}, + }, + }) + endpoints = append(endpoints, object{"addresses": []any{internal}, "conditions": object{"ready": true}}) + } + return nodes, pods, []object{{ + "metadata": object{"name": "kubernetes", "namespace": "default", "uid": "slice", "labels": object{"kubernetes.io/service-name": "kubernetes"}}, + "addressType": "IPv4", + "ports": []any{object{"name": "https", "protocol": "TCP", "port": float64(6443)}}, + "endpoints": endpoints, + }} +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_status.go b/scripts/migrate-longhorn-v2/benchmark_admission_status.go new file mode 100644 index 0000000000..20331204a7 --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_status.go @@ -0,0 +1,44 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/devantler-tech/platform/internal/strictjson" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +// A generic Forbidden is insufficient: the native admission dispatcher +// identifies both the policy and its binding in a structured Status cause. +func benchmarkDenialMatches(code int, raw []byte, group, resource, name, policy, binding, message string) bool { + if code != http.StatusForbidden || len(raw) > 64<<10 || strictjson.Validate(raw, map[string][]string{ + "": {"apiVersion", "kind", "status", "reason", "code", "message", "details"}, + "details": {"name", "group", "kind", "causes"}, + "details.causes[]": {"message", "reason", "field"}, + }) != nil { + return false + } + var status metav1.Status + if json.Unmarshal(raw, &status) != nil || status.APIVersion != "v1" || status.Kind != "Status" || + status.Status != metav1.StatusFailure || status.Reason != metav1.StatusReasonForbidden || + status.Code != http.StatusForbidden || status.Details == nil || + status.Details.Group != group || status.Details.Kind != resource || status.Details.Name != name || + len(status.Details.Causes) != 1 { + return false + } + cause := status.Details.Causes[0] + if cause.Type != "" || cause.Field != "" { + return false + } + attribution, hasPrefix := strings.CutPrefix(cause.Message, "ValidatingAdmissionPolicy '") + denyingPolicy, attribution, hasBinding := strings.Cut(attribution, "' with binding '") + denyingBinding, denial, hasDenial := strings.Cut(attribution, "' denied request: ") + if !hasPrefix || !hasBinding || !hasDenial || denyingPolicy != policy || denyingBinding != binding || denial != message { + return false + } + qualified := schema.GroupResource{Group: group, Resource: resource} + return status.Message == fmt.Sprintf("%s %q is forbidden: %s", qualified.String(), name, cause.Message) +} diff --git a/scripts/migrate-longhorn-v2/benchmark_admission_status_test.go b/scripts/migrate-longhorn-v2/benchmark_admission_status_test.go new file mode 100644 index 0000000000..344878be73 --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_admission_status_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "net/http" + "strconv" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestBenchmarkDenialMustComeFromExactNativePolicy(t *testing.T) { + const name = "pvc-11111111-1111-4111-8111-111111111111" + const policy = "lh-benchmark-retire-pv-v1-1234" + const binding = "lh-benchmark-retire-pv-v1-1234" + const explanation = "Benchmark backend references are fenced for retirement" + const cause = "ValidatingAdmissionPolicy '" + policy + "' with binding '" + binding + "' denied request: " + explanation + messagePrefix := "persistentvolumes " + strconv.Quote(name) + " is forbidden: " + for _, test := range []struct { + name string + code int + edit func(*metav1.Status) + proven bool + }{ + {"exact denial", 403, func(*metav1.Status) {}, true}, + {"successful HTTP response", 200, func(*metav1.Status) {}, false}, + {"unauthenticated", 401, func(s *metav1.Status) { s.Reason = metav1.StatusReasonUnauthorized }, false}, + {"RBAC denial", 403, func(s *metav1.Status) { s.Details.Causes = nil }, false}, + {"only top-level echo", 403, func(s *metav1.Status) { s.Details.Causes[0].Message = "unrelated" }, false}, + {"another binding", 403, func(s *metav1.Status) { + s.Details.Causes[0].Message = "ValidatingAdmissionPolicy '" + policy + "' with binding 'other' denied request: " + explanation + s.Message = messagePrefix + s.Details.Causes[0].Message + }, false}, + {"another policy", 403, func(s *metav1.Status) { + s.Details.Causes[0].Message = "ValidatingAdmissionPolicy 'other' with binding '" + binding + "' denied request: " + explanation + s.Message = messagePrefix + s.Details.Causes[0].Message + }, false}, + {"extra cause field", 403, func(s *metav1.Status) { s.Details.Causes[0].Field = "spec" }, false}, + {"evaluation error", 403, func(s *metav1.Status) { + s.Details.Causes[0].Message = "ValidatingAdmissionPolicy '" + policy + "' with binding '" + binding + "' denied request: evaluation error" + }, false}, + {"different resource", 403, func(s *metav1.Status) { s.Details.Kind = "pods" }, false}, + {"different name", 403, func(s *metav1.Status) { s.Details.Name = "other" }, false}, + {"wrong API group", 403, func(s *metav1.Status) { s.Details.Group = "other" }, false}, + {"incomplete status", 403, func(s *metav1.Status) { s.Kind = "" }, false}, + } { + t.Run(test.name, func(t *testing.T) { + s := metav1.Status{ + TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "Status"}, + Status: metav1.StatusFailure, Reason: metav1.StatusReasonForbidden, Code: http.StatusForbidden, + Message: messagePrefix + cause, + Details: &metav1.StatusDetails{Name: name, Kind: "persistentvolumes", Causes: []metav1.StatusCause{{Message: cause}}}, + } + test.edit(&s) + raw, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + if got := benchmarkDenialMatches(test.code, raw, "", "persistentvolumes", name, policy, binding, explanation); got != test.proven { + t.Fatalf("denial attributed = %t, want %t", got, test.proven) + } + }) + } +} diff --git a/scripts/migrate-longhorn-v2/benchmark_assets.go b/scripts/migrate-longhorn-v2/benchmark_assets.go index 03330899e9..9df74f0967 100644 --- a/scripts/migrate-longhorn-v2/benchmark_assets.go +++ b/scripts/migrate-longhorn-v2/benchmark_assets.go @@ -2,7 +2,6 @@ package main import ( "errors" - "regexp" "time" ) @@ -45,11 +44,29 @@ for profile in randread randwrite randrw; do done ` -func benchmarkResources(phase, runID, tag string) ([]object, error) { - if (phase != "v1" && phase != "v2") || !regexp.MustCompile(`^[1-9][0-9]*$`).MatchString(runID) || tag != "longhorn-benchmark-"+phase+"-"+runID { +const benchmarkNamespace = "longhorn-benchmark" + +func benchmarkTag(phase, runID string) string { + return benchmarkNamespace + "-" + phase + "-" + runID +} + +func benchmarkNamespaceMatches(record benchmarkRecord) bool { + switch record.Version { + case 2: + return record.Namespace == benchmarkTag(record.Phase, record.Run) + case 3: + return record.Namespace == benchmarkNamespace + default: + return false + } +} + +func benchmarkResources(phase, runID, namespace string) ([]object, error) { + tag := benchmarkTag(phase, runID) + if (phase != "v1" && phase != "v2") || !benchmarkRunPattern.MatchString(runID) || (namespace != tag && namespace != benchmarkNamespace) { return nil, errors.New("unreviewed benchmark identity") } - namespace, class := tag, "lh-benchmark-"+phase+"-"+runID + class := "lh-benchmark-" + phase + "-" + runID meta := func(name string, namespaced bool) object { m := object{"name": name, "labels": object{"storage.platform.devantler.tech/benchmark-run": runID, "velero.io/exclude-from-backup": "true"}} if namespaced { @@ -63,7 +80,7 @@ func benchmarkResources(phase, runID, tag string) ([]object, error) { nsMeta := meta(namespace, false) nsMeta["labels"].(object)["pod-security.kubernetes.io/enforce"] = "restricted" nsMeta["labels"].(object)["pod-security.kubernetes.io/enforce-version"] = "latest" - return []object{ + assets := []object{ {"apiVersion": "v1", "kind": "Namespace", "metadata": nsMeta}, {"apiVersion": "networking.k8s.io/v1", "kind": "NetworkPolicy", "metadata": meta("fio-default-deny", true), "spec": object{"podSelector": object{}, "policyTypes": []any{"Ingress", "Egress"}}}, @@ -86,5 +103,9 @@ func benchmarkResources(phase, runID, tag string) ([]object, error) { "volumeMounts": []any{object{"name": "data", "mountPath": "/data"}, object{"name": "scratch", "mountPath": "/tmp"}}}}, "volumes": []any{object{"name": "data", "persistentVolumeClaim": object{"claimName": "fio-data"}}, object{"name": "scratch", "emptyDir": object{"sizeLimit": "32Mi"}}}}}, - }, nil + } + if namespace == benchmarkNamespace { + return assets[1:], nil + } + return assets, nil } diff --git a/scripts/migrate-longhorn-v2/benchmark_cleanup.go b/scripts/migrate-longhorn-v2/benchmark_cleanup.go index 95a142302a..231acc274e 100644 --- a/scripts/migrate-longhorn-v2/benchmark_cleanup.go +++ b/scripts/migrate-longhorn-v2/benchmark_cleanup.go @@ -74,8 +74,34 @@ func verifyScratchBinding(pv, volume, claim, class object) error { return nil } -func cleanupBenchmark(ctx context.Context, c client, source, namespace string, created map[string]object, pv, volume object) error { - claim, class := created["PersistentVolumeClaim"], created["StorageClass"] +func observeBenchmarkWorkspace(ctx context.Context, c client, expectedUID string) (object, error) { + namespace, err := c.get(ctx, "namespace", "", benchmarkNamespace, false) + uid := text(namespace, "metadata", "uid") + if err != nil || text(namespace, "apiVersion") != "v1" || text(namespace, "kind") != "Namespace" || + text(namespace, "metadata", "name") != benchmarkNamespace || uid == "" || expectedUID != "" && uid != expectedUID || + text(namespace, "status", "phase") != "Active" || value(namespace, "metadata", "deletionTimestamp") != nil || + text(namespace, "metadata", "labels", "pod-security.kubernetes.io/enforce") != "restricted" || + text(namespace, "metadata", "labels", "pod-security.kubernetes.io/enforce-version") != "latest" || + text(namespace, "metadata", "labels", "velero.io/exclude-from-backup") != "true" || + text(namespace, "metadata", "annotations", "kustomize.toolkit.fluxcd.io/prune") != "disabled" { + return nil, errors.New("the reviewed GitOps benchmark namespace is missing, replaced or unverified") + } + return namespace, nil +} + +func cleanupBenchmark(ctx context.Context, c client, record *benchmarkRecord, created map[string]object, claim, pv, volume object) error { + source, namespace := record.Source, record.Namespace + if record.Version != 3 || namespace != benchmarkNamespace || record.NamespaceUID == "" { + return errors.New("scratch cleanup requires the dedicated benchmark namespace receipt") + } + if _, err := observeBenchmarkWorkspace(ctx, c, record.NamespaceUID); err != nil { + return err + } + class := created["StorageClass"] + if benchmarkIdentity(claim) != benchmarkIdentity(created["PersistentVolumeClaim"]) || + text(claim, "status", "phase") != "Bound" || text(claim, "spec", "volumeName") != text(pv, "metadata", "name") { + return errors.New("the bound benchmark claim differs from its original creation receipt") + } if err := verifyScratchBinding(pv, volume, claim, class); err != nil { return err } @@ -113,8 +139,8 @@ func cleanupBenchmark(ctx context.Context, c client, source, namespace string, c return err } for _, a := range attachments { - if text(a, "spec", "source", "persistentVolumeName") == text(pv, "metadata", "name") && value(a, "status", "attached") == true { - return errors.New("scratch PV still has an attached consumer") + if text(a, "spec", "source", "persistentVolumeName") == text(pv, "metadata", "name") { + return errors.New("scratch PV still has an attachment or pending consumer") } } engines, err := c.list(ctx, "engines.longhorn.io", "longhorn-system") @@ -130,11 +156,20 @@ func cleanupBenchmark(ctx context.Context, c client, source, namespace string, c }); err != nil { return err } - if err := removeScratch(ctx, c, source, "persistentvolume", "", "/api/v1/persistentvolumes/"+text(pv, "metadata", "name"), pv); err != nil { - return err - } - if err := removeScratch(ctx, c, source, "volumes.longhorn.io", "longhorn-system", "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/"+text(volume, "metadata", "name"), volume); err != nil { - return err + if record.Phase == "v1" { + if err := retireBenchmarkV1PV(ctx, c, record, volume, claim, class); err != nil { + return err + } + if err := retireBenchmarkV1Backend(ctx, c, record); err != nil { + return err + } + } else { + if err := removeScratch(ctx, c, source, "persistentvolume", "", "/api/v1/persistentvolumes/"+text(pv, "metadata", "name"), pv); err != nil { + return err + } + if err := removeScratch(ctx, c, source, "volumes.longhorn.io", "longhorn-system", "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/"+text(volume, "metadata", "name"), volume); err != nil { + return err + } } if err := waitReady(ctx, func(ctx context.Context) error { replicas, err := c.list(ctx, "replicas.longhorn.io", "longhorn-system") @@ -159,8 +194,69 @@ func cleanupBenchmark(ctx context.Context, c client, source, namespace string, c return errors.New("scratch class or namespace has another claim") } } + pvs, err := c.list(ctx, "persistentvolumes", "") + if err != nil { + return err + } + for _, remaining := range pvs { + if text(remaining, "spec", "storageClassName") == text(class, "metadata", "name") { + return errors.New("scratch class still has a persistent volume") + } + } if err = removeScratch(ctx, c, source, "storageclass", "", "/apis/storage.k8s.io/v1/storageclasses/"+text(class, "metadata", "name"), class); err != nil { return err } - return removeScratch(ctx, c, source, "namespace", "", "/api/v1/namespaces/"+namespace, created["Namespace"]) + if err = removeScratch(ctx, c, source, "networkpolicy", namespace, "/apis/networking.k8s.io/v1/namespaces/"+namespace+"/networkpolicies/fio-default-deny", created["NetworkPolicy"]); err != nil { + return err + } + if record.Phase == "v1" { + if err := removeCompletedBenchmarkFences(ctx, c, *record); err != nil { + return err + } + } + _, err = observeBenchmarkWorkspace(ctx, c, record.NamespaceUID) + return err +} + +func retireBenchmarkV1PV(ctx context.Context, c client, record *benchmarkRecord, volume, claim, class object) error { + policies, err := benchmarkRetirementFences(*record) + if err != nil { + return err + } + identity := record.Resources["PersistentVolume"] + pv, err := c.get(ctx, "persistentvolume", "", identity.Name, false) + if err != nil || benchmarkIdentity(pv) != identity || text(pv, "status", "phase") != "Released" || + value(pv, "metadata", "deletionTimestamp") != nil || verifyRecoveredBenchmarkStorage(*record, claim, class, pv, volume) != nil { + return errors.New("original retained benchmark PV is not released and verified") + } + if err := installBenchmarkReferenceFence(ctx, c, record, policies[0]); err != nil { + return err + } + if err := proveBenchmarkReferenceFenceOnServingAPIs(ctx, c, *record, pv); err != nil { + return err + } + attachments, err := c.list(ctx, "volumeattachments.storage.k8s.io", "") + if err != nil { + return err + } + for _, attachment := range attachments { + if text(attachment, "spec", "source", "persistentVolumeName") == identity.Name { + return errors.New("benchmark PV still has a Kubernetes attachment request") + } + } + // Keep the resource version that passed both native dry-runs. A newer + // object must pass those same gates before it can be deleted. + if err := c.deleteObserved(ctx, record.Source, "/api/v1/persistentvolumes/"+identity.Name, pv); err != nil { + return err + } + return waitReady(ctx, func(ctx context.Context) error { + current, err := c.get(ctx, "persistentvolume", "", identity.Name, true) + if err != nil { + return err + } + if current != nil { + return errors.New("original retained benchmark PV is still present") + } + return nil + }) } diff --git a/scripts/migrate-longhorn-v2/benchmark_recovery_fence.go b/scripts/migrate-longhorn-v2/benchmark_recovery_fence.go index d8159e9b9b..86f7e7a48e 100644 --- a/scripts/migrate-longhorn-v2/benchmark_recovery_fence.go +++ b/scripts/migrate-longhorn-v2/benchmark_recovery_fence.go @@ -13,7 +13,7 @@ import ( // attachment/instance/dependency fences must be resolved before that is added. // The backend policy is for the stage after natural Kubernetes-status clearance. func verifyRetirementReceipts(record benchmarkRecord) error { - if (record.Phase != "v1" && record.Phase != "v2") || !regexp.MustCompile(`^[1-9][0-9]{0,18}$`).MatchString(record.Run) || record.Namespace != "longhorn-benchmark-"+record.Phase+"-"+record.Run || record.NamespaceUID == "" || record.PendingCreate != "" || record.Cleaned || len(record.Resources) != 7 || validateBenchmarkResources(record) != nil { + if (record.Phase != "v1" && record.Phase != "v2") || !benchmarkRunPattern.MatchString(record.Run) || !benchmarkNamespaceMatches(record) || record.NamespaceUID == "" || record.PendingCreate != "" || record.Cleaned || len(record.Resources) != 7 || validateBenchmarkResources(record) != nil { return errors.New("complete original benchmark receipts are required for retirement fences") } seen := map[string]bool{} @@ -85,7 +85,7 @@ func benchmarkRetirementFences(record benchmarkRecord) ([]object, error) { "resourceRules": []any{object{"apiGroups": []any{group}, "apiVersions": []any{version}, "resources": resources, "operations": []any{"CREATE", "UPDATE"}, "scope": scope}}, }, "matchConditions": []any{object{"name": "references-original-benchmark", "expression": match}}, - "validations": []any{object{"expression": invariant, "reason": "Forbidden", "message": "Benchmark backend references are fenced for retirement"}}, + "validations": []any{object{"expression": invariant, "reason": "Forbidden", "message": benchmarkReferenceFenceMessage}}, }, } } diff --git a/scripts/migrate-longhorn-v2/benchmark_recovery_fence_test.go b/scripts/migrate-longhorn-v2/benchmark_recovery_fence_test.go index 0556111488..8895268fdc 100644 --- a/scripts/migrate-longhorn-v2/benchmark_recovery_fence_test.go +++ b/scripts/migrate-longhorn-v2/benchmark_recovery_fence_test.go @@ -77,8 +77,11 @@ func TestNativeRetirementFenceDecisions(t *testing.T) { } _, groups := retirementInstanceFixture(t) for kind, items := range groups { - if kind != "volumeattachments" { - items[0]["spec"].(object)["dataEngine"] = phase + if kind == "volumeattachments" { + continue + } + for _, item := range items { + item["spec"].(object)["dataEngine"] = phase } } noStart, err := benchmarkRetirementInstanceFence(record, groups, "5678-"+strings.Repeat("a", 32)) diff --git a/scripts/migrate-longhorn-v2/benchmark_recovery_instances.go b/scripts/migrate-longhorn-v2/benchmark_recovery_instances.go index 2b744d83ee..4c34838048 100644 --- a/scripts/migrate-longhorn-v2/benchmark_recovery_instances.go +++ b/scripts/migrate-longhorn-v2/benchmark_recovery_instances.go @@ -29,7 +29,7 @@ func qualifyRetirementInstances(record benchmarkRecord, groups map[string][]obje if err := verifyRetirementReceipts(record); err != nil { return nil, err } - if len(groups) != 4 { + if len(groups) != len(benchmarkInstanceKinds) { return nil, errors.New("complete native child inventories are required") } backend := record.Resources["LonghornVolume"] @@ -40,7 +40,7 @@ func qualifyRetirementInstances(record benchmarkRecord, groups map[string][]obje names := map[string]bool{} children := []retirementChild{} engines := map[string]bool{} - for _, resource := range []string{"engines", "replicas", "enginefrontends", "volumeattachments"} { + for _, resource := range []string{"engines", "replicas", "enginefrontends", "volumeattachments", "snapshots"} { items, present := groups[resource] if !present || items == nil { return nil, errors.New("a native child inventory is unread") @@ -54,7 +54,7 @@ func qualifyRetirementInstances(record benchmarkRecord, groups map[string][]obje } seen[identity.UID], names[key] = true, true association := "volumeName" - if resource == "volumeattachments" { + if resource == "volumeattachments" || resource == "snapshots" { association = "volume" } volume, valid := value(item, "spec", association).(string) @@ -73,10 +73,16 @@ func qualifyRetirementInstances(record benchmarkRecord, groups map[string][]obje } ownerMatches = ownerMatches || text(ref, "uid") == backend.UID } - if volume != backend.Name && !ownerMatches { + volumeLabel := text(item, "metadata", "labels", "longhornvolume") + reservedAttachment := resource == "volumeattachments" && identity.Name == backend.Name + if volume != backend.Name && !ownerMatches && volumeLabel != backend.Name && !reservedAttachment { continue } - if volume != backend.Name || len(refs) != 1 || !ownerMatches || value(item, "metadata", "deletionTimestamp") != nil { + if resource == "snapshots" { + return nil, errors.New("the original benchmark backend has a native snapshot dependency") + } + if volume != backend.Name || volumeLabel != backend.Name || resource == "volumeattachments" && !reservedAttachment || + len(refs) != 1 || !ownerMatches || value(item, "metadata", "deletionTimestamp") != nil { return nil, errors.New("a backend child has a foreign or changing owner") } owner := refs[0].(object) diff --git a/scripts/migrate-longhorn-v2/benchmark_recovery_instances_test.go b/scripts/migrate-longhorn-v2/benchmark_recovery_instances_test.go index 75637c7b98..6e0a8b7786 100644 --- a/scripts/migrate-longhorn-v2/benchmark_recovery_instances_test.go +++ b/scripts/migrate-longhorn-v2/benchmark_recovery_instances_test.go @@ -8,10 +8,11 @@ import ( func retirementInstanceFixture(t *testing.T) (benchmarkRecord, map[string][]object) { t.Helper() record := retirementFenceFixture(t) - groups := map[string][]object{} + groups := map[string][]object{"snapshots": {}} for _, kind := range []string{"engines", "replicas", "enginefrontends", "volumeattachments"} { child := resource("scratch-" + kind) child["metadata"].(object)["namespace"] = "longhorn-system" + child["metadata"].(object)["labels"] = object{"longhornvolume": "scratch-backend"} child["metadata"].(object)["ownerReferences"] = []any{object{"apiVersion": "longhorn.io/v1beta2", "kind": "Volume", "name": "scratch-backend", "uid": record.Resources["LonghornVolume"].UID}} child["spec"] = object{"volumeName": "scratch-backend", "dataEngine": "v1", "desireState": "stopped", "nodeID": "prod-worker-1"} child["status"] = object{"ownerID": "prod-worker-1", "instanceManagerName": "scratch-im", "currentState": "stopped", "starting": false, "started": false} @@ -19,6 +20,7 @@ func retirementInstanceFixture(t *testing.T) (benchmarkRecord, map[string][]obje child["spec"].(object)["engineName"] = "scratch-engines" } if kind == "volumeattachments" { + child["metadata"].(object)["name"] = "scratch-backend" child["spec"] = object{"volume": "scratch-backend", "attachmentTickets": object{}} child["status"] = object{"attachmentTicketStatuses": object{}} } @@ -50,7 +52,7 @@ func TestRetirementInstancesRequireOriginalVolumeOwnerAndStoppedState(t *testing if err != nil || len(children) != 4 { t.Fatalf("native stopped owner chain refused: %v", err) } - for _, fault := range []string{"missing inventory", "missing owner", "foreign owner UID", "foreign owner kind", "foreign owner name", "controller-owner mismatch", "spec owner mismatch", "wrong engine", "unknown engine link", "starting", "started", "empty current state", "running desire", "unknown owner process", "linked clone", "pending attachment", "stale attachment status", "duplicate UID"} { + for _, fault := range []string{"missing inventory", "missing owner", "foreign owner UID", "foreign owner kind", "foreign owner name", "controller-owner mismatch", "spec owner mismatch", "selector label mismatch", "foreign reserved attachment", "native snapshot dependency", "wrong engine", "unknown engine link", "starting", "started", "empty current state", "running desire", "unknown owner process", "linked clone", "pending attachment", "stale attachment status", "duplicate UID"} { t.Run(fault, func(t *testing.T) { r, g := retirementInstanceFixture(t) engine := g["engines"][0] @@ -72,6 +74,18 @@ func TestRetirementInstancesRequireOriginalVolumeOwnerAndStoppedState(t *testing owner["controller"] = true case "spec owner mismatch": engine["spec"].(object)["volumeName"] = "foreign" + case "selector label mismatch": + engine["metadata"].(object)["labels"] = object{"longhornvolume": "foreign"} + case "foreign reserved attachment": + attachment["metadata"].(object)["labels"] = object{"longhornvolume": "foreign"} + delete(attachment["metadata"].(object), "ownerReferences") + attachment["spec"].(object)["volume"] = "foreign" + case "native snapshot dependency": + snapshot := resource("foreign-snapshot") + snapshot["metadata"].(object)["namespace"] = "longhorn-system" + snapshot["metadata"].(object)["labels"] = object{"longhornvolume": r.Resources["LonghornVolume"].Name} + snapshot["spec"] = object{"volume": "foreign"} + g["snapshots"] = []object{snapshot} case "wrong engine": replica["spec"].(object)["dataEngine"] = "v2" case "unknown engine link": diff --git a/scripts/migrate-longhorn-v2/benchmark_recovery_no_start.go b/scripts/migrate-longhorn-v2/benchmark_recovery_no_start.go index 85d8fce541..e73dfefb65 100644 --- a/scripts/migrate-longhorn-v2/benchmark_recovery_no_start.go +++ b/scripts/migrate-longhorn-v2/benchmark_recovery_no_start.go @@ -11,26 +11,30 @@ import ( const retirementCheckpointAnnotation = "storage.platform.devantler.tech/retirement-checkpoint" +var benchmarkCheckpointPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}-[a-f0-9]{32}$`) + // This builds an unbound policy, not a checkpoint or physical barrier. Activation // on all API servers, actual UID/RV metadata writes and direct physical-instance // observations under pinned process epochs are still required. func benchmarkRetirementInstanceFence(record benchmarkRecord, groups map[string][]object, checkpoint string) (object, error) { - if !regexp.MustCompile(`^[1-9][0-9]{0,18}-[a-f0-9]{32}$`).MatchString(checkpoint) { + if !benchmarkCheckpointPattern.MatchString(checkpoint) { return nil, errors.New("a fresh recovery checkpoint identity is required") } children, err := qualifyRetirementInstances(record, groups) if err != nil { return nil, err } - uids := map[string][]string{"engines": {}, "replicas": {}, "enginefrontends": {}, "volumeattachments": {}} + uids := map[string][]string{"engines": {}, "replicas": {}, "enginefrontends": {}, "volumeattachments": {}, "snapshots": {}} names, owners, managers := map[string]string{}, map[string]string{}, map[string]string{} references := map[string][]object{} - reserved := map[string][]string{"engines": {}, "replicas": {}, "enginefrontends": {}, "volumeattachments": {}} + reserved := map[string][]string{"engines": {}, "replicas": {}, "enginefrontends": {}, "volumeattachments": {record.Resources["LonghornVolume"].Name}, "snapshots": {}} for _, child := range children { uids[child.Resource] = append(uids[child.Resource], child.Identity.UID) names[child.Identity.UID], owners[child.Identity.UID], managers[child.Identity.UID] = child.Identity.Name, child.OwnerID, child.InstanceManagerName references[child.Identity.UID] = []object{child.OwnerReference} - reserved[child.Resource] = append(reserved[child.Resource], child.Identity.Name) + if child.Resource != "volumeattachments" { + reserved[child.Resource] = append(reserved[child.Resource], child.Identity.Name) + } } for _, values := range uids { sort.Strings(values) @@ -49,8 +53,11 @@ func benchmarkRetirementInstanceFence(record benchmarkRecord, groups map[string] } quote := strconv.Quote backend := record.Resources["LonghornVolume"] + volumeLabel := func(root string) string { + return "(has(" + root + ".metadata.labels) && 'longhornvolume' in " + root + ".metadata.labels && " + root + ".metadata.labels['longhornvolume'] == " + quote(backend.Name) + ")" + } refers := func(root string) string { - return "(" + root + ".metadata.name in " + literal(reserved) + "[request.resource.resource] || (has(" + root + ".spec.volumeName) && " + root + ".spec.volumeName == " + quote(backend.Name) + ") || (has(" + root + ".spec.volume) && " + root + ".spec.volume == " + quote(backend.Name) + ") || (has(" + root + ".metadata.ownerReferences) && " + root + ".metadata.ownerReferences.exists(r,r.uid == " + quote(backend.UID) + ")))" + return "(" + root + ".metadata.name in " + literal(reserved) + "[request.resource.resource] || " + volumeLabel(root) + " || (has(" + root + ".spec.volumeName) && " + root + ".spec.volumeName == " + quote(backend.Name) + ") || (has(" + root + ".spec.volume) && " + root + ".spec.volume == " + quote(backend.Name) + ") || (has(" + root + ".metadata.ownerReferences) && " + root + ".metadata.ownerReferences.exists(r,r.uid == " + quote(backend.UID) + ")))" } uid := "dyn(object.metadata).uid" falseFlag := func(path string) string { return "(!has(" + path + ") || " + path + " == false)" } @@ -67,15 +74,16 @@ func benchmarkRetirementInstanceFence(record benchmarkRecord, groups map[string] `request.operation == "UPDATE"`, uid + " in " + literal(uids) + "[request.resource.resource]", "dyn(oldObject.metadata).uid == " + uid, "object.metadata.name == " + literal(names) + "[" + uid + "]", `(!has(object.metadata.namespace) || object.metadata.namespace == "longhorn-system")`, + volumeLabel("object"), "has(dyn(object.metadata).ownerReferences) && dyn(object.metadata).ownerReferences == " + literal(references) + "[" + uid + "]", "object.spec == oldObject.spec", "has(object.metadata.annotations) && " + quote(retirementCheckpointAnnotation) + " in object.metadata.annotations && object.metadata.annotations[" + quote(retirementCheckpointAnnotation) + "] == " + quote(checkpoint), "(request.resource.resource == \"volumeattachments\" ? (" + attachment + ") : (" + instance + "))", }, " && ") policy := object{"apiVersion": "admissionregistration.k8s.io/v1", "kind": "ValidatingAdmissionPolicy", "metadata": object{"name": "lh-benchmark-retire-instances-" + record.Phase + "-" + record.Run}, "spec": object{ "failurePolicy": "Fail", - "matchConstraints": object{"matchPolicy": "Equivalent", "objectSelector": object{}, "namespaceSelector": object{}, "resourceRules": []any{object{"apiGroups": []any{"longhorn.io"}, "apiVersions": []any{"v1beta2"}, "resources": []any{"engines", "engines/status", "replicas", "replicas/status", "enginefrontends", "enginefrontends/status", "volumeattachments", "volumeattachments/status"}, "operations": []any{"CREATE", "UPDATE"}, "scope": "Namespaced"}}}, + "matchConstraints": object{"matchPolicy": "Equivalent", "objectSelector": object{}, "namespaceSelector": object{}, "resourceRules": []any{object{"apiGroups": []any{"longhorn.io"}, "apiVersions": []any{"v1beta2"}, "resources": []any{"engines", "engines/status", "replicas", "replicas/status", "enginefrontends", "enginefrontends/status", "volumeattachments", "volumeattachments/status", "snapshots", "snapshots/status"}, "operations": []any{"CREATE", "UPDATE"}, "scope": "Namespaced"}}}, "matchConditions": []any{object{"name": "references-original-benchmark", "expression": "request.namespace == \"longhorn-system\" && (" + refers("object") + " || (oldObject != null && " + refers("oldObject") + "))"}}, - "validations": []any{object{"expression": invariant, "reason": "Forbidden", "message": "Benchmark instance creation and attachment are fenced for retirement"}}, + "validations": []any{object{"expression": invariant, "reason": "Forbidden", "message": benchmarkInstanceFenceMessage}}, }} if literalErr != nil { return nil, literalErr diff --git a/scripts/migrate-longhorn-v2/benchmark_recovery_quiescence.go b/scripts/migrate-longhorn-v2/benchmark_recovery_quiescence.go index 65a03aaa1b..a2444efec7 100644 --- a/scripts/migrate-longhorn-v2/benchmark_recovery_quiescence.go +++ b/scripts/migrate-longhorn-v2/benchmark_recovery_quiescence.go @@ -20,18 +20,11 @@ func absentOrEmptyMap(v any) bool { // checks, mutation fences and recovery ownership at each destructive boundary. func verifyRecoveredBenchmarkDetached(volume object, attachments, engines, frontends []object) error { name := text(volume, "metadata", "name") - if name == "" || text(volume, "metadata", "uid") == "" || text(volume, "metadata", "resourceVersion") == "" || text(volume, "status", "state") != "detached" || attachments == nil || engines == nil || frontends == nil { + if attachments == nil || engines == nil || frontends == nil { return errors.New("complete detached backend observations are required") } - for _, key := range []string{"nodeID", "engineNodeID", "migrationNodeID"} { - if !absentOrEmptyString(value(volume, "spec", key)) { - return errors.New("backend still has a desired attachment or migration target") - } - } - for _, key := range []string{"currentNodeID", "currentEngineNodeID", "currentMigrationNodeID", "switchoverState"} { - if !absentOrEmptyString(value(volume, "status", key)) { - return errors.New("backend attachment or switchover remains observed") - } + if err := verifyBenchmarkDetachedBackendState(volume); err != nil { + return err } seen := map[string]bool{text(volume, "metadata", "uid"): true} for group, items := range [][]object{attachments, engines, frontends} { @@ -60,6 +53,24 @@ func verifyRecoveredBenchmarkDetached(volume object, attachments, engines, front return nil } +func verifyBenchmarkDetachedBackendState(volume object) error { + if text(volume, "metadata", "name") == "" || text(volume, "metadata", "uid") == "" || + text(volume, "metadata", "resourceVersion") == "" || text(volume, "status", "state") != "detached" { + return errors.New("complete detached backend observations are required") + } + for _, key := range []string{"nodeID", "engineNodeID", "migrationNodeID"} { + if !absentOrEmptyString(value(volume, "spec", key)) { + return errors.New("backend still has a desired attachment or migration target") + } + } + for _, key := range []string{"currentNodeID", "currentEngineNodeID", "currentMigrationNodeID", "switchoverState"} { + if !absentOrEmptyString(value(volume, "status", key)) { + return errors.New("backend attachment or switchover remains observed") + } + } + return nil +} + // Longhorn's native deletion controller follows these stored names without // original Kubernetes UID preconditions. Natural clearance is necessary, but // it is not a lasting fence: its PV controller can repopulate status after a diff --git a/scripts/migrate-longhorn-v2/benchmark_retirement_runtime.go b/scripts/migrate-longhorn-v2/benchmark_retirement_runtime.go new file mode 100644 index 0000000000..b9165a259d --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_retirement_runtime.go @@ -0,0 +1,388 @@ +package main + +import ( + "context" + "errors" + "maps" + "reflect" + "regexp" + "slices" + "sort" + "strings" + "time" +) + +var benchmarkRetirementImage = regexp.MustCompile(`^ghcr\.io/devantler-tech/platform-longhorn-instance-manager(?::v1\.13\.0-retirement\.1-[a-f0-9]{40}-1)?@(sha256:[a-f0-9]{64})$`) +var benchmarkContainerID = regexp.MustCompile(`^containerd://[a-f0-9]{64}$`) +var benchmarkDynamicVolume = regexp.MustCompile(`^pvc-[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$`) + +type benchmarkManagerEpoch struct { + Pod, PodUID, ManagerUID, Node, ContainerID string + StartedAt time.Time +} + +type benchmarkV1Runtime struct { + Image, SettingUID, SettingVersion string + Epochs []benchmarkManagerEpoch +} + +func validateBenchmarkRuntimeReceipt(record benchmarkRecord) error { + runtime := record.V1Runtime + if record.Version != 3 || record.Phase != "v1" { + if runtime != nil { + return errors.New("a V1 runtime receipt is outside this benchmark journal") + } + return nil + } + unproven := errors.New("the benchmark journal lacks its complete original V1 runtime") + if runtime == nil || !benchmarkRetirementImage.MatchString(runtime.Image) || runtime.SettingUID == "" || + runtime.SettingVersion == "" || len(runtime.Epochs) < len(record.Machines) { + return unproven + } + nodes, pods, uids, containers := map[string]bool{}, map[string]bool{}, map[string]bool{}, map[string]bool{} + for _, epoch := range runtime.Epochs { + if !strings.HasPrefix(epoch.Pod, "instance-manager-") || safeKubeRequest("pod", "longhorn-system", epoch.Pod, false) != nil || + !kubeNamespace.MatchString(epoch.Node) || epoch.PodUID == "" || epoch.ManagerUID == "" || epoch.PodUID == epoch.ManagerUID || + !benchmarkContainerID.MatchString(epoch.ContainerID) || epoch.StartedAt.IsZero() || + nodes[epoch.Node] || pods[epoch.Pod] || uids[epoch.PodUID] || uids[epoch.ManagerUID] || containers[epoch.ContainerID] { + return unproven + } + nodes[epoch.Node], pods[epoch.Pod] = true, true + uids[epoch.PodUID], uids[epoch.ManagerUID], containers[epoch.ContainerID] = true, true, true + } + for node := range record.Machines { + if !nodes[node] { + return unproven + } + } + return nil +} + +func captureBenchmarkV1Runtime(ctx context.Context, c client, requiredNodes []string) (benchmarkV1Runtime, error) { + setting, err := c.get(ctx, "settings.longhorn.io", "longhorn-system", "default-instance-manager-image", false) + if err != nil { + return benchmarkV1Runtime{}, errors.New("the approved V1 retirement runtime must be deployed before creating scratch") + } + return observeBenchmarkV1Runtime(ctx, c, text(setting, "value"), requiredNodes) +} + +func retireBenchmarkV1Backend(ctx context.Context, c client, record *benchmarkRecord) error { + if !activeBenchmarkV1FenceOwner(c, *record) || validateBenchmarkRuntimeReceipt(*record) != nil || record.V1Runtime == nil { + return errors.New("only the active benchmark with its original runtime may retire this V1 backend") + } + baseline := *record.V1Runtime + requiredNodes := slices.Sorted(maps.Keys(record.Machines)) + verifyRuntime := func() error { + current, err := observeBenchmarkV1Runtime(ctx, c, baseline.Image, requiredNodes) + if err != nil || !reflect.DeepEqual(baseline, current) { + return errors.New("the approved V1 runtime epochs changed since the benchmark started") + } + return nil + } + if err := verifyRuntime(); err != nil { + return err + } + policies, err := benchmarkRetirementFences(*record) + if err != nil { + return err + } + var backend object + identity := record.Resources["LonghornVolume"] + if err := waitReady(ctx, func(ctx context.Context) error { + current, err := c.get(ctx, "volumes.longhorn.io", "longhorn-system", identity.Name, false) + if err != nil || benchmarkIdentity(current) != identity || text(current, "spec", "dataEngine") != "v1" || + text(current, "spec", "accessMode") != "rwo" || value(current, "metadata", "deletionTimestamp") != nil { + return errors.New("original benchmark V1 backend identity changed") + } + if err := verifyBenchmarkDetachedBackendState(current); err != nil { + return err + } + if err := verifyRecoveredBackendCascadeCleared(current); err != nil { + return err + } + backend = current + return nil + }); err != nil { + return err + } + if err := installBenchmarkReferenceFence(ctx, c, record, policies[1]); err != nil { + return err + } + if err := proveBenchmarkReferenceFenceOnServingAPIs(ctx, c, *record, backend); err != nil { + return err + } + if err := proveAndCheckpointBenchmarkInstances(ctx, c, record); err != nil { + return err + } + if err := verifyRuntime(); err != nil { + return err + } + for _, epoch := range baseline.Epochs { + if err := sourceCurrent(ctx, c.run, record.Source); err != nil { + return err + } + if err := requestBenchmarkV1Seal(ctx, c, *record, epoch); err != nil { + return err + } + } + if err := verifyRuntime(); err != nil { + return err + } + // Rebind the original backend revision after the physical seals. The + // subsequent precondition refuses even a status update between proof and + // DELETE, so the Longhorn controller can only retire the object proven here. + current, err := c.get(ctx, "volumes.longhorn.io", "longhorn-system", identity.Name, false) + if err != nil || benchmarkIdentity(current) != identity || text(current, "spec", "accessMode") != "rwo" || + value(current, "metadata", "deletionTimestamp") != nil || verifyBenchmarkDetachedBackendState(current) != nil || + verifyRecoveredBackendCascadeCleared(current) != nil { + return errors.New("the original V1 backend changed while the physical instances were sealed") + } + groups, observations, err := readBenchmarkInstanceGroups(ctx, c) + if err != nil { + return err + } + children, err := qualifyRetirementInstances(*record, groups) + if err != nil || len(children) != 5 { + return errors.New("original checkpointed V1 children changed before retirement") + } + for _, child := range children { + if text(observations[child.Identity.UID], "metadata", "annotations", retirementCheckpointAnnotation) != record.FenceCheckpoint { + return errors.New("a V1 instance lost its native retirement checkpoint") + } + } + if err := proveBenchmarkReferenceFenceOnServingAPIs(ctx, c, *record, current); err != nil { + return err + } + policy, err := benchmarkRetirementInstanceFence(*record, groups, record.FenceCheckpoint) + if err != nil { + return err + } + if _, err := observeBenchmarkReferenceFence(ctx, c, policy, record.Fences[text(policy, "metadata", "name")]); err != nil { + return err + } + if _, err := observeBenchmarkReferenceFence(ctx, c, policies[0], record.Fences[text(policies[0], "metadata", "name")]); err != nil { + return err + } + if err := verifyRuntime(); err != nil { + return err + } + _, owned, err := readBenchmark(ctx, c, record.Phase) + if err != nil || !reflect.DeepEqual(owned, *record) { + return errors.New("the current benchmark no longer owns its V1 retirement journal") + } + if err := c.deleteObserved(ctx, record.Source, "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/"+identity.Name, current); err != nil { + return err + } + return waitReady(ctx, func(ctx context.Context) error { + current, err := c.get(ctx, "volumes.longhorn.io", "longhorn-system", identity.Name, true) + if err != nil { + return err + } + if current != nil { + return errors.New("original V1 benchmark backend has not retired") + } + return nil + }) +} + +func observeBenchmarkV1Runtime(ctx context.Context, c client, image string, requiredNodes []string) (benchmarkV1Runtime, error) { + if !benchmarkRetirementImage.MatchString(image) { + return benchmarkV1Runtime{}, errors.New("an immutable V1 retirement image is required") + } + beforeUID, beforeVersion, err := readBenchmarkDefaultManagerImage(ctx, c, image) + if err != nil { + return benchmarkV1Runtime{}, err + } + managers, err := c.list(ctx, "instancemanagers.longhorn.io", "longhorn-system") + if err != nil { + return benchmarkV1Runtime{}, err + } + pods, err := c.list(ctx, "pods", "longhorn-system") + if err != nil { + return benchmarkV1Runtime{}, err + } + epochs, err := qualifyV1RetirementManagerEpochs(image, requiredNodes, managers, pods) + if err != nil { + return benchmarkV1Runtime{}, err + } + afterUID, afterVersion, err := readBenchmarkDefaultManagerImage(ctx, c, image) + if err != nil || afterUID != beforeUID || afterVersion != beforeVersion { + return benchmarkV1Runtime{}, errors.New("the V1 default instance manager changed during observation") + } + return benchmarkV1Runtime{Image: image, SettingUID: beforeUID, SettingVersion: beforeVersion, Epochs: epochs}, nil +} + +func readBenchmarkDefaultManagerImage(ctx context.Context, c client, image string) (uid, version string, err error) { + const name = "default-instance-manager-image" + const namespace = "longhorn-system" + setting, err := c.get(ctx, "settings.longhorn.io", namespace, name, false) + if err != nil || text(setting, "metadata", "namespace") != namespace || + text(setting, "metadata", "name") != name || value(setting, "metadata", "deletionTimestamp") != nil || + text(setting, "value") != image { + return "", "", errors.New("the selected V1 default instance manager is unverified") + } + return text(setting, "metadata", "uid"), text(setting, "metadata", "resourceVersion"), nil +} + +// The caller must recheck current main, ownership, active admission and the +// observed epoch on both sides of this request. Its reply is runtime evidence +// for this Pod alone, and does not authorize deletion. +func requestBenchmarkV1Seal(ctx context.Context, c client, record benchmarkRecord, epoch benchmarkManagerEpoch) error { + pv := record.Resources["PersistentVolume"] + backend := record.Resources["LonghornVolume"] + claim := record.Resources["PersistentVolumeClaim"] + if c.context != "admin@prod" || record.Phase != "v1" || verifyRetirementReceipts(record) != nil || + !benchmarkDynamicVolume.MatchString(backend.Name) || pv.Name != backend.Name || + backend.Name != "pvc-"+claim.UID || !strings.HasPrefix(epoch.Pod, "instance-manager-") || + safeKubeRequest("pod", "longhorn-system", epoch.Pod, false) != nil { + return errors.New("the original dynamic V1 benchmark volume is unverified") + } + reply, err := c.kubectl(ctx, nil, "exec", "--namespace=longhorn-system", epoch.Pod, "--container=instance-manager", + "--", "/usr/local/bin/longhorn-instance-manager", "process", "seal-volume-for-retirement", "--volume", backend.Name) + if err != nil || string(reply) != "SEALED\n" { + return errors.New("V1 benchmark volume has no complete native retirement seal") + } + return nil +} + +// Use with the separately verified default V1 image and manager deployment. +// Longhorn v1.13.0 places new V1 instances only in its selected default image. +// Other image generations can continue to host previously attached volumes. +func qualifyV1RetirementManagerEpochs(image string, requiredNodes []string, managers, pods []object) ([]benchmarkManagerEpoch, error) { + if !benchmarkRetirementImage.MatchString(image) || len(requiredNodes) == 0 || managers == nil || pods == nil { + return nil, errors.New("complete V1 retirement runtime inputs are required") + } + managerByName, err := indexed(managers) + if err != nil { + return nil, err + } + podByName, err := indexed(pods) + if err != nil { + return nil, err + } + + epochs := make([]benchmarkManagerEpoch, 0, len(managers)) + selected := map[string]bool{} + for name, manager := range managerByName { + if text(manager, "spec", "image") != image || text(manager, "spec", "dataEngine") == "v2" { + continue + } + node := text(manager, "spec", "nodeID") + if !kubeNamespace.MatchString(node) || selected[node] || + safeKubeRequest("pod", "longhorn-system", name, false) != nil || + text(manager, "metadata", "namespace") != "longhorn-system" || + text(manager, "spec", "dataEngine") != "v1" || text(manager, "spec", "type") != "aio" || + text(manager, "status", "currentState") != "running" { + return nil, errors.New("default V1 instance manager selection is ambiguous") + } + epoch, err := verifyBenchmarkManagerPod(podByName[name], manager, image) + if err != nil { + return nil, err + } + selected[node] = true + delete(podByName, name) + epochs = append(epochs, epoch) + } + for _, pod := range podByName { + containers, _ := value(pod, "spec", "containers").([]any) + for _, entry := range containers { + container, _ := entry.(object) + if text(container, "image") == image && text(managerByName[text(pod, "metadata", "name")], "spec", "dataEngine") != "v2" { + return nil, errors.New("unpaired V1 retirement runtime Pod") + } + } + } + for _, node := range requiredNodes { + if !selected[node] { + return nil, errors.New("benchmark node lacks the approved V1 retirement runtime") + } + } + sort.Slice(epochs, func(i, j int) bool { return epochs[i].Pod < epochs[j].Pod }) + return epochs, nil +} + +func verifyBenchmarkManagerPod(pod, manager object, image string) (benchmarkManagerEpoch, error) { + name := text(manager, "metadata", "name") + epoch := benchmarkManagerEpoch{ + Pod: name, PodUID: text(pod, "metadata", "uid"), + ManagerUID: text(manager, "metadata", "uid"), Node: text(manager, "spec", "nodeID"), + } + refs, _ := value(pod, "metadata", "ownerReferences").([]any) + containers, _ := value(pod, "spec", "containers").([]any) + statuses, _ := value(pod, "status", "containerStatuses").([]any) + if text(pod, "metadata", "namespace") != "longhorn-system" || + text(pod, "spec", "nodeName") != epoch.Node || text(pod, "spec", "restartPolicy") != "Never" || + value(pod, "spec", "hostPID") == true || value(pod, "spec", "shareProcessNamespace") == true || + text(pod, "status", "phase") != "Running" || !ready(valueObject(pod, "status")) || + len(refs) != 1 || len(containers) != 1 || len(statuses) != 1 { + return benchmarkManagerEpoch{}, errors.New("retirement manager Pod identity or readiness is incomplete") + } + owner, _ := refs[0].(object) + container, _ := containers[0].(object) + running, _ := statuses[0].(object) + // Helm renders a release tag with its immutable digest. Containerd can + // report the same image by repository and digest without the tag. + declaredDigest := benchmarkRetirementDigest(image) + runningDigest := benchmarkRetirementDigest(strings.TrimPrefix(text(running, "imageID"), "docker-pullable://")) + if text(owner, "kind") != "InstanceManager" || text(owner, "apiVersion") != "longhorn.io/v1beta2" || + text(owner, "name") != name || text(owner, "uid") != epoch.ManagerUID || value(owner, "controller") != true || + text(container, "name") != "instance-manager" || text(container, "image") != image || + value(container, "securityContext", "privileged") != true || + value(container, "securityContext", "runAsUser") != float64(0) || + value(container, "securityContext", "runAsGroup") != float64(0) || + value(container, "securityContext", "runAsNonRoot") == true || + text(running, "name") != "instance-manager" || + declaredDigest == "" || runningDigest != declaredDigest || + value(running, "ready") != true || value(running, "started") != true || + value(running, "restartCount") != float64(0) || !benchmarkManagerHasDurableLog(pod, container) { + return benchmarkManagerEpoch{}, errors.New("retirement manager container, image or durable marker mount is unverified") + } + epoch.ContainerID = text(running, "containerID") + var err error + epoch.StartedAt, err = time.Parse(time.RFC3339Nano, text(running, "state", "running", "startedAt")) + if err != nil || epoch.StartedAt.IsZero() || !benchmarkContainerID.MatchString(epoch.ContainerID) { + return benchmarkManagerEpoch{}, errors.New("retirement manager container epoch is unverified") + } + return epoch, nil +} + +func benchmarkRetirementDigest(image string) string { + match := benchmarkRetirementImage.FindStringSubmatch(image) + if len(match) != 2 { + return "" + } + return match[1] +} + +func benchmarkManagerHasDurableLog(pod, container object) bool { + mounts, _ := value(container, "volumeMounts").([]any) + volumes, _ := value(pod, "spec", "volumes").([]any) + matches := 0 + for _, entry := range mounts { + mount, _ := entry.(object) + if text(mount, "mountPath") != "/log" { + continue + } + if text(mount, "name") != "log" || value(mount, "readOnly") == true || + !absentOrEmptyString(value(mount, "subPath")) || !absentOrEmptyString(value(mount, "subPathExpr")) { + return false + } + matches++ + } + if matches != 1 { + return false + } + matches = 0 + for _, entry := range volumes { + volume, _ := entry.(object) + if text(volume, "name") != "log" { + continue + } + if text(volume, "hostPath", "path") != "/var/lib/longhorn/logs" || + text(volume, "hostPath", "type") != "DirectoryOrCreate" { + return false + } + matches++ + } + return matches == 1 +} diff --git a/scripts/migrate-longhorn-v2/benchmark_retirement_runtime_test.go b/scripts/migrate-longhorn-v2/benchmark_retirement_runtime_test.go new file mode 100644 index 0000000000..35e8626036 --- /dev/null +++ b/scripts/migrate-longhorn-v2/benchmark_retirement_runtime_test.go @@ -0,0 +1,293 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "testing" + "time" +) + +var testRetirementImage = "ghcr.io/devantler-tech/platform-longhorn-instance-manager@sha256:" + strings.Repeat("a", 64) + +func TestBenchmarkV1RuntimeReceiptRequiresOriginalDistinctEpochs(t *testing.T) { + for _, fault := range []string{"", "missing", "wrong image", "reused container", "reused UID", "missing worker", "missing start", "unexpected V2"} { + t.Run(fault, func(t *testing.T) { + record := recoveryFixture(t) + if err := validateBenchmarkRuntimeReceipt(record); err != nil { + t.Fatalf("legacy journals must remain readable: %v", err) + } + record.Version = 3 + runtime := &benchmarkV1Runtime{Image: testRetirementImage, SettingUID: "setting-uid", SettingVersion: "17"} + for _, index := range []string{"1", "2", "3"} { + runtime.Epochs = append(runtime.Epochs, benchmarkManagerEpoch{ + Pod: "instance-manager-" + index, PodUID: "pod-" + index, ManagerUID: "manager-" + index, + Node: "prod-worker-" + index, ContainerID: "containerd://" + strings.Repeat(index, 64), + StartedAt: time.Date(2026, 11, 10, 1, 2, 3, 0, time.UTC), + }) + } + record.V1Runtime = runtime + switch fault { + case "missing": + record.V1Runtime = nil + case "wrong image": + runtime.Image = "longhornio/longhorn-instance-manager:v1.13.0" + case "reused container": + runtime.Epochs[1].ContainerID = runtime.Epochs[0].ContainerID + case "reused UID": + runtime.Epochs[1].ManagerUID = runtime.Epochs[0].PodUID + case "missing worker": + runtime.Epochs[2].Node = "different-worker" + case "missing start": + runtime.Epochs[2].StartedAt = time.Time{} + case "unexpected V2": + record.Phase = "v2" + } + if err := validateBenchmarkRuntimeReceipt(record); (err == nil) != (fault == "") { + t.Fatalf("runtime receipt validation did not match %q: %v", fault, err) + } + }) + } +} + +func TestObserveBenchmarkV1RuntimeRequiresStableDefaultImage(t *testing.T) { + for _, scenario := range []string{"stable", "stock default", "changed default epoch"} { + t.Run(scenario, func(t *testing.T) { + reads := 0 + c := client{context: "admin@prod", run: func(_ context.Context, _ []byte, program string, args ...string) ([]byte, error) { + if program != "kubectl" || args[2] != "get" { + t.Fatal("runtime observation attempted a mutation") + } + switch args[3] { + case "settings.longhorn.io": + reads++ + image, version := testRetirementImage, "1" + if scenario == "stock default" { + image = "longhornio/longhorn-instance-manager:v1.13.0" + } + if scenario == "changed default epoch" && reads == 2 { + version = "2" + } + return json.Marshal(object{ + "metadata": object{"name": "default-instance-manager-image", "namespace": "longhorn-system", + "uid": "setting-uid", "resourceVersion": version}, + "value": image, + }) + case "instancemanagers.longhorn.io": + return json.Marshal(object{"items": []object{retirementManager("instance-manager-a", "worker-a", testRetirementImage)}}) + case "pods": + return json.Marshal(object{"items": []object{retirementPod("instance-manager-a", "worker-a")}}) + default: + t.Fatal("unexpected runtime observation") + return nil, nil + } + }} + observed, err := observeBenchmarkV1Runtime(t.Context(), c, testRetirementImage, []string{"worker-a"}) + if scenario == "stable" { + if err != nil || reads != 2 || observed.SettingUID != "setting-uid" || len(observed.Epochs) != 1 { + t.Fatalf("native runtime was not verified: %v", err) + } + } else if err == nil { + t.Fatal("unverified default image qualified as the V1 runtime") + } + }) + } +} + +func TestV1RetirementSealRequiresExactVolumeAndNativeCompletion(t *testing.T) { + const claimUID = "a1b2c3d4-1234-5678-9abc-def012345678" + for _, tc := range []struct { + name, response string + change func(*benchmarkRecord) + execError error + wantCall, ok bool + }{ + {name: "sealed", response: "SEALED\n", wantCall: true, ok: true}, + {name: "pending with success exit", response: "PENDING\n", wantCall: true}, + {name: "ambiguous output", response: "PENDING\nSEALED\n", wantCall: true}, + {name: "failed transport", response: "SEALED\n", execError: errors.New("unknown response"), wantCall: true}, + {name: "different backend", change: func(r *benchmarkRecord) { + b := r.Resources["LonghornVolume"] + b.Name = "pvc-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + r.Resources["LonghornVolume"] = b + }}, + {name: "V2 volume", change: func(r *benchmarkRecord) { r.Phase = "v2" }}, + } { + t.Run(tc.name, func(t *testing.T) { + record := retirementFenceFixture(t) + pvc := record.Resources["PersistentVolumeClaim"] + pvc.UID = claimUID + record.Resources["PersistentVolumeClaim"] = pvc + for _, kind := range []string{"PersistentVolume", "LonghornVolume"} { + resource := record.Resources[kind] + resource.Name = "pvc-" + claimUID + record.Resources[kind] = resource + } + if tc.change != nil { + tc.change(&record) + } + called := false + c := client{context: "admin@prod", run: func(_ context.Context, _ []byte, program string, args ...string) ([]byte, error) { + called = true + want := []string{"--context=admin@prod", "--request-timeout=20s", "exec", "--namespace=longhorn-system", + "instance-manager-a", "--container=instance-manager", "--", "/usr/local/bin/longhorn-instance-manager", + "process", "seal-volume-for-retirement", "--volume", "pvc-" + claimUID} + if program != "kubectl" || !slices.Equal(args, want) { + t.Fatalf("seal invocation escaped the reviewed scope") + } + return []byte(tc.response), tc.execError + }} + err := requestBenchmarkV1Seal(t.Context(), c, record, benchmarkManagerEpoch{Pod: "instance-manager-a"}) + if (err == nil) != tc.ok || called != tc.wantCall { + t.Fatalf("seal disposition: called=%t, error=%v", called, err) + } + }) + } +} + +func TestQualifyV1RetirementManagerEpochs(t *testing.T) { + for _, tc := range []struct { + name string + change func(*[]object, *[]object) + ok bool + }{ + {name: "exact image and durable host mount", ok: true}, + { + name: "stock runtime cannot satisfy a required node", + change: func(managers, _ *[]object) { + valueObject((*managers)[0], "spec")["image"] = "longhornio/longhorn-instance-manager:v1.13.0" + }, + }, + { + name: "mismatched running image", + change: func(_, pods *[]object) { + value((*pods)[0], "status", "containerStatuses").([]any)[0].(object)["imageID"] = "sha256:" + strings.Repeat("b", 64) + }, + }, + { + name: "unpaired runtime", + change: func(_ *[]object, pods *[]object) { + *pods = append(*pods, retirementPod("instance-manager-unpaired", "worker-a")) + }, + }, + { + name: "transient marker path", + change: func(_, pods *[]object) { + value((*pods)[0], "spec", "volumes").([]any)[0].(object)["hostPath"] = nil + }, + }, + { + name: "storage runtime identity not explicitly selected", + change: func(_, pods *[]object) { + container := value((*pods)[0], "spec", "containers").([]any)[0].(object) + delete(valueObject(container, "securityContext"), "runAsUser") + }, + }, + { + name: "stock runtime group not explicitly selected", + change: func(_, pods *[]object) { + container := value((*pods)[0], "spec", "containers").([]any)[0].(object) + delete(valueObject(container, "securityContext"), "runAsGroup") + }, + }, + { + name: "different controller UID", + change: func(_, pods *[]object) { + value((*pods)[0], "metadata", "ownerReferences").([]any)[0].(object)["uid"] = "replacement" + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + managers := []object{ + retirementManager("instance-manager-a", "worker-a", testRetirementImage), + retirementManager("instance-manager-b", "worker-b", testRetirementImage), + retirementManager("instance-manager-old", "worker-a", "longhornio/longhorn-instance-manager:v1.13.0"), + } + pods := []object{ + retirementPod("instance-manager-a", "worker-a"), + retirementPod("instance-manager-b", "worker-b"), + } + if tc.change != nil { + tc.change(&managers, &pods) + } + + epochs, err := qualifyV1RetirementManagerEpochs(testRetirementImage, []string{"worker-a", "worker-b"}, managers, pods) + if tc.ok { + if err != nil || len(epochs) != 2 || epochs[0].Pod != "instance-manager-a" || epochs[0].PodUID != "pod-instance-manager-a" { + t.Fatalf("wrong active retirement epochs: %+v, %v", epochs, err) + } + } else if err == nil { + t.Fatal("an ambiguous native runtime was admitted") + } + }) + } +} + +func TestQualifyV1RetirementManagerTagAndDigest(t *testing.T) { + repository, digest, _ := strings.Cut(testRetirementImage, "@") + tag := "v1.13.0-retirement.1-" + strings.Repeat("b", 40) + "-1" + pinned := repository + ":" + tag + "@" + digest + for _, tc := range []struct { + name, declared, running string + ok bool + }{ + {"published tag with digest", pinned, "docker-pullable://" + testRetirementImage, true}, + {"tag without digest", repository + ":" + tag, testRetirementImage, false}, + {"unrecognized tag", repository + ":latest@" + digest, testRetirementImage, false}, + {"foreign image", pinned, repository + "-other@" + digest, false}, + {"different digest", pinned, repository + "@sha256:" + strings.Repeat("c", 64), false}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := retirementManager("instance-manager-a", "worker-a", tc.declared) + pod := retirementPod("instance-manager-a", "worker-a") + value(pod, "spec", "containers").([]any)[0].(object)["image"] = tc.declared + value(pod, "status", "containerStatuses").([]any)[0].(object)["imageID"] = tc.running + _, err := qualifyV1RetirementManagerEpochs(tc.declared, []string{"worker-a"}, []object{manager}, []object{pod}) + if (err == nil) != tc.ok { + t.Fatalf("runtime qualification disagrees with pinned image provenance: %v", err) + } + }) + } +} + +func retirementManager(name, node, image string) object { + return object{ + "metadata": object{"name": name, "namespace": "longhorn-system", "uid": "manager-" + name}, + "spec": object{"nodeID": node, "image": image, "dataEngine": "v1", "type": "aio"}, + "status": object{"currentState": "running"}, + } +} + +func retirementPod(name, node string) object { + return object{ + "metadata": object{ + "name": name, "namespace": "longhorn-system", "uid": "pod-" + name, + "ownerReferences": []any{object{ + "apiVersion": "longhorn.io/v1beta2", "kind": "InstanceManager", "name": name, + "uid": "manager-" + name, "controller": true, + }}, + }, + "spec": object{ + "nodeName": node, "restartPolicy": "Never", + "containers": []any{object{ + "name": "instance-manager", "image": testRetirementImage, + "volumeMounts": []any{object{"name": "log", "mountPath": "/log"}}, + "securityContext": object{"privileged": true, "runAsUser": float64(0), "runAsGroup": float64(0)}, + }}, + "volumes": []any{object{"name": "log", "hostPath": object{ + "path": "/var/lib/longhorn/logs", "type": "DirectoryOrCreate", + }}}, + }, + "status": object{ + "phase": "Running", "conditions": readyConditions(), + "containerStatuses": []any{object{ + "name": "instance-manager", "imageID": testRetirementImage, + "containerID": "containerd://" + strings.Repeat("b", 64), + "ready": true, "started": true, "restartCount": float64(0), + "state": object{"running": object{"startedAt": "2026-11-10T09:00:00Z"}}, + }}, + }, + } +} diff --git a/scripts/migrate-longhorn-v2/benchmark_test.go b/scripts/migrate-longhorn-v2/benchmark_test.go index 365fc9ce9d..5a28ce6073 100644 --- a/scripts/migrate-longhorn-v2/benchmark_test.go +++ b/scripts/migrate-longhorn-v2/benchmark_test.go @@ -103,3 +103,25 @@ func TestScratchResourcesStayOutOfBackupsAndGeneratedNames(t *testing.T) { } } } +func TestCurrentBenchmarkUsesReviewedWorkspaceWithoutNamespaceCreate(t *testing.T) { + const run = "1234" + for _, phase := range []string{"v1", "v2"} { + assets, err := benchmarkResources(phase, run, benchmarkNamespace) + if err != nil || len(assets) != 4 { + t.Fatalf("expected four scratch assets in the dedicated workspace: %v", err) + } + for _, asset := range assets { + switch text(asset, "kind") { + case "StorageClass": + if text(asset, "parameters", "nodeSelector") != benchmarkTag(phase, run) || + text(asset, "metadata", "namespace") != "" { + t.Fatal("the storage class must select only this run's temporary node tag") + } + default: + if text(asset, "kind") == "Namespace" || text(asset, "metadata", "namespace") != benchmarkNamespace { + t.Fatal("benchmark asset escaped the dedicated workspace") + } + } + } + } +} diff --git a/scripts/migrate-longhorn-v2/testdata/retirement-fence-proof/instances_test.go b/scripts/migrate-longhorn-v2/testdata/retirement-fence-proof/instances_test.go index 01f58c18de..d545b6efd7 100644 --- a/scripts/migrate-longhorn-v2/testdata/retirement-fence-proof/instances_test.go +++ b/scripts/migrate-longhorn-v2/testdata/retirement-fence-proof/instances_test.go @@ -14,10 +14,14 @@ func instanceCases(phase string) []decisionCase { for _, resource := range []string{"engines", "replicas", "enginefrontends", "volumeattachments"} { kind := map[string]string{"engines": "Engine", "replicas": "Replica", "enginefrontends": "EngineFrontend", "volumeattachments": "VolumeAttachment"}[resource] name := "scratch-" + resource + if resource == "volumeattachments" { + name = "scratch-backend" + } child := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "longhorn.io/v1beta2", "kind": kind, - "metadata": map[string]any{"name": name, "namespace": "longhorn-system", "uid": "uid-" + name, "resourceVersion": "1", "finalizers": []any{"longhorn.io"}, + "metadata": map[string]any{"name": name, "namespace": "longhorn-system", "uid": "uid-scratch-" + resource, "resourceVersion": "1", "finalizers": []any{"longhorn.io"}, "ownerReferences": []any{map[string]any{"apiVersion": "longhorn.io/v1beta2", "kind": "Volume", "name": "scratch-backend", "uid": "uid-scratch-backend"}}, + "labels": map[string]any{"longhornvolume": "scratch-backend"}, "annotations": map[string]any{"storage.platform.devantler.tech/retirement-checkpoint": "5678-" + strings.Repeat("a", 32)}}, "spec": map[string]any{"volumeName": "scratch-backend", "dataEngine": phase, "desireState": "stopped", "nodeID": "prod-worker-1"}, "status": map[string]any{"ownerID": "prod-worker-1", "instanceManagerName": "scratch-im", "currentState": "stopped", "starting": false, "started": false}, @@ -50,6 +54,7 @@ func instanceCases(phase string) []decisionCase { add("replacement UID", func(p *unstructured.Unstructured) { p.SetUID("replacement") }, admission.Update, true, false) add("foreign owner", func(p *unstructured.Unstructured) { p.SetOwnerReferences([]metav1.OwnerReference{{UID: "foreign"}}) }, admission.Update, true, false) add("missing owner", func(p *unstructured.Unstructured) { p.SetOwnerReferences(nil) }, admission.Update, true, false) + add("missing native selector label", func(p *unstructured.Unstructured) { p.SetLabels(nil) }, admission.Update, true, false) add("spec mutation", func(p *unstructured.Unstructured) { p.Object["spec"].(map[string]any)["nodeID"] = "foreign" }, admission.Update, true, false) add("metadata namespace escape", func(p *unstructured.Unstructured) { p.SetNamespace("foreign") }, admission.Update, true, false) add("original name recreation", func(p *unstructured.Unstructured) { p.SetUID("") }, admission.Create, true, false) @@ -75,12 +80,22 @@ func instanceCases(phase string) []decisionCase { add("unrelated create", func(p *unstructured.Unstructured) { p.SetName("foreign") p.SetOwnerReferences(nil) + p.SetLabels(nil) key := "volumeName" if resource == "volumeattachments" { key = "volume" } p.Object["spec"].(map[string]any)[key] = "foreign" }, admission.Create, false, false) + add("native selector label without owner or spec reference", func(p *unstructured.Unstructured) { + p.SetName("foreign") + p.SetOwnerReferences(nil) + key := "volumeName" + if resource == "volumeattachments" { + key = "volume" + } + p.Object["spec"].(map[string]any)[key] = "foreign" + }, admission.Create, true, false) for _, operation := range []admission.Operation{admission.Create, admission.Update} { foreign := child.DeepCopy() foreign.SetName("scratch-engines") @@ -89,6 +104,7 @@ func instanceCases(phase string) []decisionCase { } foreign.SetUID("foreign") foreign.SetOwnerReferences(nil) + foreign.SetLabels(nil) key := "volumeName" if resource == "volumeattachments" { key = "volume"