fix(storage): safely retire the Longhorn V1 benchmark storage

.github/workflows/prepare-longhorn-v2-node.yaml

@@ -199,9 +199,16 @@ jobs:
           HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
         run: ./scripts/use-prod-stable-api-endpoint.sh
       - name: Run the confirmed migration preparation
+        if: ${{ inputs.operation != 'benchmark-v1' }}
         env:
           GH_TOKEN: ${{ github.token }}
         run: go run ./scripts/migrate-longhorn-v2
+      - name: Run the confirmed V1 benchmark
+        if: ${{ inputs.operation == 'benchmark-v1' }}
+        env:
+          GH_TOKEN: ${{ github.token }}
+          HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
+        run: go run ./scripts/migrate-longhorn-v2
       - name: Remove runner credentials
         if: ${{ always() }}
         shell: bash

go.mod

@@ -46,6 +46,7 @@ require (
 	github.com/aws/aws-sdk-go-v2 v1.47.5
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
 	github.com/google/go-github/v88 v88.0.0
+	k8s.io/apimachinery v0.37.2
 	k8s.io/client-go v0.37.2
 )
 
@@ -99,7 +100,6 @@ require (
 	golang.org/x/term v0.47.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.3 // indirect
-	k8s.io/apimachinery v0.37.2 // indirect
 	k8s.io/klog/v2 v2.140.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect
 	k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect

internal/namespaceinventory/credentials.go

@@ -3,7 +3,9 @@ package namespaceinventory
 import (
 	"errors"
 	"io"
+	"net"
 	"net/http"
+	"net/netip"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -16,6 +18,7 @@ import (
 )
 
 var credentialPath = regexp.MustCompile(`^/dev/shm/longhorn-namespace-inventory\.[A-Za-z0-9]+/kubeconfig$`)
+var benchmarkCredentialPath = regexp.MustCompile(`^/dev/shm/longhorn-v2\.[A-Za-z0-9]+/kubeconfig$`)
 var sourcePattern = regexp.MustCompile(`^[0-9a-f]{40}$`)
 var runPattern = regexp.MustCompile(`^[1-9][0-9]{0,18}$`)
 
@@ -84,6 +87,38 @@ func LoadRetainedEmailClient(getenv func(string) string) (*http.Client, *url.URL
 	return newProductionClient(config)
 }
 
+// LoadBenchmarkAdmissionClient only admits the protected migration benchmark
+// operation. The caller must first join the selected address to the serving
+// Kubernetes process and independently verified Hetzner server.
+func LoadBenchmarkAdmissionClient(getenv func(string) string, directIP netip.Addr) (*http.Client, *url.URL, error) {
+	for key, want := range map[string]string{
+		"GITHUB_ACTIONS": "true", "GITHUB_EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main",
+		"GITHUB_REPOSITORY": "devantler-tech/platform", "GITHUB_RUN_ATTEMPT": "1", "RUNNER_ENVIRONMENT": "github-hosted",
+		"GITHUB_WORKFLOW_REF":   "devantler-tech/platform/.github/workflows/prepare-longhorn-v2-node.yaml@refs/heads/main",
+		"LONGHORN_V2_CONFIRM":   "benchmark-longhorn-storage",
+		"LONGHORN_V2_OPERATION": "benchmark-v1",
+	} {
+		if getenv(key) != want {
+			return nil, nil, errors.New("protected benchmark admission dispatch is unverified")
+		}
+	}
+	path := getenv("KUBECONFIG")
+	if !sourcePattern.MatchString(getenv("GITHUB_SHA")) || !runPattern.MatchString(getenv("GITHUB_RUN_ID")) ||
+		!benchmarkCredentialPath.MatchString(path) || filepath.Dir(path) != getenv("LONGHORN_V2_CREDENTIAL_DIR") ||
+		!directIP.Is4() || !directIP.IsGlobalUnicast() || directIP.IsPrivate() {
+		return nil, nil, errors.New("fixed benchmark admission credentials and direct IPv4 peer are required")
+	}
+	raw, err := readPrivateKubeconfig(path)
+	if err != nil {
+		return nil, nil, err
+	}
+	config, err := clientcmd.Load(raw)
+	if err != nil {
+		return nil, nil, errors.New("protected benchmark kubeconfig could not be decoded")
+	}
+	return newProductionClientAt(config, directIP)
+}
+
 // AuthorizeRetainedEmailDispatch refuses unless the environment is the first attempt of the
 // confirmed, maintainer-dispatched disposal workflow on main in this repository, with a
 // well-formed source commit and run id.
@@ -142,6 +177,10 @@ func readPrivateKubeconfig(path string) (raw []byte, err error) {
 }
 
 func newProductionClient(raw *clientcmdapi.Config) (*http.Client, *url.URL, error) {
+	return newProductionClientAt(raw, netip.Addr{})
+}
+
+func newProductionClientAt(raw *clientcmdapi.Config, directIP netip.Addr) (*http.Client, *url.URL, error) {
 	if raw == nil || raw.Contexts["admin@prod"] == nil {
 		return nil, nil, errors.New("fixed production context is missing")
 	}
@@ -173,10 +212,42 @@ func newProductionClient(raw *clientcmdapi.Config) (*http.Client, *url.URL, erro
 	config.Timeout = 20 * time.Second
 	config.DisableCompression = true
 	config.Proxy = func(*http.Request) (*url.URL, error) { return nil, nil }
+	if directIP.IsValid() {
+		if !directIP.Is4() || !directIP.IsGlobalUnicast() || directIP.IsPrivate() {
+			return nil, nil, errors.New("benchmark direct API address is unverified")
+		}
+		endpoint.Host = net.JoinHostPort(directIP.String(), "6443")
+		config.Host, config.ServerName = endpoint.String(), directIP.String()
+	}
 	client, err := rest.HTTPClientFor(config)
 	if err != nil {
 		return nil, nil, errors.New("verified production HTTP transport could not be constructed")
 	}
 	client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
+	if directIP.IsValid() {
+		client.Transport = directAPIPeer{host: endpoint.Host, transport: client.Transport}
+	}
 	return client, endpoint, nil
 }
+
+type directAPIPeer struct {
+	host      string
+	transport http.RoundTripper
+}
+
+func (peer directAPIPeer) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.URL.Scheme != "https" || req.URL.Host != peer.host || req.URL.User != nil || req.URL.Opaque != "" ||
+		(req.Host != "" && req.Host != peer.host) {
+		return nil, errors.New("direct API request addressed another peer")
+	}
+	if req.Method != http.MethodGet && (req.Method != http.MethodPatch || req.URL.RawQuery != "dryRun=All") {
+		return nil, errors.New("direct API client permits only observations and server dry-runs")
+	}
+	return peer.transport.RoundTrip(req)
+}
+
+func (peer directAPIPeer) CloseIdleConnections() {
+	if transport, ok := peer.transport.(interface{ CloseIdleConnections() }); ok {
+		transport.CloseIdleConnections()
+	}
+}

internal/namespaceinventory/credentials_test.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/netip"
 	"strings"
 	"testing"
 
@@ -123,3 +124,42 @@ func TestCredentialLoaderRefusesOutsideProtectedDispatch(t *testing.T) {
 		t.Fatal("credential loader admitted an unverified dispatch")
 	}
 }
+
+func TestBenchmarkDirectClientPinsAddressAndNeverFollowsRedirect(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+		t.Error("the old endpoint must not receive direct-client credentials")
+	}))
+	t.Cleanup(server.Close)
+	raw := fixtureConfig(server)
+	raw.Clusters["prod"].TLSServerName = "old-dns-name.invalid"
+	direct := netip.MustParseAddr("203.0.113.10")
+	client, endpoint, err := newProductionClientAt(raw, direct)
+	if err != nil || endpoint.String() != "https://203.0.113.10:6443" {
+		t.Fatalf("direct endpoint is not pinned: %v", err)
+	}
+	if _, err := client.Get(server.URL + "/version"); err == nil {
+		t.Fatal("direct client accepted a request for a different peer")
+	}
+	if client.CheckRedirect(nil, nil) != http.ErrUseLastResponse {
+		t.Fatal("direct client permits a redirect")
+	}
+	for _, test := range []struct{ method, query string }{
+		{http.MethodPatch, ""},
+		{http.MethodPatch, "?dryRun=All&dryRun="},
+		{http.MethodDelete, "?dryRun=All"},
+	} {
+		req, err := http.NewRequest(test.method, endpoint.String()+"/api/v1/persistentvolumes/scratch"+test.query, nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if _, err := client.Do(req); err == nil || !strings.Contains(err.Error(), "only observations and server dry-runs") {
+			t.Fatal("direct API client did not reject a write-capable request locally")
+		}
+	}
+}
+
+func TestBenchmarkCredentialLoaderRefusesOutsideProtectedDispatch(t *testing.T) {
+	if _, _, err := LoadBenchmarkAdmissionClient(func(string) string { return "" }, netip.MustParseAddr("203.0.113.10")); err == nil {
+		t.Fatal("direct credential loader admitted an unverified dispatch")
+	}
+}

k8s/providers/hetzner/infrastructure/controllers/longhorn/kustomization.yaml

@@ -3,6 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
+  - namespace-benchmark.yaml
   - helm-repository.yaml
   - helm-release.yaml
   # Default WaitForFirstConsumer StorageClass (see storage-class-longhorn-wffc.yaml);

k8s/providers/hetzner/infrastructure/controllers/longhorn/namespace-benchmark.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-benchmark
+  labels:
+    pod-security.kubernetes.io/enforce: restricted
+    pod-security.kubernetes.io/enforce-version: latest
+    velero.io/exclude-from-backup: "true"
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled

scripts/migrate-longhorn-v2/README.md

@@ -232,10 +232,11 @@ rebuilds, volume health and the recorded configuration are observed. Scratch obj
 from backups. Compare receipts knowing that, and avoid dispatching around the 02:17 UTC backup,
 whose load on the same disks is not measured. V2's necessary CPU reservation is reported alongside the V1 context;
 the comparison does not assume the engines have identical CPU costs.
-A completed receipt requires healthy storage and
-explicit cleanup of only the owned scratch Pod, claim, retained PV, backend,
-class and namespace. An incomplete run, including one that reaches the Pod's 25 minute
-deadline, holds with its journal and resources preserved for explicit recovery.
+A completed receipt requires healthy storage and explicit cleanup of the recorded
+scratch Pod, network policy, claim, retained PV, backend and class. Both phases use
+the GitOps-managed `longhorn-benchmark` namespace. The workflow records its UID and
+leaves the namespace in place. An incomplete run, including one that reaches the
+Pod's 25 minute deadline, holds with its journal and resources preserved for explicit recovery.
 
 Run `benchmark-v2` only after every volume uses V2 and every replica is back on
 the same three permanent workers. It requires the baseline's hardware, OS,