diff --git a/.github/workflows/publish-longhorn-v2-only.yaml b/.github/workflows/publish-longhorn-v2-only.yaml new file mode 100644 index 000000000..44ac2cd0e --- /dev/null +++ b/.github/workflows/publish-longhorn-v2-only.yaml @@ -0,0 +1,178 @@ +--- +name: Publish Longhorn V2-only Runtime + +"on": + pull_request: + branches: [main] + paths: + - .github/workflows/publish-longhorn-v2-only.yaml + - .github/scripts/setup-supply-chain-tools.sh + - scripts/build-longhorn-v2-only.sh + - k8s/providers/hetzner/infrastructure/controllers/longhorn/ManagerV2Only.Dockerfile + - k8s/providers/hetzner/infrastructure/controllers/longhorn/longhorn-*-v1.13.0-v2-only.patch + push: + branches: [main] + paths: + - .github/workflows/publish-longhorn-v2-only.yaml + - .github/scripts/setup-supply-chain-tools.sh + - scripts/build-longhorn-v2-only.sh + - k8s/providers/hetzner/infrastructure/controllers/longhorn/ManagerV2Only.Dockerfile + - k8s/providers/hetzner/infrastructure/controllers/longhorn/longhorn-*-v1.13.0-v2-only.patch + workflow_dispatch: {} + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +env: + IMAGE: ghcr.io/devantler-tech/platform-longhorn-manager + CHART_REGISTRY: ghcr.io/devantler-tech/platform-charts + HELM_VERSION: v4.3.1 + HELM_SHA256: 0403a88ae20913d11b10a2467b50087721fea7581736da6b9d3992f5357fe54b + +jobs: + verify: + name: Build and verify the pinned Longhorn V2-only artifacts + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + steps: + - name: Harden runner + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 + with: + egress-policy: audit + - name: Check out the reviewed compatibility patches + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Install the reviewed Go compiler + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + - name: Install verified Helm + run: | + set -euo pipefail + native_dir="$(mktemp -d)" + archive="helm-${HELM_VERSION}-linux-amd64.tar.gz" + curl --fail --location --retry 3 --output "$native_dir/$archive" "https://get.helm.sh/$archive" + printf '%s %s\n' "$HELM_SHA256" "$native_dir/$archive" | sha256sum --check - + tar -xzf "$native_dir/$archive" -C "$native_dir" linux-amd64/helm + printf '%s\n' "$native_dir/linux-amd64" >> "$GITHUB_PATH" + - name: Test both compatibility patches and build the artifacts + run: ./scripts/build-longhorn-v2-only.sh "${RUNNER_TEMP}/longhorn-v2-only" "${GITHUB_RUN_NUMBER}" + - name: Exercise the packaged manager with its default user + run: | + set -euo pipefail + validation_image="longhorn-v2-only:verify-${GITHUB_RUN_ID}" + docker build --platform linux/amd64 \ + --file k8s/providers/hetzner/infrastructure/controllers/longhorn/ManagerV2Only.Dockerfile \ + --tag "$validation_image" "${RUNNER_TEMP}/longhorn-v2-only/manager" + test "$(docker image inspect --format '{{.Config.User}}' "$validation_image")" = '65532:65532' + docker run --rm --network none --read-only --cap-drop ALL \ + --entrypoint /usr/local/sbin/longhorn-manager "$validation_image" daemon --help | + grep -F -- '--data-engine-mode' + - name: Keep this run's checked artifacts + if: ${{ github.ref == 'refs/heads/main' && github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: longhorn-v2-only-${{ github.run_id }} + path: ${{ runner.temp }}/longhorn-v2-only + if-no-files-found: error + retention-days: 1 + + publish: + name: Publish and verify the signed image and OCI chart + if: ${{ github.ref == 'refs/heads/main' && github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') }} + needs: verify + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: read # Check out and verify the current main revision. + packages: write # Publish the verified manager image and OCI chart to GHCR. + id-token: write # Obtain the identity for keyless Cosign signing and attestation. + steps: + - name: Harden runner + uses: step-security/harden-runner@46b06cea07f9e64621fdf066ba77ac6915163d43 # v2.22.3 + with: + egress-policy: audit + - name: Check out the reviewed source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Verify current main + env: + GH_TOKEN: ${{ github.token }} + run: ./scripts/verify-prod-recovery-source.sh "${GITHUB_SHA}" + - name: Download this run's verified artifacts + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: longhorn-v2-only-${{ github.run_id }} + path: ${{ runner.temp }}/longhorn-v2-only + - name: Install verified Helm + run: | + set -euo pipefail + native_dir="$(mktemp -d)" + archive="helm-${HELM_VERSION}-linux-amd64.tar.gz" + curl --fail --location --retry 3 --output "$native_dir/$archive" "https://get.helm.sh/$archive" + printf '%s %s\n' "$HELM_SHA256" "$native_dir/$archive" | sha256sum --check - + tar -xzf "$native_dir/$archive" -C "$native_dir" linux-amd64/helm + printf '%s\n' "$native_dir/linux-amd64" >> "$GITHUB_PATH" + - name: Verify inputs and package the manager + run: | + set -euo pipefail + cd "${RUNNER_TEMP}/longhorn-v2-only" + [[ "${GITHUB_RUN_NUMBER}" =~ ^[1-9][0-9]{0,11}$ ]] + chart="chart/longhorn-1.13.0-platform.${GITHUB_RUN_NUMBER}.tgz" + [[ "$(wc -l < SHA256SUMS)" -eq 2 && -f manager/longhorn-manager && -f "$chart" ]] + cmp SHA256SUMS <(sha256sum manager/longhorn-manager "$chart") + docker build --platform linux/amd64 \ + --file "${GITHUB_WORKSPACE}/k8s/providers/hetzner/infrastructure/controllers/longhorn/ManagerV2Only.Dockerfile" \ + --label "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --label "org.opencontainers.image.revision=${GITHUB_SHA}" \ + --tag "${IMAGE}:v1.13.0-platform.${GITHUB_RUN_NUMBER}" manager + docker run --rm --network none --read-only --cap-drop ALL \ + --entrypoint /usr/local/sbin/longhorn-manager "${IMAGE}:v1.13.0-platform.${GITHUB_RUN_NUMBER}" daemon --help | + grep -F -- '--data-engine-mode' + - name: Install verified signing and inventory tools + run: bash .github/scripts/setup-supply-chain-tools.sh + - name: Publish and verify image and chart signatures + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + ./scripts/verify-prod-recovery-source.sh "${GITHUB_SHA}" + output="${RUNNER_TEMP}/longhorn-v2-only" + release="1.13.0-platform.${GITHUB_RUN_NUMBER}" + chart="$output/chart/longhorn-$release.tgz" + image_tag="${IMAGE}:v$release" + printf '%s' "$GH_TOKEN" | docker login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + printf '%s' "$GH_TOKEN" | helm registry login ghcr.io --username "$GITHUB_ACTOR" --password-stdin + docker push "$image_tag" + image_ref="$(docker image inspect --format '{{json .RepoDigests}}' "$image_tag" | + jq -er --arg prefix "${IMAGE}@sha256:" 'map(select(startswith($prefix))) | if length == 1 then .[0] else error("ambiguous image digest") end')" + helm push "$chart" "oci://${CHART_REGISTRY}" 2>&1 | tee "$output/chart-push.log" + chart_digest="$(awk '$1 == "Digest:" && NF == 2 { print $2 }' "$output/chart-push.log")" + [[ "$chart_digest" =~ ^sha256:[a-f0-9]{64}$ && "${image_ref##*@}" =~ ^sha256:[a-f0-9]{64}$ ]] + chart_ref="${CHART_REGISTRY}/longhorn@$chart_digest" + identity="https://github.com/devantler-tech/platform/.github/workflows/publish-longhorn-v2-only.yaml@refs/heads/main" + issuer="https://token.actions.githubusercontent.com" + predicate="https://devantler.tech/attestations/longhorn-v2-only-build/v1" + binary_sha="$(sha256sum "$output/manager/longhorn-manager" | cut -d ' ' -f 1)" + chart_sha="$(sha256sum "$chart" | cut -d ' ' -f 1)" + jq -n --arg revision "$GITHUB_SHA" --arg run "$GITHUB_RUN_ID" --arg release "$release" \ + --arg binary "$binary_sha" --arg chart "$chart_sha" \ + '{sourceRevision:$revision,workflowRunId:$run,release:$release,binarySha256:$binary,chartSha256:$chart}' >"$output/build-receipt.json" + syft "docker:$image_tag" --output "cyclonedx-json=$output/sbom.json" + for artifact in "$image_ref" "$chart_ref"; do + cosign sign --yes "$artifact" + cosign attest --yes --predicate "$output/build-receipt.json" --type "$predicate" "$artifact" + cosign verify --certificate-identity "$identity" --certificate-oidc-issuer "$issuer" "$artifact" + cosign verify-attestation --type "$predicate" --certificate-identity "$identity" --certificate-oidc-issuer "$issuer" "$artifact" >/dev/null + done + cosign attest --yes --predicate "$output/sbom.json" --type cyclonedx "$image_ref" + cosign verify-attestation --type cyclonedx --certificate-identity "$identity" --certificate-oidc-issuer "$issuer" "$image_ref" >/dev/null + printf 'Published and verified manager %s and chart %s\n' "$image_ref" "$chart_ref" diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/ManagerV2Only.Dockerfile b/k8s/providers/hetzner/infrastructure/controllers/longhorn/ManagerV2Only.Dockerfile new file mode 100644 index 000000000..ac5c92c71 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/ManagerV2Only.Dockerfile @@ -0,0 +1,7 @@ +FROM docker.io/longhornio/longhorn-manager:v1.13.0@sha256:dfe39d0ce270c68b333d8b32bec41ee540b7a099ccc598ea312337a85c20fd89 +COPY --chmod=0755 longhorn-manager /usr/local/sbin/longhorn-manager + +USER 65532:65532 +# Check the manager admission listener. Kubernetes keeps the chart's HTTPS readiness probe. +HEALTHCHECK --interval=30s --timeout=3s --start-period=30s \ + CMD ["/bin/bash", "-ec", "exec 3<>/dev/tcp/127.0.0.1/9502"] diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/longhorn-chart-v1.13.0-v2-only.patch b/k8s/providers/hetzner/infrastructure/controllers/longhorn/longhorn-chart-v1.13.0-v2-only.patch new file mode 100644 index 000000000..91a6b30b3 --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/longhorn-chart-v1.13.0-v2-only.patch @@ -0,0 +1,376 @@ +diff --git a/chart/README.md b/chart/README.md +index 551ffaac28997988be1db16e35b95436e1be9428..9926f4578c79c68185c419f8cb051ad98a6bcd07 100644 +--- a/chart/README.md ++++ b/chart/README.md +@@ -22,7 +22,36 @@ Longhorn is 100% open source software. Project source code is spread across a nu + 1. A container runtime compatible with Kubernetes (Docker v1.13+, containerd v1.3.7+, etc.) + 2. Kubernetes >= v1.34 + 3. Make sure `bash`, `curl`, `findmnt`, `grep`, `awk` and `blkid` has been installed in all nodes of the Kubernetes cluster. +-4. Make sure `open-iscsi` has been installed, and the `iscsid` daemon is running on all nodes of the Kubernetes cluster. For GKE, recommended Ubuntu as guest OS image since it contains `open-iscsi` already. ++4. In the default `v1-capable` mode, make sure `open-iscsi` is installed and the `iscsid` daemon is running on all nodes. This includes mixed V1/V2 installations. For GKE, Ubuntu includes `open-iscsi` already. ++ ++## V2-only installation capability ++ ++`dataEngineMode: v2-only` selects an explicit installation capability supported by a compatible Longhorn Manager release. It is separate from enabling V2 alongside V1. The default `v1-capable` mode preserves existing manager commands and iSCSI prerequisites. ++ ++For a fresh V2-only installation with a chart-owned StorageClass, set these values and use the tag and digest from the matching signed manager release: ++ ++```yaml ++dataEngineMode: v2-only ++image: ++ longhorn: ++ manager: ++ registry: ghcr.io ++ repository: devantler-tech/platform-longhorn-manager ++ tag: @sha256: ++defaultSettings: ++ v1DataEngine: false ++ v2DataEngine: true ++persistence: ++ dataEngine: v2 ++``` ++ ++The chart rejects the stock manager repository and unresolved or conflicting engine settings, including when upgrade hooks are disabled. The Platform manager repository requires a published `v1.13.0-platform.` tag. Daemon, global manager and pre-upgrade commands receive the capability explicitly. The manager separately checks live storage and runtime before starting. V2 nodes still require their normal V2 disk, CPU, kernel, NVMe/TCP and memory/hugepage prerequisites. ++ ++For an existing installation, retain iSCSI throughout migration. Move or restore every V1 volume and retained snapshot that must be kept, prove V2 recovery, and retire V1 engines, replicas and instance-manager runtime before selecting V2-only mode. This value does not convert existing data, snapshots or immutable StorageClass parameters. Remove host iSCSI dependencies only after the V2-only rollout is verified on every node. ++ ++Apply and verify the explicit engine settings and StorageClass ownership while still in `v1-capable` mode first, then opt into `v2-only` in a separate upgrade. The pre-upgrade check receives the requested capability but validates the installed settings and runtime before the new ConfigMap is applied. ++ ++The chart's `longhorn-storageclass` ConfigMap causes the manager to delete and recreate the generated `longhorn` StorageClass when its embedded definition changes. Prepare and select an external V2 StorageClass and set `persistence.createStorageClass=false` during an existing installation's migration, or establish that the chart-owned class already has matching parameters. The chart cannot verify that live ownership or parameter match during rendering. + + ## Upgrading to Kubernetes v1.25+ + +@@ -342,6 +371,7 @@ For more details, see the [ocp-readme](https://github.com/longhorn/longhorn/blob + | Key | Default | Description | + |-----|---------|-------------| + | annotations | `{}` | Annotation for the Longhorn Manager DaemonSet pods. This setting is optional. | ++| dataEngineMode | `"v1-capable"` | Installation capability. v1-capable preserves mixed-engine behavior and iSCSI requirements. v2-only requires explicit V1=false/V2=true settings and a V2 StorageClass when chart-owned. Use a manager release supporting this option. | + | defaultBackupStore | `{"backupTarget":null,"backupTargetCredentialSecret":null,"pollInterval":null}` | Setting that allows you to update the default backupstore. | + | defaultBackupStore.backupTarget | `nil` | Endpoint used to access the default backupstore. (Options: "NFS", "CIFS", "AWS", "GCP", "AZURE") | + | defaultBackupStore.backupTargetCredentialSecret | `nil` | Name of the Kubernetes secret associated with the default backup target. | +diff --git a/chart/README.md.gotmpl b/chart/README.md.gotmpl +index 727a6ac4b96fcb0c8366fd22175600b426c7fd7f..5065e4178e9d9d79f7ce5508e58ca2a602203fad 100644 +--- a/chart/README.md.gotmpl ++++ b/chart/README.md.gotmpl +@@ -22,7 +22,36 @@ Longhorn is 100% open source software. Project source code is spread across a nu + 1. A container runtime compatible with Kubernetes (Docker v1.13+, containerd v1.3.7+, etc.) + 2. Kubernetes >= v1.34 + 3. Make sure `bash`, `curl`, `findmnt`, `grep`, `awk` and `blkid` has been installed in all nodes of the Kubernetes cluster. +-4. Make sure `open-iscsi` has been installed, and the `iscsid` daemon is running on all nodes of the Kubernetes cluster. For GKE, recommended Ubuntu as guest OS image since it contains `open-iscsi` already. ++4. In the default `v1-capable` mode, make sure `open-iscsi` is installed and the `iscsid` daemon is running on all nodes. This includes mixed V1/V2 installations. For GKE, Ubuntu includes `open-iscsi` already. ++ ++## V2-only installation capability ++ ++`dataEngineMode: v2-only` selects an explicit installation capability supported by a compatible Longhorn Manager release. It is separate from enabling V2 alongside V1. The default `v1-capable` mode preserves existing manager commands and iSCSI prerequisites. ++ ++For a fresh V2-only installation with a chart-owned StorageClass, set these values and use the tag and digest from the matching signed manager release: ++ ++```yaml ++dataEngineMode: v2-only ++image: ++ longhorn: ++ manager: ++ registry: ghcr.io ++ repository: devantler-tech/platform-longhorn-manager ++ tag: @sha256: ++defaultSettings: ++ v1DataEngine: false ++ v2DataEngine: true ++persistence: ++ dataEngine: v2 ++``` ++ ++The chart rejects the stock manager repository and unresolved or conflicting engine settings, including when upgrade hooks are disabled. The Platform manager repository requires a published `v1.13.0-platform.` tag. Daemon, global manager and pre-upgrade commands receive the capability explicitly. The manager separately checks live storage and runtime before starting. V2 nodes still require their normal V2 disk, CPU, kernel, NVMe/TCP and memory/hugepage prerequisites. ++ ++For an existing installation, retain iSCSI throughout migration. Move or restore every V1 volume and retained snapshot that must be kept, prove V2 recovery, and retire V1 engines, replicas and instance-manager runtime before selecting V2-only mode. This value does not convert existing data, snapshots or immutable StorageClass parameters. Remove host iSCSI dependencies only after the V2-only rollout is verified on every node. ++ ++Apply and verify the explicit engine settings and StorageClass ownership while still in `v1-capable` mode first, then opt into `v2-only` in a separate upgrade. The pre-upgrade check receives the requested capability but validates the installed settings and runtime before the new ConfigMap is applied. ++ ++The chart's `longhorn-storageclass` ConfigMap causes the manager to delete and recreate the generated `longhorn` StorageClass when its embedded definition changes. Prepare and select an external V2 StorageClass and set `persistence.createStorageClass=false` during an existing installation's migration, or establish that the chart-owned class already has matching parameters. The chart cannot verify that live ownership or parameter match during rendering. + + ## Upgrading to Kubernetes v1.25+ + +diff --git a/chart/questions.yaml b/chart/questions.yaml +index 120da852bfff00c27811a502b8115e5513b0ebd5..5b58a560dc6b87b760187c7168bfcb2ffa40cf97 100644 +--- a/chart/questions.yaml ++++ b/chart/questions.yaml +@@ -2,6 +2,15 @@ categories: + - storage + namespace: longhorn-system + questions: ++ - variable: dataEngineMode ++ default: v1-capable ++ description: Installation capability. v2-only requires explicit V1 disabled and V2 enabled settings, a V2 chart-owned StorageClass, and a compatible manager release. Migrate existing V1 storage before opting in. ++ label: Data Engine Installation Capability ++ type: enum ++ options: ++ - v1-capable ++ - v2-only ++ group: Longhorn Settings + - variable: image.defaultImage + default: 'true' + description: Use default Longhorn images +diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl +index db6b9350e94dccca7a147330ed6300015bb69fa3..95f1aa2eb0452e5ae70aee01deae90859a8f9864 100644 +--- a/chart/templates/_helpers.tpl ++++ b/chart/templates/_helpers.tpl +@@ -89,3 +89,31 @@ When .Values.global.timezone is set, this snippet renders a TZ env var. + value: {{ .Values.global.timezone | quote }} + {{- end }} + {{- end -}} ++ ++{{/* Validate installation capability even when all optional hooks are disabled. */}} ++{{- define "longhorn.validateDataEngineMode" -}} ++{{- $mode := .Values.dataEngineMode -}} ++{{- if not (kindIs "string" $mode) -}} ++ {{- fail "dataEngineMode must be v1-capable or v2-only" -}} ++{{- else if not (or (eq $mode "v1-capable") (eq $mode "v2-only")) -}} ++ {{- fail "dataEngineMode must be v1-capable or v2-only" -}} ++{{- end -}} ++{{- if eq $mode "v2-only" -}} ++ {{- if or (ne (toString .Values.defaultSettings.v1DataEngine) "false") (ne (toString .Values.defaultSettings.v2DataEngine) "true") -}} ++ {{- fail "v2-only requires explicit defaultSettings.v1DataEngine=false and defaultSettings.v2DataEngine=true" -}} ++ {{- end -}} ++ {{- if and .Values.persistence.createStorageClass (ne .Values.persistence.dataEngine "v2") -}} ++ {{- fail "v2-only requires persistence.dataEngine=v2 when persistence.createStorageClass=true" -}} ++ {{- end -}} ++ {{- $managerRepository := .Values.image.longhorn.manager.repository -}} ++ {{- if or (empty $managerRepository) (eq $managerRepository "longhornio/longhorn-manager") -}} ++ {{- fail "v2-only requires an explicit compatible image.longhorn.manager.repository" -}} ++ {{- end -}} ++ {{- if eq $managerRepository "devantler-tech/platform-longhorn-manager" -}} ++ {{- $managerTag := toString .Values.image.longhorn.manager.tag -}} ++ {{- if not (regexMatch `^v1\.13\.0-platform\.[1-9][0-9]{0,11}(@sha256:[a-f0-9]{64})?$` $managerTag) -}} ++ {{- fail "v2-only requires a published v1.13.0-platform. tag for the Platform manager" -}} ++ {{- end -}} ++ {{- end -}} ++{{- end -}} ++{{- end -}} +diff --git a/chart/templates/daemonset-sa.yaml b/chart/templates/daemonset-sa.yaml +index 35159428ff1dd9200cc81e3b8d98f08652fefab2..5715b5b6b8a89e289b2e2f50f93b5b93ba13a882 100644 +--- a/chart/templates/daemonset-sa.yaml ++++ b/chart/templates/daemonset-sa.yaml +@@ -24,6 +24,10 @@ spec: + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true ++ {{- if eq .Values.dataEngineMode "v2-only" }} ++ runAsUser: 0 ++ runAsGroup: 0 ++ {{- end }} + command: + - longhorn-manager + - -d +@@ -31,6 +35,10 @@ spec: + - -j + {{- end }} + - daemon ++ {{- if eq .Values.dataEngineMode "v2-only" }} ++ - --data-engine-mode ++ - v2-only ++ {{- end }} + - --engine-image + - "{{ with (coalesce .Values.global.imageRegistry (include "registry_url" .) .Values.image.longhorn.engine.registry "docker.io") }}{{ . }}/{{ end }}{{ .Values.image.longhorn.engine.repository }}:{{ .Values.image.longhorn.engine.tag }}" + - --instance-manager-image +diff --git a/chart/templates/deployment-driver.yaml b/chart/templates/deployment-driver.yaml +index 664471fc0c5dc08ff047ad47735d7c5df3ff59b3..32ff26a1f6f8fc7432736c8f1ec8d592a0487b04 100644 +--- a/chart/templates/deployment-driver.yaml ++++ b/chart/templates/deployment-driver.yaml +@@ -153,6 +153,9 @@ spec: + serviceAccountName: longhorn-service-account + securityContext: + runAsUser: 0 ++ {{- if eq .Values.dataEngineMode "v2-only" }} ++ runAsGroup: 0 ++ {{- end }} + {{- if .Values.enableGoCoverDir }} + volumes: + - name: go-cover-dir +diff --git a/chart/templates/deployment-global-manager.yaml b/chart/templates/deployment-global-manager.yaml +index 46385acd4e164b53197e875f68feb7325bb82b5d..7d5ad21bb48e0eb71ba63df52b61e645faa357ed 100644 +--- a/chart/templates/deployment-global-manager.yaml ++++ b/chart/templates/deployment-global-manager.yaml +@@ -30,6 +30,10 @@ spec: + - -j + {{- end }} + - global ++ {{- if eq .Values.dataEngineMode "v2-only" }} ++ - --data-engine-mode ++ - v2-only ++ {{- end }} + ports: + - containerPort: 9505 + name: healthz +diff --git a/chart/templates/preupgrade-job.yaml b/chart/templates/preupgrade-job.yaml +index 05b6d020825147f906217c5fd4b4dd58e4e8ed36..e103759b72f3376d8f95a96e15806e8e2d7646c2 100644 +--- a/chart/templates/preupgrade-job.yaml ++++ b/chart/templates/preupgrade-job.yaml +@@ -22,9 +22,17 @@ spec: + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true ++ {{- if eq .Values.dataEngineMode "v2-only" }} ++ runAsUser: 0 ++ runAsGroup: 0 ++ {{- end }} + command: + - longhorn-manager + - pre-upgrade ++ {{- if eq .Values.dataEngineMode "v2-only" }} ++ - --data-engine-mode ++ - v2-only ++ {{- end }} + volumeMounts: + - name: proc + mountPath: /host/proc/ +diff --git a/chart/templates/storageclass.yaml b/chart/templates/storageclass.yaml +index c901003da6b876ce51e05a09b149d970a45dc497..e6ce3db7173f74f7725fb4f003e93711cbbd7fab 100644 +--- a/chart/templates/storageclass.yaml ++++ b/chart/templates/storageclass.yaml +@@ -63,8 +63,8 @@ data: + {{- end }} + {{- if .Values.persistence.disableRevisionCounter }} + disableRevisionCounter: "{{ .Values.persistence.disableRevisionCounter }}" +- dataEngine: "{{ .Values.persistence.dataEngine }}" + {{- end }} ++ dataEngine: "{{ .Values.persistence.dataEngine }}" + {{- if .Values.persistence.backupTargetName }} + backupTargetName: "{{ .Values.persistence.backupTargetName }}" + {{- end }} +diff --git a/chart/templates/validate-data-engine-mode.yaml b/chart/templates/validate-data-engine-mode.yaml +new file mode 100644 +index 0000000000000000000000000000000000000000..db5629dc450920863218854f554d5fae04f032f8 +--- /dev/null ++++ b/chart/templates/validate-data-engine-mode.yaml +@@ -0,0 +1 @@ ++{{- include "longhorn.validateDataEngineMode" . -}} +diff --git a/chart/values.yaml b/chart/values.yaml +index 4d57d3c159fc355ea9be0ca5081ca0fa0f6448ba..b7422f04e0f4c6b7c816d2d2f1eca8fb3997ee8d 100644 +--- a/chart/values.yaml ++++ b/chart/values.yaml +@@ -1,6 +1,8 @@ + # Default values for longhorn. + # This is a YAML-formatted file. + # Declare variables to be passed into your templates. ++# -- Installation capability. v1-capable preserves mixed-engine behavior and iSCSI requirements. v2-only requires explicit V1=false/V2=true settings and a V2 StorageClass when chart-owned. Use a manager release supporting this option. ++dataEngineMode: v1-capable + global: + # -- Global override for container image registry. + imageRegistry: "" +diff --git a/scripts/test-data-engine-mode-chart.sh b/scripts/test-data-engine-mode-chart.sh +new file mode 100644 +index 0000000000000000000000000000000000000000..93722a7fa14c005956793b9538f4c28f1fac863b +--- /dev/null ++++ b/scripts/test-data-engine-mode-chart.sh +@@ -0,0 +1,111 @@ ++#!/usr/bin/env bash ++set -euo pipefail ++ ++chart_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" ++chart_scratch="$(mktemp -d)" ++trap 'rm -rf "$chart_scratch"' EXIT ++failed=0 ++ ++render() { ++ helm template longhorn "$chart_root/chart" --namespace longhorn-system --kube-version 1.34.0 "$@" ++} ++ ++expect_failure() { ++ if render "$@" >"$chart_scratch/rejected.yaml" 2>"$chart_scratch/rejected.stderr"; then ++ printf 'FAIL: invalid capability values rendered: %s\n' "$*" >&2 ++ failed=1 ++ fi ++} ++ ++for mode in unknown V2-Only '' null 1; do ++ expect_failure --set "dataEngineMode=$mode" ++done ++expect_failure --set dataEngineMode=v2-only ++expect_failure --set dataEngineMode=v2-only --set defaultSettings.v1DataEngine=true --set defaultSettings.v2DataEngine=true --set persistence.dataEngine=v2 ++expect_failure --set dataEngineMode=v2-only --set defaultSettings.v1DataEngine=false --set defaultSettings.v2DataEngine=false --set persistence.dataEngine=v2 ++expect_failure --set dataEngineMode=v2-only --set defaultSettings.v1DataEngine=false --set defaultSettings.v2DataEngine=true ++expect_failure --set dataEngineMode=v2-only --set preUpgradeChecker.jobEnabled=false --set preUpgradeChecker.upgradeVersionCheck=false ++ ++render >"$chart_scratch/default.yaml" ++render --set dataEngineMode=v1-capable >"$chart_scratch/explicit-default.yaml" ++cmp "$chart_scratch/default.yaml" "$chart_scratch/explicit-default.yaml" ++if grep -q -- '--data-engine-mode' "$chart_scratch/default.yaml"; then ++ printf '%s\n' 'FAIL: default argv changed' >&2 ++ failed=1 ++fi ++if [[ -n "${CHART_DEFAULT_BASELINE:-}" ]]; then ++ cmp "$CHART_DEFAULT_BASELINE" "$chart_scratch/default.yaml" ++fi ++ ++v2=(--set dataEngineMode=v2-only --set defaultSettings.v1DataEngine=false --set defaultSettings.v2DataEngine=true --set persistence.dataEngine=v2) ++expect_failure "${v2[@]}" ++expect_failure "${v2[@]}" --set image.longhorn.manager.registry=mirror.example ++expect_failure "${v2[@]}" --set-string 'image.longhorn.manager.repository=' ++platform_manager=(--set image.longhorn.manager.registry=ghcr.io --set image.longhorn.manager.repository=devantler-tech/platform-longhorn-manager) ++expect_failure "${v2[@]}" --set image.longhorn.manager.repository=devantler-tech/platform-longhorn-manager ++expect_failure "${v2[@]}" "${platform_manager[@]}" ++for tag in latest v1.13.0-platform.0 v1.13.0-platform.1@sha256:deadbeef; do ++ expect_failure "${v2[@]}" "${platform_manager[@]}" --set-string "image.longhorn.manager.tag=$tag" ++done ++render "${v2[@]}" --set image.longhorn.manager.registry=registry.example --set image.longhorn.manager.repository=compatible/longhorn-manager >"$chart_scratch/custom.yaml" ++grep -q 'image: registry.example/compatible/longhorn-manager:v1.13.0' "$chart_scratch/custom.yaml" ++manager=("${platform_manager[@]}" --set image.longhorn.manager.tag=v1.13.0-platform.1) ++v2+=("${manager[@]}") ++render "${v2[@]}" >"$chart_scratch/v2.yaml" ++grep -q 'image: ghcr.io/devantler-tech/platform-longhorn-manager:v1.13.0-platform.1' "$chart_scratch/v2.yaml" ++pinned_tag="v1.13.0-platform.1@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" ++render "${v2[@]}" --set-string "image.longhorn.manager.tag=$pinned_tag" >"$chart_scratch/pinned.yaml" ++grep -Fq "image: ghcr.io/devantler-tech/platform-longhorn-manager:$pinned_tag" "$chart_scratch/pinned.yaml" ++if [[ "$(grep -c -- '--data-engine-mode' "$chart_scratch/v2.yaml" || true)" != 3 ]]; then ++ printf '%s\n' 'FAIL: V2 capability must reach daemon, global manager and pre-upgrade' >&2 ++ failed=1 ++fi ++for launch in daemon global pre-upgrade; do ++ if ! awk -v launch="$launch" ' ++ $0 ~ "^[[:space:]]*-[[:space:]]+" launch "$" { state=1; next } ++ state==1 { if ($0 !~ /^[[:space:]]*-[[:space:]]+--data-engine-mode$/) exit 1; state=2; next } ++ state==2 { if ($0 !~ /^[[:space:]]*-[[:space:]]+v2-only$/) exit 1; found=1; state=0 } ++ END { if (!found) exit 1 } ++ ' "$chart_scratch/v2.yaml"; then ++ printf 'FAIL: explicit V2 capability missing after %s\n' "$launch" >&2 ++ failed=1 ++ fi ++done ++for template in daemonset-sa preupgrade-job deployment-driver; do ++ render "${v2[@]}" --show-only "templates/$template.yaml" >"$chart_scratch/runtime.yaml" ++ if ! grep -q 'runAsUser: 0' "$chart_scratch/runtime.yaml" || ! grep -q 'runAsGroup: 0' "$chart_scratch/runtime.yaml"; then ++ printf '%s\n' "FAIL: host runtime identity missing from $template" >&2 ++ failed=1 ++ fi ++done ++render "${v2[@]}" --show-only templates/deployment-global-manager.yaml >"$chart_scratch/runtime.yaml" ++if grep -Eq 'runAs(User|Group): 0' "$chart_scratch/runtime.yaml"; then ++ printf '%s\n' 'FAIL: the global controller inherited a root override' >&2 ++ failed=1 ++fi ++grep -q 'v1-data-engine: false' "$chart_scratch/v2.yaml" ++grep -q 'v2-data-engine: true' "$chart_scratch/v2.yaml" ++for counter in false ''; do ++ render "${v2[@]}" --set "persistence.disableRevisionCounter=$counter" >"$chart_scratch/counter.yaml" ++ if ! grep -q 'dataEngine: "v2"' "$chart_scratch/counter.yaml"; then ++ printf '%s\n' 'FAIL: revision-counter choice erased the StorageClass engine' >&2 ++ failed=1 ++ fi ++done ++render "${v2[@]}" --set-string persistence.disableRevisionCounter=false >"$chart_scratch/counter-string.yaml" ++grep -q 'dataEngine: "v2"' "$chart_scratch/counter-string.yaml" ++render "${manager[@]}" --set dataEngineMode=v2-only --set defaultSettings.v1DataEngine=false --set defaultSettings.v2DataEngine=true --set persistence.createStorageClass=false >"$chart_scratch/external-class.yaml" ++if grep -q 'name: longhorn-storageclass' "$chart_scratch/external-class.yaml"; then ++ printf '%s\n' 'FAIL: chart claimed an externally owned StorageClass' >&2 ++ failed=1 ++fi ++render "${v2[@]}" --set preUpgradeChecker.jobEnabled=false >"$chart_scratch/no-hook.yaml" ++if [[ "$(grep -c -- '--data-engine-mode' "$chart_scratch/no-hook.yaml" || true)" != 2 ]]; then ++ printf '%s\n' 'FAIL: disabling the hook changed manager capability' >&2 ++ failed=1 ++fi ++ ++if (( failed )); then ++ exit 1 ++fi ++printf '%s\n' 'PASS: chart capability validation, native argv, StorageClass intent and default compatibility' diff --git a/k8s/providers/hetzner/infrastructure/controllers/longhorn/longhorn-manager-v1.13.0-v2-only.patch b/k8s/providers/hetzner/infrastructure/controllers/longhorn/longhorn-manager-v1.13.0-v2-only.patch new file mode 100644 index 000000000..857783e9f --- /dev/null +++ b/k8s/providers/hetzner/infrastructure/controllers/longhorn/longhorn-manager-v1.13.0-v2-only.patch @@ -0,0 +1,6346 @@ +diff --git a/api/data_engine_mode_test.go b/api/data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..df46ba177e77e63a4cb18966c0583139b33b396e +--- /dev/null ++++ b/api/data_engine_mode_test.go +@@ -0,0 +1,37 @@ ++package api ++ ++import ( ++ "testing" ++ ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++func TestVolumeSchemaDefaultsToTheInstallationEngine(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ schemas, err := NewSchemaForDataEngineMode(mode) ++ if err != nil { ++ t.Fatal(err) ++ } ++ volume, ok := schemas.CheckSchema("volume") ++ if !ok { ++ t.Fatal("missing volume schema") ++ } ++ field, ok := volume.CheckField("dataEngine") ++ want := longhorn.DataEngineTypeV1 ++ if mode == types.DataEngineModeV2Only { ++ want = longhorn.DataEngineTypeV2 ++ } ++ if !ok || field.Default != want || !field.Create || !field.Required { ++ t.Fatalf("mode=%s has the wrong new-volume engine default: %+v", mode, field) ++ } ++ } ++ if _, err := NewSchemaForDataEngineMode("unknown"); err == nil { ++ t.Fatal("unknown capability published an API schema") ++ } ++ legacy, ok := NewSchema().CheckSchema("volume") ++ if !ok || legacy.ResourceFields["dataEngine"].Default != longhorn.DataEngineTypeV1 { ++ t.Fatal("V2 schema construction mutated the legacy schema") ++ } ++} +diff --git a/api/model.go b/api/model.go +index af700d0d14ef685d42021c4c29256bc03118a025..38547dc5f08f4129046e1bc9755c429e56b13fa3 100644 +--- a/api/model.go ++++ b/api/model.go +@@ -773,6 +773,27 @@ type SnapshotCRListOutput struct { + Type string `json:"type"` + } + ++func NewSchemaForDataEngineMode(mode types.DataEngineMode) (*client.Schemas, error) { ++ parsed, err := types.ParseDataEngineMode(string(mode)) ++ if err != nil { ++ return nil, err ++ } ++ schemas := NewSchema() ++ if parsed == types.DataEngineModeV2Only { ++ volume, ok := schemas.CheckSchema("volume") ++ if !ok { ++ return nil, fmt.Errorf("missing volume schema for installation capability") ++ } ++ field, ok := volume.CheckField("dataEngine") ++ if !ok { ++ return nil, fmt.Errorf("missing volume data engine field for installation capability") ++ } ++ field.Default = longhorn.DataEngineTypeV2 ++ volume.ResourceFields["dataEngine"] = field ++ } ++ return schemas, nil ++} ++ + func NewSchema() *client.Schemas { + schemas := &client.Schemas{} + +diff --git a/api/router.go b/api/router.go +index 23c60479e4b4c05beb69770e98033f1ac723605e..3964f90c93f176096d0875eedbded44f42a3b4c8 100644 +--- a/api/router.go ++++ b/api/router.go +@@ -45,8 +45,11 @@ func writeErr(rw http.ResponseWriter, req *http.Request, err error, statusCode i + } + } + +-func NewRouter(s *Server) *mux.Router { +- schemas := NewSchema() ++func NewRouter(s *Server) (*mux.Router, error) { ++ schemas, err := NewSchemaForDataEngineMode(s.m.GetDataEngineMode()) ++ if err != nil { ++ return nil, err ++ } + r := mux.NewRouter().StrictSlash(true) + f := HandleError + +@@ -330,5 +333,5 @@ func NewRouter(s *Server) *mux.Router { + r.Path("/v1/ws/volumeattachments").Handler(f(schemas, volumeAttachmentListStream)) + r.Path("/v1/ws/{period}/volumeattachments").Handler(f(schemas, volumeAttachmentListStream)) + +- return r ++ return r, nil + } +diff --git a/app/daemon.go b/app/daemon.go +index 286a21a1e534e29c339fbb87fd120bc0e786a894..643d365db532f41c5d0527cbe18b881d83cafbe4 100644 +--- a/app/daemon.go ++++ b/app/daemon.go +@@ -69,6 +69,7 @@ func DaemonCmd() *cli.Command { + return &cli.Command{ + Name: "daemon", + Flags: []cli.Flag{ ++ dataEngineModeFlag(), + &cli.StringFlag{ + Name: FlagEngineImage, + Usage: "Specify Longhorn engine image", +@@ -128,7 +129,7 @@ func DaemonCmd() *cli.Command { + // - ctx: The context used to manage the webhook servers lifecycle. + // - kubeconfigPath: The path to the kubeconfig file. + // - currentNodeID: The ID of the current node attempting to acquire the leadership. +-func startWebhooksByLeaderElection(ctx context.Context, kubeconfigPath, currentNodeID string) (*client.Clients, error) { ++func startWebhooksByLeaderElection(ctx context.Context, kubeconfigPath, currentNodeID string, mode types.DataEngineMode) (*client.Clients, error) { + config, err := rest.InClusterConfig() + if err != nil { + return nil, errors.Wrap(err, "failed to get client config") +@@ -170,7 +171,7 @@ func startWebhooksByLeaderElection(ctx context.Context, kubeconfigPath, currentN + // The services in the longhorn-manager can then start to receive the requests. + // Conversion webhook does not use datastore, since it is a prerequisite for + // datastore operation. +- clientsWithoutDatastore, err := client.NewClients(kubeconfigPath, false, ctx.Done()) ++ clientsWithoutDatastore, err := client.NewClients(kubeconfigPath, false, mode, ctx.Done()) + if err != nil { + return err + } +@@ -199,7 +200,7 @@ func startWebhooksByLeaderElection(ctx context.Context, kubeconfigPath, currentN + var err error + // Neither the admission webhook (longhorn-system Pods only, AWS-IAM secret path) nor the + // node-local controllers read workload Pods, so this process stays off a cluster-wide Pod watch. +- clients, err = client.NewClientsForNodeLocal(kubeconfigPath, ctx.Done()) ++ clients, err = client.NewClientsForNodeLocal(kubeconfigPath, mode, ctx.Done()) + if err != nil { + return err + } +@@ -266,9 +267,10 @@ func startWebhooksByLeaderElection(ctx context.Context, kubeconfigPath, currentN + } + + func startManager(cmd *cli.Command) error { +- var ( +- err error +- ) ++ mode, err := types.ParseDataEngineMode(cmd.String(FlagDataEngineMode)) ++ if err != nil { ++ return err ++ } + + engineImage := cmd.String(FlagEngineImage) + if engineImage == "" { +@@ -300,7 +302,7 @@ func startManager(cmd *cli.Command) error { + } + kubeconfigPath := cmd.String(FlagKubeConfig) + +- if err := environmentCheck(); err != nil { ++ if err := environmentCheck(mode); err != nil { + return errors.Wrap(err, "failed to check environment, please make sure you have iscsiadm/open-iscsi installed on the host") + } + +@@ -318,10 +320,13 @@ func startManager(cmd *cli.Command) error { + + logger := logrus.StandardLogger().WithField("node", currentNodeID) + +- clients, err := startWebhooksByLeaderElection(ctx, kubeconfigPath, currentNodeID) ++ clients, err := startWebhooksByLeaderElection(ctx, kubeconfigPath, currentNodeID, mode) + if err != nil { + return err + } ++ if err := clients.Datastore.ValidateDataEngineModeStartup(ctx); err != nil { ++ return err ++ } + + // Leader-elected periodic webhook cert rotation; only the leader writes. + go webhookcert.RotateLoop(ctx, clients.K8s, clients.Namespace, currentNodeID) +@@ -337,6 +342,26 @@ func startManager(cmd *cli.Command) error { + if err := upgrade.Upgrade(kubeconfigPath, currentNodeID, managerImage, cmd.Bool(FlagUpgradeVersionCheck)); err != nil { + return err + } ++ if err := clients.Datastore.ValidateDataEngineModeStartup(ctx); err != nil { ++ return err ++ } ++ if err := clients.Datastore.InitializeDataEngineModeSettings(ctx); err != nil { ++ return err ++ } ++ defaultImageSettings := map[types.SettingName]string{ ++ types.SettingNameDefaultEngineImage: engineImage, ++ types.SettingNameDefaultInstanceManagerImage: instanceManagerImage, ++ types.SettingNameDefaultBackingImageManagerImage: backingImageManagerImage, ++ types.SettingNameSupportBundleManagerImage: supportBundleManagerImage, ++ } ++ if mode == types.DataEngineModeV2Only { ++ if err := clients.Datastore.UpdateCustomizedSettings(defaultImageSettings); err != nil { ++ return err ++ } ++ if err := clients.Datastore.ValidateDataEngineModeState(ctx); err != nil { ++ return err ++ } ++ } + + proxyConnCounter := util.NewAtomicCounter() + +@@ -355,14 +380,10 @@ func startManager(cmd *cli.Command) error { + + metricscollector.InitMetricsCollectorSystem(logger, currentNodeID, clients.Datastore, kubeconfigPath, proxyConnCounter) + +- defaultImageSettings := map[types.SettingName]string{ +- types.SettingNameDefaultEngineImage: engineImage, +- types.SettingNameDefaultInstanceManagerImage: instanceManagerImage, +- types.SettingNameDefaultBackingImageManagerImage: backingImageManagerImage, +- types.SettingNameSupportBundleManagerImage: supportBundleManagerImage, +- } +- if err := clients.Datastore.UpdateCustomizedSettings(defaultImageSettings); err != nil { +- return err ++ if mode != types.DataEngineModeV2Only { ++ if err := clients.Datastore.UpdateCustomizedSettings(defaultImageSettings); err != nil { ++ return err ++ } + } + + if err := clients.Datastore.CreateOrUpdateDefaultBackupTarget(); err != nil { +@@ -382,7 +403,11 @@ func startManager(cmd *cli.Command) error { + } + + server := api.NewServer(m, wsc) +- router := http.Handler(api.NewRouter(server)) ++ apiRouter, err := api.NewRouter(server) ++ if err != nil { ++ return err ++ } ++ router := http.Handler(apiRouter) + router = util.FilteredLoggingHandler(os.Stdout, router) + router = handlers.ProxyHeaders(router) + +@@ -408,7 +433,11 @@ func startManager(cmd *cli.Command) error { + return nil + } + +-func environmentCheck() error { ++func environmentCheck(mode types.DataEngineMode) error { ++ return checkISCSIForDataEngineMode(mode, checkISCSIInitiator) ++} ++ ++func checkISCSIInitiator() error { + // Here we only check if the necessary tool the iscsiadm is installed when Longhorn starts up. + // Others tools and settings like kernel versions, multipathd, nfs client, etc. are checked in the node controller (every 30 sec). + namespaces := []lhtypes.Namespace{lhtypes.NamespaceMnt, lhtypes.NamespaceNet} +diff --git a/app/data_engine_mode.go b/app/data_engine_mode.go +new file mode 100644 +index 0000000000000000000000000000000000000000..b6868c14c61d88469a64e062eac31487b5deaee2 +--- /dev/null ++++ b/app/data_engine_mode.go +@@ -0,0 +1,38 @@ ++package app ++ ++import ( ++ "fmt" ++ ++ "github.com/urfave/cli/v3" ++ ++ "github.com/longhorn/longhorn-manager/types" ++) ++ ++func checkISCSIForDataEngineMode(mode types.DataEngineMode, check func() error) error { ++ parsed, err := types.ParseDataEngineMode(string(mode)) ++ if err != nil { ++ return err ++ } ++ if !parsed.RequiresISCSI() { ++ return nil ++ } ++ if check == nil { ++ return fmt.Errorf("V1-capable mode requires an iSCSI host check") ++ } ++ return check() ++} ++ ++const FlagDataEngineMode = "data-engine-mode" ++ ++func dataEngineModeFlag() cli.Flag { ++ return &cli.StringFlag{ ++ Name: FlagDataEngineMode, ++ Value: string(types.DataEngineModeV1Capable), ++ Usage: "Data engine installation capability: v1-capable or v2-only", ++ ValidateDefaults: true, ++ Validator: func(value string) error { ++ _, err := types.ParseDataEngineMode(value) ++ return err ++ }, ++ } ++} +diff --git a/app/data_engine_mode_test.go b/app/data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..78293d4970d21f26785c56f02446b39c9ebaa3bb +--- /dev/null ++++ b/app/data_engine_mode_test.go +@@ -0,0 +1,81 @@ ++package app ++ ++import ( ++ "context" ++ "errors" ++ "io" ++ "strings" ++ "testing" ++ ++ "github.com/urfave/cli/v3" ++ ++ "github.com/longhorn/longhorn-manager/types" ++) ++ ++func TestDataEngineModeSkipsOnlyTheISCSIHostCheck(t *testing.T) { ++ sentinel := errors.New("initiator unavailable") ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable, types.DataEngineModeV2Only, "unknown"} { ++ called := 0 ++ err := checkISCSIForDataEngineMode(mode, func() error { ++ called++ ++ return sentinel ++ }) ++ switch mode { ++ case types.DataEngineModeV2Only: ++ if err != nil || called != 0 { ++ t.Fatalf("V2-only required an iSCSI host executor: calls=%d err=%v", called, err) ++ } ++ case "unknown": ++ if err == nil || called != 0 || !strings.Contains(err.Error(), "unsupported data engine mode") { ++ t.Fatalf("unknown mode reached the host executor: calls=%d err=%v", called, err) ++ } ++ default: ++ if !errors.Is(err, sentinel) || called != 1 { ++ t.Fatalf("default mode lost the host prerequisite or its error: calls=%d err=%v", called, err) ++ } ++ } ++ } ++} ++ ++func TestDataEngineModeFlagAtEveryEntrypoint(t *testing.T) { ++ for _, constructor := range []func() *cli.Command{DaemonCmd, GlobalCmd, PreUpgradeCmd, SystemRolloutCmd} { ++ for _, test := range []struct { ++ input, want string ++ valid bool ++ }{ ++ {"", "v1-capable", true}, ++ {"v1-capable", "v1-capable", true}, ++ {"v2-only", "v2-only", true}, ++ {"v2", "", false}, ++ {"V2-ONLY", "", false}, ++ {" v2-only", "", false}, ++ } { ++ command := constructor() ++ t.Run(command.Name+"/"+test.input, func(t *testing.T) { ++ called, observed := false, "" ++ // Exercise the actual entrypoint's flag parser without starting ++ // its host executor, clients or process-wide fatal handler. ++ command.Action = func(_ context.Context, c *cli.Command) error { ++ called, observed = true, c.String("data-engine-mode") ++ return nil ++ } ++ command.Writer, command.ErrWriter = io.Discard, io.Discard ++ args := []string{"manager"} ++ if test.input != "" { ++ args = append(args, "--data-engine-mode", test.input) ++ } ++ if command.Name == "pre-upgrade" { ++ args = append(args, "--namespace", "longhorn-system") ++ } ++ err := command.Run(t.Context(), args) ++ if test.valid { ++ if err != nil || !called || observed != test.want { ++ t.Fatalf("mode intake: called=%t observed=%q err=%v", called, observed, err) ++ } ++ } else if err == nil || called || !strings.Contains(err.Error(), "unsupported data engine mode") { ++ t.Fatalf("invalid mode reached the command action or lost its diagnostic: called=%t err=%v", called, err) ++ } ++ }) ++ } ++ } ++} +diff --git a/app/global.go b/app/global.go +index 56a8fe426f5ff5fee31d4aa50e3787be88888e1b..d8d1ab087bc09a9e11510f3bb7967bdb5daf1021 100644 +--- a/app/global.go ++++ b/app/global.go +@@ -37,6 +37,7 @@ func GlobalCmd() *cli.Command { + Name: "global", + Usage: "Run the longhorn-global-manager workload (leader-elected).", + Flags: []cli.Flag{ ++ dataEngineModeFlag(), + &cli.StringFlag{ + Name: FlagKubeConfig, + Usage: "Specify path to kube config (optional)", +@@ -52,6 +53,10 @@ func GlobalCmd() *cli.Command { + } + + func startGlobalManager(cmd *cli.Command) error { ++ mode, err := types.ParseDataEngineMode(cmd.String(FlagDataEngineMode)) ++ if err != nil { ++ return err ++ } + kubeconfigPath := cmd.String(FlagKubeConfig) + + podName, err := util.GetRequiredEnv(types.EnvPodName) +@@ -98,10 +103,15 @@ func startGlobalManager(cmd *cli.Command) error { + + // Build clients/informers before leader election so every replica keeps a + // warm cache; a new leader starts against a synced cache (no fresh LIST). +- clients, err := client.NewClients(kubeconfigPath, true, electionCtx.Done()) ++ clients, err := client.NewClients(kubeconfigPath, true, mode, electionCtx.Done()) + if err != nil { + return errors.Wrap(err, "failed to build longhorn clients") + } ++ // The daemon bootstraps compatible engine Settings. A global replica must ++ // pass the uncached gate before it can acquire leadership and start workers. ++ if err := clients.Datastore.ValidateDataEngineModeState(electionCtx); err != nil { ++ return errors.Wrap(err, "data engine capability is not ready for global controllers") ++ } + + lock := &resourcelock.LeaseLock{ + LeaseMeta: metav1.ObjectMeta{ +diff --git a/app/pre_upgrade.go b/app/pre_upgrade.go +index 9a7383e068cb621bb8bea29cbde783927bc3b1be..a9eb375929df37cefb3542bf68669a7ea9654cb8 100644 +--- a/app/pre_upgrade.go ++++ b/app/pre_upgrade.go +@@ -13,8 +13,10 @@ import ( + "k8s.io/client-go/tools/record" + + corev1 "k8s.io/api/core/v1" ++ kubeclientset "k8s.io/client-go/kubernetes" + + "github.com/longhorn/longhorn-manager/constant" ++ "github.com/longhorn/longhorn-manager/datastore" + "github.com/longhorn/longhorn-manager/types" + + apputil "github.com/longhorn/longhorn-manager/app/util" +@@ -31,6 +33,7 @@ func PreUpgradeCmd() *cli.Command { + return &cli.Command{ + Name: "pre-upgrade", + Flags: []cli.Flag{ ++ dataEngineModeFlag(), + &cli.StringFlag{ + Name: FlagKubeConfig, + Usage: "Specify path to kube config (optional)", +@@ -46,7 +49,7 @@ func PreUpgradeCmd() *cli.Command { + logrus.Info("Running pre-upgrade...") + defer logrus.Info("Completed pre-upgrade.") + +- if err := preUpgrade(cmd); err != nil { ++ if err := preUpgrade(ctx, cmd); err != nil { + logrus.WithError(err).Fatalf("Failed to run pre-upgrade") + } + return nil +@@ -54,7 +57,11 @@ func PreUpgradeCmd() *cli.Command { + } + } + +-func preUpgrade(cmd *cli.Command) error { ++func preUpgrade(ctx context.Context, cmd *cli.Command) error { ++ mode, err := types.ParseDataEngineMode(cmd.String(FlagDataEngineMode)) ++ if err != nil { ++ return err ++ } + namespace := cmd.String(FlagNamespace) + + config, err := clientcmd.BuildConfigFromFlags("", cmd.String(FlagKubeConfig)) +@@ -84,7 +91,11 @@ func preUpgrade(cmd *cli.Command) error { + return errors.Wrap(err, "failed to get clientset") + } + +- err = newPreUpgrader(namespace, lhClient, eventRecorder).Run() ++ kubeClient, err := kubeclientset.NewForConfig(config) ++ if err != nil { ++ return errors.Wrap(err, "failed to get Kubernetes clientset") ++ } ++ err = newPreUpgrader(namespace, lhClient, kubeClient, eventRecorder, mode).Run(ctx) + if err != nil { + logrus.Warnf("Done with Run() ... err is %v", err) + } +@@ -93,16 +104,18 @@ func preUpgrade(cmd *cli.Command) error { + } + + type preUpgrader struct { +- namespace string +- lhClient lhclientset.Interface +- eventRecorder record.EventRecorder ++ namespace string ++ lhClient lhclientset.Interface ++ kubeClient kubeclientset.Interface ++ eventRecorder record.EventRecorder ++ dataEngineMode types.DataEngineMode + } + +-func newPreUpgrader(namespace string, lhClient lhclientset.Interface, eventRecorder record.EventRecorder) *preUpgrader { +- return &preUpgrader{namespace, lhClient, eventRecorder} ++func newPreUpgrader(namespace string, lhClient lhclientset.Interface, kubeClient kubeclientset.Interface, eventRecorder record.EventRecorder, mode types.DataEngineMode) *preUpgrader { ++ return &preUpgrader{namespace: namespace, lhClient: lhClient, kubeClient: kubeClient, eventRecorder: eventRecorder, dataEngineMode: mode} + } + +-func (u *preUpgrader) Run() error { ++func (u *preUpgrader) Run(ctx context.Context) error { + var err error + defer func() { + if err != nil { +@@ -114,11 +127,14 @@ func (u *preUpgrader) Run() error { + } + }() + ++ if err = datastore.ValidateDataEngineModeBootstrapState(ctx, u.dataEngineMode, u.namespace, u.lhClient, u.kubeClient); err != nil { ++ return err ++ } + if err = upgradeutil.CheckUpgradePath(u.namespace, u.lhClient, u.eventRecorder, true); err != nil { + return err + } + +- if err = environmentCheck(); err != nil { ++ if err = environmentCheck(u.dataEngineMode); err != nil { + err = errors.Wrap(err, "failed to check environment, please make sure you have iscsiadm/open-iscsi installed on the host") + return err + } +diff --git a/app/pre_upgrade_data_engine_mode_test.go b/app/pre_upgrade_data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..090cdf3666fe220caca33f41debbaebbc98c8602 +--- /dev/null ++++ b/app/pre_upgrade_data_engine_mode_test.go +@@ -0,0 +1,41 @@ ++package app ++ ++import ( ++ "strings" ++ "testing" ++ ++ "k8s.io/apimachinery/pkg/runtime" ++ "k8s.io/client-go/tools/record" ++ ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func TestV2OnlyPreUpgradeRejectsLegacyBeforeUpgradeAndHostChecks(t *testing.T) { ++ for _, legacy := range []bool{false, true} { ++ objects := []runtime.Object{} ++ if legacy { ++ objects = append(objects, &longhorn.Volume{ObjectMeta: metav1.ObjectMeta{Name: "retained-with-empty-engine", Namespace: "longhorn-system"}}) ++ } ++ lh, kube := lhfake.NewClientset(objects...), kubefake.NewClientset() ++ upgrader := newPreUpgrader("longhorn-system", lh, kube, record.NewFakeRecorder(10), types.DataEngineModeV2Only) ++ err := upgrader.Run(t.Context()) ++ if legacy { ++ if err == nil || !strings.Contains(err.Error(), "cannot use v2-only") { ++ t.Fatalf("legacy pre-upgrade did not fail at the storage gate: %v", err) ++ } ++ for _, action := range lh.Actions() { ++ if action.GetVerb() != "get" && action.GetVerb() != "list" { ++ t.Fatal("legacy pre-upgrade mutated storage") ++ } ++ } ++ } else if err != nil { ++ t.Fatalf("empty V2-only bootstrap required iSCSI: %v", err) ++ } ++ } ++} +diff --git a/app/system_rollout.go b/app/system_rollout.go +index 9029d162944b9dbfc4da6d9d8ae7cdf1f1ce1fc3..f5c4a625e5af0d3afb7571b2b52365f73657e277 100644 +--- a/app/system_rollout.go ++++ b/app/system_rollout.go +@@ -27,6 +27,7 @@ func SystemRolloutCmd() *cli.Command { + return &cli.Command{ + Name: "system-rollout", + Flags: []cli.Flag{ ++ dataEngineModeFlag(), + &cli.StringFlag{ + Name: FlagKubeConfig, + Usage: "Specify path to kube config (optional)", +@@ -37,7 +38,7 @@ func SystemRolloutCmd() *cli.Command { + }, + }, + Action: func(ctx context.Context, cmd *cli.Command) error { +- if err := systemRollout(cmd); err != nil { ++ if err := systemRollout(ctx, cmd); err != nil { + logrus.Fatalln(err) + } + return nil +@@ -45,7 +46,11 @@ func SystemRolloutCmd() *cli.Command { + } + } + +-func systemRollout(cmd *cli.Command) error { ++func systemRollout(ctx context.Context, cmd *cli.Command) error { ++ mode, err := types.ParseDataEngineMode(cmd.String(FlagDataEngineMode)) ++ if err != nil { ++ return err ++ } + if cmd.NArg() != 1 { + return errors.New("require SystemRestore name") + } +@@ -88,7 +93,10 @@ func systemRollout(cmd *cli.Command) error { + } + + informerFactories := util.NewInformerFactories(namespace, kubeClient, lhClient, 30*time.Second) +- ds := datastore.NewDataStoreForGlobal(namespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(namespace, lhClient, kubeClient, extensionsClient, informerFactories, mode) ++ if err := ds.ValidateDataEngineModeState(ctx); err != nil { ++ return errors.Wrap(err, "data engine capability is not ready for system rollout") ++ } + + logger := logrus.StandardLogger() + logrus.SetLevel(logrus.DebugLevel) +diff --git a/app/uninstall.go b/app/uninstall.go +index 1f34be33f833bab13522c5c1f98bfbaaf02d7823..cbdffdf3e4e97b503bae7442ac9ef9d6de48d1d4 100644 +--- a/app/uninstall.go ++++ b/app/uninstall.go +@@ -18,6 +18,7 @@ import ( + "github.com/longhorn/longhorn-manager/util" + + lhclientset "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned" ++ longhorntypes "github.com/longhorn/longhorn-manager/types" + ) + + const ( +@@ -80,7 +81,7 @@ func uninstall(cmd *cli.Command) error { + } + + informerFactories := util.NewInformerFactories(namespace, kubeClient, lhClient, 30*time.Second) +- ds := datastore.NewDataStoreForGlobal(namespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(namespace, lhClient, kubeClient, extensionsClient, informerFactories, longhorntypes.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + +diff --git a/controller/backup_target_controller_test.go b/controller/backup_target_controller_test.go +index b4594efd8f5f8ae0a0d9618908a86f98f322bf24..00805068b8ee1a94a7bb3c96b6a0e5c0e233e589 100644 +--- a/controller/backup_target_controller_test.go ++++ b/controller/backup_target_controller_test.go +@@ -124,7 +124,7 @@ func TestIsBackupTargetSyncRequired(t *testing.T) { + + func newTestBackupTargetController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*BackupTargetController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + proxyConnCounter := util.NewAtomicCounter() +diff --git a/controller/backup_volume_controller_test.go b/controller/backup_volume_controller_test.go +index ecde5648397994afc6c52e1ecd9ce56b81ad6231..8b94231b25edc4123cfb7383a1521551fa6a912f 100644 +--- a/controller/backup_volume_controller_test.go ++++ b/controller/backup_volume_controller_test.go +@@ -26,7 +26,7 @@ import ( + + func newTestBackupVolumeController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*BackupVolumeController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + proxyConnCounter := util.NewAtomicCounter() +diff --git a/controller/data_engine_capability_test.go b/controller/data_engine_capability_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..4514d46d899147803a52a927f0bb240967ffcbdc +--- /dev/null ++++ b/controller/data_engine_capability_test.go +@@ -0,0 +1,100 @@ ++package controller ++ ++import ( ++ "fmt" ++ "strings" ++ "testing" ++ ++ "github.com/sirupsen/logrus" ++ ++ "k8s.io/client-go/tools/record" ++ ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ ++ "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" ++ "github.com/longhorn/longhorn-manager/util" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func newRuntimeCapabilityTestStore(mode types.DataEngineMode) *datastore.DataStore { ++ lh, kube := lhfake.NewClientset(), kubefake.NewClientset() ++ factories := util.NewInformerFactories(TestNamespace, kube, lh, 0) ++ return datastore.NewDataStoreForNodeLocal(TestNamespace, lh, kube, nil, factories, mode) ++} ++ ++func TestV2OnlyRuntimeCreationCeilingPreservesCleanup(t *testing.T) { ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2, longhorn.DataEngineTypeAll, "unknown"} { ++ stub := &stubInstanceManagerHandler{getErr: fmt.Errorf("cannot find process")} ++ handler := &InstanceHandler{ds: newRuntimeCapabilityTestStore(types.DataEngineModeV2Only), instanceManagerHandler: stub, eventRecorder: record.NewFakeRecorder(10)} ++ object := &longhorn.Engine{ObjectMeta: metav1.ObjectMeta{Name: "guarded-instance", Namespace: TestNamespace}, Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: engine}}} ++ err := handler.createInstance(object.Name, engine, object) ++ allowed := engine == longhorn.DataEngineTypeV2 ++ if (err == nil) != allowed || stub.createCalled != allowed { ++ t.Fatalf("runtime ceiling: engine=%q create=%t err=%v", engine, stub.createCalled, err) ++ } ++ if err := handler.deleteInstance(object.Name, object); err != nil || !stub.deleteCalled { ++ t.Fatalf("runtime ceiling blocked cleanup: engine=%q err=%v", engine, err) ++ } ++ } ++ stub := &stubInstanceManagerHandler{getInstance: &longhorn.InstanceProcess{Status: longhorn.InstanceProcessStatus{State: longhorn.InstanceStateStopped}}} ++ handler := &InstanceHandler{ds: newRuntimeCapabilityTestStore(types.DataEngineModeV2Only), instanceManagerHandler: stub, eventRecorder: record.NewFakeRecorder(10)} ++ if err := handler.createInstance("stale-v1", longhorn.DataEngineTypeV1, &longhorn.Engine{}); err != errStaleInstanceReaped || !stub.deleteCalled || stub.createCalled { ++ t.Fatalf("stale V1 cleanup was blocked or reactivated: err=%v delete=%t create=%t", err, stub.deleteCalled, stub.createCalled) ++ } ++} ++ ++func TestV2OnlyNodeSelectionOverridesStaleV1Setting(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ lh, kube := lhfake.NewClientset(), kubefake.NewClientset() ++ factories := util.NewInformerFactories(TestNamespace, kube, lh, 0) ++ store := datastore.NewDataStoreForNodeLocal(TestNamespace, lh, kube, nil, factories, mode) ++ for name, value := range map[types.SettingName]string{types.SettingNameV1DataEngine: "true", types.SettingNameV2DataEngine: "false"} { ++ if err := factories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer().Add(&longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: TestNamespace}, Value: value}); err != nil { ++ t.Fatal(err) ++ } ++ } ++ controller := &NodeController{ds: store, baseController: &baseController{logger: logrus.NewEntry(logrus.New())}} ++ node := &longhorn.Node{ObjectMeta: metav1.ObjectMeta{Name: "storage-node"}, Spec: longhorn.NodeSpec{Disks: map[string]longhorn.DiskSpec{"disk": {}}}} ++ engines := controller.getImTypeDataEngines(node) ++ for kind, selected := range engines { ++ want := 1 ++ if mode == types.DataEngineModeV2Only { ++ want = 0 ++ } ++ if len(selected) != want { ++ t.Fatalf("mode=%s IM-kind=%s selected unsupported engines: %v", mode, kind, selected) ++ } ++ } ++ } ++} ++ ++func TestV2OnlyInstanceManagerCreationBoundaries(t *testing.T) { ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll, "unknown"} { ++ store := newRuntimeCapabilityTestStore(types.DataEngineModeV2Only) ++ nodeController := &NodeController{ds: store} ++ _, err := nodeController.createInstanceManager(&longhorn.Node{ObjectMeta: metav1.ObjectMeta{Name: "storage-node"}}, "old-im", "image", longhorn.InstanceManagerTypeAllInOne, engine) ++ if err == nil || !strings.Contains(err.Error(), "capability") { ++ t.Fatalf("unsupported IM creation did not fail at the capability boundary: engine=%q err=%v", engine, err) ++ } ++ imController := &InstanceManagerController{ds: store, baseController: &baseController{logger: logrus.NewEntry(logrus.New())}} ++ err = imController.createInstanceManagerPod(&longhorn.InstanceManager{ObjectMeta: metav1.ObjectMeta{Name: "old-im"}, Spec: longhorn.InstanceManagerSpec{DataEngine: engine}}) ++ if err == nil || !strings.Contains(err.Error(), "capability") { ++ t.Fatalf("unsupported IM Pod recreation did not fail at the capability boundary: engine=%q err=%v", engine, err) ++ } ++ } ++} ++ ++func TestV2OnlyEngineUpgradeCannotLaunchLegacyReplacement(t *testing.T) { ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll, "unknown"} { ++ controller := &EngineController{ds: newRuntimeCapabilityTestStore(types.DataEngineModeV2Only)} ++ object := &longhorn.Engine{Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: engine}}} ++ err := controller.UpgradeEngineInstance(object, logrus.NewEntry(logrus.New())) ++ if err == nil || !strings.Contains(err.Error(), "capability") { ++ t.Fatalf("legacy engine replacement reached client setup: engine=%q err=%v", engine, err) ++ } ++ } ++} +diff --git a/controller/engine_controller.go b/controller/engine_controller.go +index 9e4807586818e7747dc37341e1564b25afe283e4..007423920dc72b9579c4d95ecb81c619ae2758e2 100644 +--- a/controller/engine_controller.go ++++ b/controller/engine_controller.go +@@ -1172,15 +1172,8 @@ func (m *EngineMonitor) refresh(engine *longhorn.Engine) error { + engine.Status.CurrentSize = volumeInfo.Size + engine.Status.IsExpanding = volumeInfo.IsExpanding + +- if types.IsDataEngineV1(engine.Spec.DataEngine) { +- // For v1, the engine owns and starts its frontend directly. +- // For v2, frontend lifecycle is managed by EngineFrontendController. +- if engine.Status.Endpoint == "" && !engine.Spec.DisableFrontend && engine.Spec.Frontend != longhorn.VolumeFrontendEmpty { +- m.logger.Infof("Starting frontend %v", engine.Spec.Frontend) +- if err := engineClientProxy.VolumeFrontendStart(engine); err != nil { +- return errors.Wrapf(err, "failed to start frontend %v", engine.Spec.Frontend) +- } +- } ++ if err := m.startFrontend(engine, engineClientProxy); err != nil { ++ return err + } + + // The rebuild failure will be handled by ec.startRebuilding() +@@ -2276,6 +2269,9 @@ func (ec *EngineController) startRebuilding(e *longhorn.Engine, replicaName, add + defer func() { + err = errors.Wrapf(err, "failed to start rebuild for %v of %v", replicaName, e.Name) + }() ++ if err := validateEngineActivationCapability(ec.ds.DataEngineMode(), e.Spec.DataEngine); err != nil { ++ return err ++ } + + log := ec.logger.WithFields(logrus.Fields{"volume": e.Spec.VolumeName, "engine": e.Name}) + +@@ -2510,6 +2506,10 @@ func (ec *EngineController) runRebuild(rc *rebuildContext) { + if rc.cleanupProxy != rc.rebuildProxy { + defer rc.cleanupProxy.Close() + } ++ if err := ec.validateRebuildCapability(rc); err != nil { ++ rc.log.WithError(err).Warn("Cannot activate a rebuild outside the installation capability") ++ return ++ } + + if rc.autoCleanup { + allowSnapshotPurge, err := ec.snapshotConcurrentLimiter.CanStartSnapshotPurge(rc.cleanupProxy, rc.currentEngine, ec.ds) +@@ -2535,7 +2535,7 @@ func (ec *EngineController) runRebuild(rc *rebuildContext) { + // TODO: Before calling ReplicaAdd for the v2 frontend path, fetch the + // latest size/currentSize from currentEngine and pass them through once + // the proxy API consumes those fields for EngineFrontend-based rebuild. +- replicaAddErr = rc.rebuildProxy.ReplicaAdd(rc.rebuildObj, rc.replicaName, rc.replicaURL, false, rc.fastReplicaRebuild, nil, 0, rc.grpcTimeoutSeconds, rc.linkedCloneSource) ++ replicaAddErr = ec.addReplicaForRebuild(rc, false, nil, 0, rc.grpcTimeoutSeconds) + switch { + case replicaAddErr == nil: + // ok +@@ -2556,12 +2556,12 @@ func (ec *EngineController) runRebuild(rc *rebuildContext) { + if rc.engine.Spec.NodeID != "" { + ec.eventRecorder.Eventf(rc.engine, corev1.EventTypeNormal, constant.EventReasonRebuilding, + "Start rebuilding replica %v with Address %v for restore engine %v and volume %v", rc.replicaName, rc.addr, rc.engine.Name, rc.engine.Spec.VolumeName) +- replicaAddErr = rc.rebuildProxy.ReplicaAdd(rc.rebuildObj, rc.replicaName, rc.replicaURL, true, rc.fastReplicaRebuild, localSync, rc.fileSyncHTTPClientTO, 0, rc.linkedCloneSource) ++ replicaAddErr = ec.addReplicaForRebuild(rc, true, localSync, rc.fileSyncHTTPClientTO, 0) + } + } else { + ec.eventRecorder.Eventf(rc.engine, corev1.EventTypeNormal, constant.EventReasonRebuilding, + "Start rebuilding replica %v with Address %v for normal engine %v and volume %v", rc.replicaName, rc.addr, rc.engine.Name, rc.engine.Spec.VolumeName) +- replicaAddErr = rc.rebuildProxy.ReplicaAdd(rc.rebuildObj, rc.replicaName, rc.replicaURL, false, rc.fastReplicaRebuild, localSync, rc.fileSyncHTTPClientTO, rc.grpcTimeoutSeconds, rc.linkedCloneSource) ++ replicaAddErr = ec.addReplicaForRebuild(rc, false, localSync, rc.fileSyncHTTPClientTO, rc.grpcTimeoutSeconds) + } + } + +@@ -2869,6 +2869,9 @@ func (ec *EngineController) Upgrade(e *longhorn.Engine, log *logrus.Entry) (err + } + + func (ec *EngineController) UpgradeEngineInstance(e *longhorn.Engine, log *logrus.Entry) error { ++ if !ec.ds.DataEngineMode().AllowsDataEngine(e.Spec.DataEngine) { ++ return fmt.Errorf("data engine %q is outside the %q installation capability", e.Spec.DataEngine, ec.ds.DataEngineMode()) ++ } + frontend := e.Spec.Frontend + if e.Spec.DisableFrontend { + frontend = longhorn.VolumeFrontendEmpty +diff --git a/controller/engine_frontend_capability.go b/controller/engine_frontend_capability.go +new file mode 100644 +index 0000000000000000000000000000000000000000..a1b488a04d1f94570e573b3ecf16142faddd6ec9 +--- /dev/null ++++ b/controller/engine_frontend_capability.go +@@ -0,0 +1,33 @@ ++package controller ++ ++import ( ++ "github.com/cockroachdb/errors" ++ ++ "github.com/longhorn/longhorn-manager/engineapi" ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++func (m *EngineMonitor) startFrontend(engine *longhorn.Engine, client engineapi.EngineClientProxy) error { ++ // V2 frontend lifecycle belongs to EngineFrontendController. ++ if !types.IsDataEngineV1(engine.Spec.DataEngine) || engine.Status.Endpoint != "" || engine.Spec.DisableFrontend || engine.Spec.Frontend == longhorn.VolumeFrontendEmpty { ++ return nil ++ } ++ if err := validateEngineActivationCapability(m.ds.DataEngineMode(), engine.Spec.DataEngine); err != nil { ++ return err ++ } ++ m.logger.Infof("Starting frontend %v", engine.Spec.Frontend) ++ return errors.Wrapf(client.VolumeFrontendStart(engine), "failed to start frontend %v", engine.Spec.Frontend) ++} ++ ++func validateEngineActivationCapability(mode types.DataEngineMode, engine longhorn.DataEngineType) error { ++ parsed, err := types.ParseDataEngineMode(string(mode)) ++ if err != nil { ++ return err ++ } ++ if parsed == types.DataEngineModeV2Only && engine != longhorn.DataEngineTypeV2 { ++ return errors.New("v2-only capability cannot activate a legacy engine") ++ } ++ return nil ++} +diff --git a/controller/engine_frontend_capability_test.go b/controller/engine_frontend_capability_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..f980bf6eca35e4edc7c688fdf15d71cf6a4511d5 +--- /dev/null ++++ b/controller/engine_frontend_capability_test.go +@@ -0,0 +1,61 @@ ++package controller ++ ++import ( ++ "io" ++ "testing" ++ ++ "github.com/sirupsen/logrus" ++ ++ "github.com/longhorn/longhorn-manager/engineapi" ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++type frontendCapabilityRecorder struct { ++ engineapi.EngineClientProxy ++ starts int ++} ++ ++func (client *frontendCapabilityRecorder) VolumeFrontendStart(*longhorn.Engine) error { ++ client.starts++ ++ return nil ++} ++ ++func TestV2OnlyRetainedEngineCannotRestartLegacyFrontend(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ for _, engineType := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll, "unknown", longhorn.DataEngineTypeV2} { ++ t.Run(string(mode)+"/"+string(engineType), func(t *testing.T) { ++ logger := logrus.New() ++ logger.Out = io.Discard ++ monitor := &EngineMonitor{ds: newRuntimeCapabilityTestStore(mode), logger: logger} ++ engine := &longhorn.Engine{Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: engineType}, Frontend: longhorn.VolumeFrontendBlockDev}} ++ engine.Status.CurrentState = longhorn.InstanceStateRunning ++ client := &frontendCapabilityRecorder{} ++ err := monitor.startFrontend(engine, client) ++ legacy := engineType != longhorn.DataEngineTypeV2 ++ blocked := mode == types.DataEngineModeV2Only && legacy ++ expectedStarts := 0 ++ if legacy && !blocked { ++ expectedStarts = 1 ++ } ++ if (err != nil) != blocked || client.starts != expectedStarts { ++ t.Fatalf("frontend capability: mode=%q engine=%q starts=%d err=%v", mode, engineType, client.starts, err) ++ } ++ engine.Status.Endpoint = "existing" ++ client.starts = 0 ++ if err := monitor.startFrontend(engine, client); err != nil || client.starts != 0 { ++ t.Fatal("non-activating observation was blocked or restarted the frontend") ++ } ++ engine.Status.Endpoint, engine.Spec.DisableFrontend = "", true ++ if err := monitor.startFrontend(engine, client); err != nil || client.starts != 0 { ++ t.Fatal("disabled frontend was activated or its observation was blocked") ++ } ++ engine.Spec.DisableFrontend, engine.Spec.Frontend = false, longhorn.VolumeFrontendEmpty ++ if err := monitor.startFrontend(engine, client); err != nil || client.starts != 0 { ++ t.Fatal("empty frontend was activated or its observation was blocked") ++ } ++ }) ++ } ++ } ++} +diff --git a/controller/engine_frontend_controller_test.go b/controller/engine_frontend_controller_test.go +index 2f70c8f2f4dac39cdeed87ac57e72550e5650246..e08edad639e3de319b6d260bd6fababc753ffd9b 100644 +--- a/controller/engine_frontend_controller_test.go ++++ b/controller/engine_frontend_controller_test.go +@@ -372,7 +372,7 @@ func newTestEngineFrontendController( + informerFactories *util.InformerFactories, + controllerID string, + ) (*EngineFrontendController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + efc, err := NewEngineFrontendController( + logrus.StandardLogger(), +diff --git a/controller/engine_image_controller_test.go b/controller/engine_image_controller_test.go +index fc0f83be15223ad1c69f0a6f67683ce8dfca0822..acd332263789058794f811677a60599f187b1907 100644 +--- a/controller/engine_image_controller_test.go ++++ b/controller/engine_image_controller_test.go +@@ -55,7 +55,7 @@ func newTestEngineImageController(lhClient *lhfake.Clientset, kubeClient *fake.C + // Skip the Lister check that occurs on creation of an Instance Manager. + datastore.SkipListerCheck = true + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + ic, err := NewEngineImageController( +diff --git a/controller/engine_rebuild_capability.go b/controller/engine_rebuild_capability.go +new file mode 100644 +index 0000000000000000000000000000000000000000..249842729fcb2c89dac2b2e6b11371b4dc215714 +--- /dev/null ++++ b/controller/engine_rebuild_capability.go +@@ -0,0 +1,39 @@ ++package controller ++ ++import ( ++ "fmt" ++ ++ etypes "github.com/longhorn/longhorn-engine/pkg/types" ++ ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++func (ec *EngineController) addReplicaForRebuild(rc *rebuildContext, restore bool, localSync *etypes.FileLocalSync, fileSyncTimeout, grpcTimeout int64) error { ++ if err := ec.validateRebuildCapability(rc); err != nil { ++ return err ++ } ++ return rc.rebuildProxy.ReplicaAdd(rc.rebuildObj, rc.replicaName, rc.replicaURL, restore, rc.fastReplicaRebuild, localSync, fileSyncTimeout, grpcTimeout, rc.linkedCloneSource) ++} ++ ++func (ec *EngineController) validateRebuildCapability(rc *rebuildContext) error { ++ mode, err := types.ParseDataEngineMode(string(ec.ds.DataEngineMode())) ++ if err != nil || mode != types.DataEngineModeV2Only { ++ return err ++ } ++ if rc == nil || rc.engine == nil || rc.currentEngine == nil || rc.engine.Spec.DataEngine != longhorn.DataEngineTypeV2 || rc.currentEngine.Spec.DataEngine != longhorn.DataEngineTypeV2 { ++ return fmt.Errorf("v2-only rebuild requires explicitly V2 original and current engines") ++ } ++ switch object := rc.rebuildObj.(type) { ++ case *longhorn.Engine: ++ if object != nil && object.Spec.DataEngine == longhorn.DataEngineTypeV2 { ++ return nil ++ } ++ case *longhorn.EngineFrontend: ++ if object != nil && object.Spec.DataEngine == longhorn.DataEngineTypeV2 { ++ return nil ++ } ++ } ++ return fmt.Errorf("v2-only rebuild requires an explicitly V2 RPC target") ++} +diff --git a/controller/engine_rebuild_capability_test.go b/controller/engine_rebuild_capability_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..8969a3f5dfcbef34e336340888f0638f9319a25e +--- /dev/null ++++ b/controller/engine_rebuild_capability_test.go +@@ -0,0 +1,94 @@ ++package controller ++ ++import ( ++ "strings" ++ "testing" ++ ++ "github.com/sirupsen/logrus" ++ ++ etypes "github.com/longhorn/longhorn-engine/pkg/types" ++ imrpc "github.com/longhorn/types/pkg/generated/imrpc" ++ ++ "github.com/longhorn/longhorn-manager/engineapi" ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++type rebuildCapabilityRecorder struct { ++ engineapi.EngineClientProxy ++ adds int ++ restore bool ++} ++ ++func (client *rebuildCapabilityRecorder) ReplicaAdd(_ engineapi.DataEngineObject, _, _ string, restore, _ bool, _ *etypes.FileLocalSync, _, _ int64, _ *imrpc.LinkedCloneSource) error { ++ client.adds++ ++ client.restore = restore ++ return nil ++} ++ ++func TestV2OnlyRebuildCannotReactivateRetainedLegacyEngine(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ for _, engineType := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll, "unknown", longhorn.DataEngineTypeV2} { ++ for _, restore := range []bool{false, true} { ++ engine := &longhorn.Engine{Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: engineType}}} ++ client := &rebuildCapabilityRecorder{} ++ controller := &EngineController{ds: newRuntimeCapabilityTestStore(mode)} ++ context := &rebuildContext{engine: engine, currentEngine: engine.DeepCopy(), rebuildObj: engine, rebuildProxy: client} ++ err := controller.addReplicaForRebuild(context, restore, nil, 30, 60) ++ blocked := mode == types.DataEngineModeV2Only && engineType != longhorn.DataEngineTypeV2 ++ if blocked { ++ if err == nil || client.adds != 0 { ++ t.Fatalf("legacy rebuild activation escaped: mode=%q engine=%q restore=%t adds=%d err=%v", mode, engineType, restore, client.adds, err) ++ } ++ } else if err != nil || client.adds != 1 || client.restore != restore { ++ t.Fatalf("supported rebuild changed: mode=%q engine=%q restore=%t adds=%d err=%v", mode, engineType, restore, client.adds, err) ++ } ++ } ++ } ++ } ++} ++ ++func TestV2OnlyRebuildRebindsCurrentEngineAndRPCTarget(t *testing.T) { ++ for _, scenario := range []string{"compatible", "legacy-current", "legacy-target", "missing-current", "missing-target", "typed-nil-target"} { ++ t.Run(scenario, func(t *testing.T) { ++ engine := &longhorn.Engine{Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: longhorn.DataEngineTypeV2}}} ++ client := &rebuildCapabilityRecorder{} ++ controller := &EngineController{ds: newRuntimeCapabilityTestStore(types.DataEngineModeV2Only)} ++ frontend := &longhorn.EngineFrontend{Spec: longhorn.EngineFrontendSpec{ ++ InstanceSpec: longhorn.InstanceSpec{DataEngine: longhorn.DataEngineTypeV2}, ++ }} ++ context := &rebuildContext{engine: engine, currentEngine: engine.DeepCopy(), rebuildObj: frontend, rebuildProxy: client} ++ switch scenario { ++ case "legacy-current": ++ context.currentEngine.Spec.DataEngine = longhorn.DataEngineTypeV1 ++ case "legacy-target": ++ context.rebuildObj = &longhorn.Engine{Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: longhorn.DataEngineTypeV1}}} ++ case "missing-current": ++ context.currentEngine = nil ++ case "missing-target": ++ context.rebuildObj = nil ++ case "typed-nil-target": ++ context.rebuildObj = (*longhorn.EngineFrontend)(nil) ++ } ++ err := controller.addReplicaForRebuild(context, false, nil, 30, 60) ++ if scenario == "compatible" { ++ if err != nil || client.adds != 1 { ++ t.Fatalf("compatible V2 rebuild changed: %v", err) ++ } ++ } else if err == nil || client.adds != 0 { ++ t.Fatal("unresolved rebuild binding reached ReplicaAdd") ++ } ++ }) ++ } ++} ++ ++func TestV2OnlyRebuildFailsBeforeClientCreation(t *testing.T) { ++ for _, engineType := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll, "unknown"} { ++ controller := &EngineController{ds: newRuntimeCapabilityTestStore(types.DataEngineModeV2Only), baseController: &baseController{logger: logrus.NewEntry(logrus.New())}} ++ engine := &longhorn.Engine{Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: engineType}}} ++ if err := controller.startRebuilding(engine, "replica", "address"); err == nil || !strings.Contains(err.Error(), "v2-only capability cannot activate a legacy engine") { ++ t.Fatal("legacy rebuild did not stop at the activation guard") ++ } ++ } ++} +diff --git a/controller/instance_handler.go b/controller/instance_handler.go +index 7a5727e07759022d0cde8234953fcdb9b2bb86f6..34763639e007a5a6378c31ad28acb826ffb09465 100644 +--- a/controller/instance_handler.go ++++ b/controller/instance_handler.go +@@ -584,6 +584,12 @@ func (h *InstanceHandler) createInstance(instanceName string, dataEngine longhor + if !types.ErrorIsNotFound(err) && !isStoppedV2Engine { + return errors.Wrapf(err, "Failed to get instance process %v", instanceName) + } ++ if h.ds == nil { ++ return fmt.Errorf("instance creation requires a datastore capability") ++ } ++ if !h.ds.DataEngineMode().AllowsDataEngine(dataEngine) { ++ return fmt.Errorf("data engine %q is outside the %q installation capability", dataEngine, h.ds.DataEngineMode()) ++ } + + logrus.Infof("Creating instance %v", instanceName) + if _, err := h.instanceManagerHandler.CreateInstance(obj); err != nil { +diff --git a/controller/instance_handler_test.go b/controller/instance_handler_test.go +index ff9986adf99c37c25bec42359ffa3c69f7345129..08ef907139cd51117119d084e524c4d8c6cc9b2f 100644 +--- a/controller/instance_handler_test.go ++++ b/controller/instance_handler_test.go +@@ -666,7 +666,7 @@ func (s *TestSuite) TestReconcileInstanceState(c *C) { + } + + func newTestInstanceHandler(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, informerFactories *util.InformerFactories) *InstanceHandler { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + fakeRecorder := record.NewFakeRecorder(100) + return NewInstanceHandler(ds, &MockInstanceManagerHandler{}, fakeRecorder) + } +@@ -710,6 +710,7 @@ func (s *TestSuite) TestReconcileInstanceStateClearsStaleV2InstanceManager(c *C) + func (s *TestSuite) TestCreateInstanceRecordsFailedStartingEvent(c *C) { + fakeRecorder := record.NewFakeRecorder(5) + h := &InstanceHandler{ ++ ds: newRuntimeCapabilityTestStore(types.DataEngineModeV1Capable), + instanceManagerHandler: &failingCreateInstanceManagerHandler{ + createErr: fmt.Errorf("engine frontend create failed"), + }, +@@ -861,6 +862,7 @@ func (s *TestSuite) TestCreateInstanceReapsStaleStoppedV1Record(c *C) { + getErr: tc.getErr, + } + h := &InstanceHandler{ ++ ds: newRuntimeCapabilityTestStore(types.DataEngineModeV1Capable), + instanceManagerHandler: stub, + eventRecorder: record.NewFakeRecorder(10), + } +@@ -929,6 +931,7 @@ func (s *TestSuite) TestCreateInstanceRecoversFromStaleStoppedV1Record(c *C) { + instance: &longhorn.InstanceProcess{Status: longhorn.InstanceProcessStatus{State: longhorn.InstanceStateStopped}}, + } + h := &InstanceHandler{ ++ ds: newRuntimeCapabilityTestStore(types.DataEngineModeV1Capable), + instanceManagerHandler: stub, + eventRecorder: record.NewFakeRecorder(10), + } +diff --git a/controller/instance_manager_controller.go b/controller/instance_manager_controller.go +index 86ac3ad3a172c2ee9a65764961b114eaa91702f4..8efb876d3625ff7f4c10ed597182a5a593489c9b 100644 +--- a/controller/instance_manager_controller.go ++++ b/controller/instance_manager_controller.go +@@ -1918,6 +1918,9 @@ func (imc *InstanceManagerController) cleanupInstanceManagerPod(imName string) e + } + + func (imc *InstanceManagerController) createInstanceManagerPod(im *longhorn.InstanceManager) error { ++ if !imc.ds.DataEngineMode().AllowsDataEngine(im.Spec.DataEngine) { ++ return fmt.Errorf("data engine %q is outside the %q installation capability", im.Spec.DataEngine, imc.ds.DataEngineMode()) ++ } + log := getLoggerForInstanceManager(imc.logger, im) + + tolerations, err := imc.ds.GetSettingTaintToleration() +diff --git a/controller/instance_manager_controller_test.go b/controller/instance_manager_controller_test.go +index 9b5344086cb6003aa5a4df9e1035c77c20aeef07..076253f550e1fd7853d3018127e7ed0b44b969ae 100644 +--- a/controller/instance_manager_controller_test.go ++++ b/controller/instance_manager_controller_test.go +@@ -182,7 +182,7 @@ func fakeInstanceManagerVersionUpdater(im *longhorn.InstanceManager) error { + + func newTestInstanceManagerController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*InstanceManagerController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + +diff --git a/controller/instance_manager_live_upgrade_test.go b/controller/instance_manager_live_upgrade_test.go +index e2bf9fcaa833419230bd43bd6814114191d6e42b..d46b09f2a330cdc21d25b2fdbefb81c2d151c756 100644 +--- a/controller/instance_manager_live_upgrade_test.go ++++ b/controller/instance_manager_live_upgrade_test.go +@@ -47,7 +47,7 @@ func newInstanceManagerUpgrade(name, nodeID, targetImage string, state longhorn. + + func newTestInstanceManagerUpgradeController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*InstanceManagerUpgradeController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + imuc, err := NewInstanceManagerUpgradeController(logrus.StandardLogger(), ds, scheme.Scheme, kubeClient, TestNamespace, controllerID, util.NewAtomicCounter()) + if err != nil { +@@ -63,7 +63,7 @@ func newTestInstanceManagerUpgradeController(lhClient *lhfake.Clientset, kubeCli + + func newTestInstanceManagerUpgradeControlController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*InstanceManagerUpgradeControlController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + imuc, err := NewInstanceManagerUpgradeControlController(logrus.StandardLogger(), ds, scheme.Scheme, kubeClient, TestNamespace, controllerID) + if err != nil { +@@ -86,7 +86,7 @@ func (s *TestSuite) TestGetNodeV2InstanceManager(c *C) { + + imIndexer := informerFactories.LhInformerFactory.Longhorn().V1beta2().InstanceManagers().Informer().GetIndexer() + pIndexer := informerFactories.KubeInformerFactory.Core().V1().Pods().Informer().GetIndexer() +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + nodeIM := newInstanceManager("im-node", longhorn.InstanceManagerStateRunning, TestNode1, TestNode1, TestIP1, nil, nil, nil, longhorn.DataEngineTypeV2, TestInstanceManagerImage, false) + otherNodeIM := newInstanceManager("im-other-node", longhorn.InstanceManagerStateRunning, TestNode2, TestNode2, TestIP2, nil, nil, nil, longhorn.DataEngineTypeV2, TestInstanceManagerImage, false) +@@ -161,7 +161,7 @@ func (s *TestSuite) TestGetNodeV2InstanceManagerPrefersRunningSourceIM(c *C) { + informerFactories := util.NewInformerFactories(TestNamespace, kubeClient, lhClient, 30*time.Second) + + pIndexer := informerFactories.KubeInformerFactory.Core().V1().Pods().Informer().GetIndexer() +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + sourceIM := newInstanceManager("im-source", longhorn.InstanceManagerStateRunning, TestNode1, TestNode1, TestIP1, nil, nil, nil, longhorn.DataEngineTypeV2, TestInstanceManagerImage, false) + newDefaultIM := newInstanceManager("im-new-default", longhorn.InstanceManagerStateUpgrading, TestNode1, TestNode1, "", nil, nil, nil, longhorn.DataEngineTypeV2, TestExtraInstanceManagerImage, false) +diff --git a/controller/kubernetes_pod_controller_test.go b/controller/kubernetes_pod_controller_test.go +index 4cd170dcf9f5af532d1156d7ee34ffa2632e4157..497f5e9497984b34347bb69a33fbef2d09f7718d 100644 +--- a/controller/kubernetes_pod_controller_test.go ++++ b/controller/kubernetes_pod_controller_test.go +@@ -36,7 +36,7 @@ import ( + func newTestKubernetesPodController( + lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories) (*KubernetesPodController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + kc, err := NewKubernetesPodController(logrus.StandardLogger(), ds, scheme.Scheme, kubeClient) + if err != nil { +diff --git a/controller/kubernetes_pv_controller_test.go b/controller/kubernetes_pv_controller_test.go +index 66eff4a9cb04f9487bf6539b298c848c863ecaca..d186015b4de2f8c26a156d0a9fc09531237850a5 100644 +--- a/controller/kubernetes_pv_controller_test.go ++++ b/controller/kubernetes_pv_controller_test.go +@@ -169,7 +169,7 @@ func newPodWithPVC(podName string) *corev1.Pod { + } + + func newTestKubernetesPVController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, informerFactories *util.InformerFactories) (*KubernetesPVController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + kc, err := NewKubernetesPVController(logger, ds, scheme.Scheme, kubeClient) +diff --git a/controller/monitor/disk_monitor_test.go b/controller/monitor/disk_monitor_test.go +index b56ab5229b543cab78335b97cb0b7b5eb2ec579d..f54355d43402bf0d11593dc4f47c15f1da8ed0e2 100644 +--- a/controller/monitor/disk_monitor_test.go ++++ b/controller/monitor/disk_monitor_test.go +@@ -15,6 +15,7 @@ import ( + spdkdisk "github.com/longhorn/longhorn-spdk-engine/pkg/spdk" + + "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" + "github.com/longhorn/longhorn-manager/util" + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" +@@ -53,7 +54,7 @@ func newTestDiskMonitor(t *testing.T, getDiskConfig GetDiskConfigHandler, genera + lhClient := lhfake.NewSimpleClientset() // nolint: staticcheck + extensionsClient := apiextensionsfake.NewSimpleClientset() + informerFactories := util.NewInformerFactories(testNamespace, kubeClient, lhClient, 0) +- ds := datastore.NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + m, err := NewFakeDiskMonitor(logrus.StandardLogger(), ds, testNodeName, func(string) {}) + require.NoError(t, err) +diff --git a/controller/monitor/environment_check_monitor.go b/controller/monitor/environment_check_monitor.go +index 328abd12fe9de49e1105694b3c4ace76aba1750c..fa09ab3279c943bbeb4057aecfd0939a4eb19140 100644 +--- a/controller/monitor/environment_check_monitor.go ++++ b/controller/monitor/environment_check_monitor.go +@@ -62,6 +62,12 @@ type CollectedEnvironmentCheckInfo struct { + conditions []longhorn.Condition + } + ++type environmentPackageExecutor interface { ++ Execute([]string, string, []string, time.Duration) (string, error) ++} ++ ++type environmentPackageExecutorFactory func(string, string, []lhtypes.Namespace) (environmentPackageExecutor, error) ++ + func NewEnvironmentCheckMonitor(logger logrus.FieldLogger, ds *datastore.DataStore, nodeName string, syncCallback func(key string)) (*EnvironmentCheckMonitor, error) { + ctx, quit := context.WithCancel(context.Background()) + +@@ -163,11 +169,7 @@ func (m *EnvironmentCheckMonitor) environmentCheck(kubeNode *corev1.Node) *Colle + m.syncMultipathd(namespaces, collectedData) + m.syncNFSClientVersion(kubeNode, collectedData) + +- isV2DataEngine, err := m.ds.GetSettingAsBool(types.SettingNameV2DataEngine) +- if err != nil { +- m.logger.WithError(err).Debug("Failed to fetch v2-data-engine setting") +- isV2DataEngine = false +- } ++ isV2DataEngine := m.requiresV2EnvironmentChecks() + + m.checkKernelModulesLoaded(kubeNode, isV2DataEngine, collectedData) + +@@ -180,49 +182,12 @@ func (m *EnvironmentCheckMonitor) environmentCheck(kubeNode *corev1.Node) *Colle + + func (m *EnvironmentCheckMonitor) syncPackagesInstalled(kubeNode *corev1.Node, namespaces []lhtypes.Namespace, collectedData *CollectedEnvironmentCheckInfo) { + osImage := strings.ToLower(kubeNode.Status.NodeInfo.OSImage) +- +- packageProbeExecutables := make(map[string]string) +- +- switch { +- case strings.Contains(osImage, "talos"): ++ if strings.Contains(osImage, "talos") { + m.syncPackagesInstalledTalosLinux(namespaces, collectedData) + return +- case strings.Contains(osImage, "ubuntu"): +- fallthrough +- case strings.Contains(osImage, "debian"): +- packageProbeExecutables["nfs-common"] = "mount.nfs" +- packageProbeExecutables["open-iscsi"] = "iscsiadm" +- packageProbeExecutables["cryptsetup"] = "cryptsetup" +- packageProbeExecutables["dmsetup"] = "dmsetup" +- case strings.Contains(osImage, "centos"): +- fallthrough +- case strings.Contains(osImage, "fedora"): +- fallthrough +- case strings.Contains(osImage, "red hat"): +- fallthrough +- case strings.Contains(osImage, "rocky"): +- fallthrough +- case strings.Contains(osImage, "ol"): +- packageProbeExecutables["nfs-utils"] = "mount.nfs" +- packageProbeExecutables["iscsi-initiator-utils"] = "iscsiadm" +- packageProbeExecutables["cryptsetup"] = "cryptsetup" +- packageProbeExecutables["device-mapper"] = "dmsetup" +- case strings.Contains(osImage, "suse"): +- packageProbeExecutables["nfs-client"] = "mount.nfs" +- packageProbeExecutables["open-iscsi"] = "iscsiadm" +- packageProbeExecutables["cryptsetup"] = "cryptsetup" +- packageProbeExecutables["device-mapper"] = "dmsetup" +- case strings.Contains(osImage, "arch"): +- packageProbeExecutables["nfs-utils"] = "mount.nfs" +- packageProbeExecutables["open-iscsi"] = "iscsiadm" +- packageProbeExecutables["cryptsetup"] = "cryptsetup" +- packageProbeExecutables["device-mapper"] = "dmsetup" +- case strings.Contains(osImage, "gentoo"): +- packageProbeExecutables["net-fs/nfs-utils"] = "mount.nfs" +- packageProbeExecutables["sys-block/open-iscsi"] = "iscsiadm" +- packageProbeExecutables["sys-fs/cryptsetup"] = "cryptsetup" +- packageProbeExecutables["sys-fs/lvm2"] = "dmsetup" +- default: ++ } ++ packageProbeExecutables := requiredPackageProbeExecutables(osImage, m.ds.DataEngineMode()) ++ if packageProbeExecutables == nil { + // Skip environment check condition when + // 1.The OS is unknown to Longhorn + // 2.The OS that already has all packages installed such as Harvester +@@ -249,6 +214,12 @@ func (m *EnvironmentCheckMonitor) syncPackagesInstalled(kubeNode *corev1.Node, n + } + + func (m *EnvironmentCheckMonitor) syncPackagesInstalledTalosLinux(namespaces []lhtypes.Namespace, collectedData *CollectedEnvironmentCheckInfo) { ++ m.syncPackagesInstalledTalosLinuxWithExecutor(namespaces, collectedData, func(process, directory string, namespaces []lhtypes.Namespace) (environmentPackageExecutor, error) { ++ return lhns.NewNamespaceExecutor(process, directory, namespaces) ++ }) ++} ++ ++func (m *EnvironmentCheckMonitor) syncPackagesInstalledTalosLinuxWithExecutor(namespaces []lhtypes.Namespace, collectedData *CollectedEnvironmentCheckInfo, newExecutor environmentPackageExecutorFactory) { + type validateCommand struct { + binary string + args []string +@@ -259,7 +230,7 @@ func (m *EnvironmentCheckMonitor) syncPackagesInstalledTalosLinux(namespaces []l + // Helper function to validate packages within a namespace and update node + // status if there is an error. + validatePackages := func(process string, binaryToValidateCommand map[string]validateCommand) (ok bool) { +- nsexec, err := lhns.NewNamespaceExecutor(process, lhtypes.HostProcDirectory, namespaces) ++ nsexec, err := newExecutor(process, lhtypes.HostProcDirectory, namespaces) + if err != nil { + collectedData.conditions = types.SetCondition( + collectedData.conditions, longhorn.NodeConditionTypeRequiredPackages, longhorn.ConditionStatusFalse, +@@ -292,10 +263,12 @@ func (m *EnvironmentCheckMonitor) syncPackagesInstalledTalosLinux(namespaces []l + "iscsiadm": {binary: "iscsiadm", args: []string{"--version"}}, + } + +- // Check each set of packagesl return immediately if there is an error. ++ // Stop before probing another namespace if its executor is unavailable. + if !validatePackages(lhtypes.ProcessNone, hostPackageToValidateCmd) || +- !validatePackages(lhns.GetDefaultProcessName(), kubeletPackageToValidateCmd) || +- !validatePackages(iscsiutil.ISCSIdProcess, iscsiPackageToValidateCmd) { ++ !validatePackages(lhns.GetDefaultProcessName(), kubeletPackageToValidateCmd) { ++ return ++ } ++ if m.ds.DataEngineMode().RequiresISCSI() && !validatePackages(iscsiutil.ISCSIdProcess, iscsiPackageToValidateCmd) { + return + } + +diff --git a/controller/monitor/environment_data_engine_mode.go b/controller/monitor/environment_data_engine_mode.go +new file mode 100644 +index 0000000000000000000000000000000000000000..2a5d6c557f813a91ac92ee237eecb19e780d805f +--- /dev/null ++++ b/controller/monitor/environment_data_engine_mode.go +@@ -0,0 +1,58 @@ ++package monitor ++ ++import ( ++ "strings" ++ ++ "github.com/longhorn/longhorn-manager/types" ++) ++ ++func (m *EnvironmentCheckMonitor) requiresV2EnvironmentChecks() bool { ++ enabled, err := m.ds.GetSettingAsBool(types.SettingNameV2DataEngine) ++ if err != nil { ++ m.logger.WithError(err).Debug("Failed to fetch v2-data-engine setting") ++ return m.ds.DataEngineMode() == types.DataEngineModeV2Only ++ } ++ return m.ds.DataEngineMode() == types.DataEngineModeV2Only || enabled ++} ++ ++func requiredPackageProbeExecutables(osImage string, mode types.DataEngineMode) map[string]string { ++ osImage = strings.ToLower(osImage) ++ probes := make(map[string]string) ++ switch { ++ case strings.Contains(osImage, "ubuntu"), strings.Contains(osImage, "debian"): ++ probes["nfs-common"] = "mount.nfs" ++ probes["open-iscsi"] = "iscsiadm" ++ probes["cryptsetup"] = "cryptsetup" ++ probes["dmsetup"] = "dmsetup" ++ case strings.Contains(osImage, "centos"), strings.Contains(osImage, "fedora"), strings.Contains(osImage, "red hat"), strings.Contains(osImage, "rocky"), strings.Contains(osImage, "ol"): ++ probes["nfs-utils"] = "mount.nfs" ++ probes["iscsi-initiator-utils"] = "iscsiadm" ++ probes["cryptsetup"] = "cryptsetup" ++ probes["device-mapper"] = "dmsetup" ++ case strings.Contains(osImage, "suse"): ++ probes["nfs-client"] = "mount.nfs" ++ probes["open-iscsi"] = "iscsiadm" ++ probes["cryptsetup"] = "cryptsetup" ++ probes["device-mapper"] = "dmsetup" ++ case strings.Contains(osImage, "arch"): ++ probes["nfs-utils"] = "mount.nfs" ++ probes["open-iscsi"] = "iscsiadm" ++ probes["cryptsetup"] = "cryptsetup" ++ probes["device-mapper"] = "dmsetup" ++ case strings.Contains(osImage, "gentoo"): ++ probes["net-fs/nfs-utils"] = "mount.nfs" ++ probes["sys-block/open-iscsi"] = "iscsiadm" ++ probes["sys-fs/cryptsetup"] = "cryptsetup" ++ probes["sys-fs/lvm2"] = "dmsetup" ++ default: ++ return nil ++ } ++ if !mode.RequiresISCSI() { ++ for name, binary := range probes { ++ if binary == "iscsiadm" { ++ delete(probes, name) ++ } ++ } ++ } ++ return probes ++} +diff --git a/controller/monitor/environment_data_engine_mode_test.go b/controller/monitor/environment_data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..ae2db59bcf9e6056dfc549778a983e3a580b82e7 +--- /dev/null ++++ b/controller/monitor/environment_data_engine_mode_test.go +@@ -0,0 +1,122 @@ ++package monitor ++ ++import ( ++ "fmt" ++ "strings" ++ "testing" ++ "time" ++ ++ "github.com/sirupsen/logrus" ++ ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ ++ lhns "github.com/longhorn/go-common-libs/ns" ++ lhtypes "github.com/longhorn/go-common-libs/types" ++ iscsiutil "github.com/longhorn/go-iscsi-helper/util" ++ ++ "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" ++ "github.com/longhorn/longhorn-manager/util" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func newModeEnvironmentMonitor(t *testing.T, mode types.DataEngineMode, cachedV2 string) *EnvironmentCheckMonitor { ++ t.Helper() ++ lh, kube := lhfake.NewClientset(), kubefake.NewClientset() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := datastore.NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, mode) ++ for name, value := range map[types.SettingName]string{types.SettingNameV1DataEngine: "false", types.SettingNameV2DataEngine: cachedV2} { ++ if err := factories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer().Add(&longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: "longhorn-system"}, Value: value}); err != nil { ++ t.Fatal(err) ++ } ++ } ++ return &EnvironmentCheckMonitor{baseMonitor: &baseMonitor{logger: logrus.New(), ds: store}} ++} ++ ++func TestV2OnlyLinuxPackageProfilesRetainCommonRequirements(t *testing.T) { ++ for _, osImage := range []string{"Ubuntu", "Debian", "CentOS", "Fedora", "Red Hat", "Rocky", "OL", "openSUSE", "Arch", "Gentoo"} { ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ probes := requiredPackageProbeExecutables(osImage, mode) ++ binaries := map[string]bool{} ++ for _, binary := range probes { ++ binaries[binary] = true ++ } ++ if !binaries["mount.nfs"] || !binaries["cryptsetup"] || !binaries["dmsetup"] || binaries["iscsiadm"] != mode.RequiresISCSI() { ++ t.Fatalf("OS=%s mode=%s changed common requirements or kept the wrong iSCSI probe: %v", osImage, mode, probes) ++ } ++ } ++ } ++ if probes := requiredPackageProbeExecutables("unknown platform", types.DataEngineModeV2Only); probes != nil { ++ t.Fatal("unknown OS behavior changed") ++ } ++} ++ ++type recordingPackageExecutor struct { ++ run func(string, []string) (string, error) ++} ++ ++func (e recordingPackageExecutor) Execute(_ []string, binary string, args []string, _ time.Duration) (string, error) { ++ return e.run(binary, args) ++} ++ ++func TestTalosModeOmitsTheEntireISCSIExecutor(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ monitor := newModeEnvironmentMonitor(t, mode, "false") ++ processes, binaries := map[string]bool{}, map[string]bool{} ++ factory := func(process, directory string, namespaces []lhtypes.Namespace) (environmentPackageExecutor, error) { ++ if directory != lhtypes.HostProcDirectory || len(namespaces) != 2 { ++ t.Fatal("namespace executor binding changed") ++ } ++ processes[process] = true ++ if process == iscsiutil.ISCSIdProcess { ++ return nil, fmt.Errorf("iscsid is absent") ++ } ++ return recordingPackageExecutor{run: func(binary string, _ []string) (string, error) { binaries[binary] = true; return "", nil }}, nil ++ } ++ collected := &CollectedEnvironmentCheckInfo{} ++ monitor.syncPackagesInstalledTalosLinuxWithExecutor([]lhtypes.Namespace{lhtypes.NamespaceMnt, lhtypes.NamespaceNet}, collected, factory) ++ condition := types.GetCondition(collected.conditions, longhorn.NodeConditionTypeRequiredPackages) ++ if processes[iscsiutil.ISCSIdProcess] != mode.RequiresISCSI() || !processes[lhtypes.ProcessNone] || !processes[lhns.GetDefaultProcessName()] || !binaries["cryptsetup"] || !binaries["dmsetup"] || !binaries["dpkg"] { ++ t.Fatalf("mode=%s lost common probes or entered iscsid: processes=%v binaries=%v", mode, processes, binaries) ++ } ++ want := longhorn.ConditionStatusTrue ++ if mode.RequiresISCSI() { ++ want = longhorn.ConditionStatusFalse ++ } ++ if condition.Status != want { ++ t.Fatalf("mode=%s lost the required-package condition: %+v", mode, condition) ++ } ++ } ++} ++ ++func TestTalosV2OnlyStillReportsCommonPackageFailures(t *testing.T) { ++ monitor := newModeEnvironmentMonitor(t, types.DataEngineModeV2Only, "false") ++ factory := func(_, _ string, _ []lhtypes.Namespace) (environmentPackageExecutor, error) { ++ return recordingPackageExecutor{run: func(binary string, _ []string) (string, error) { ++ if binary == "cryptsetup" { ++ return "", fmt.Errorf("missing encryption tool") ++ } ++ return "", nil ++ }}, nil ++ } ++ collected := &CollectedEnvironmentCheckInfo{} ++ monitor.syncPackagesInstalledTalosLinuxWithExecutor([]lhtypes.Namespace{lhtypes.NamespaceMnt, lhtypes.NamespaceNet}, collected, factory) ++ condition := types.GetCondition(collected.conditions, longhorn.NodeConditionTypeRequiredPackages) ++ if condition.Status != longhorn.ConditionStatusFalse || !strings.Contains(condition.Message, "cryptsetup") { ++ t.Fatalf("V2-only suppressed encryption prerequisites: %+v", condition) ++ } ++} ++ ++func TestV2OnlyRequiresV2HostChecksDespiteStaleOrUnreadableSettings(t *testing.T) { ++ for _, value := range []string{"true", "false", "invalid"} { ++ for _, mode := range []types.DataEngineMode{types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ monitor := newModeEnvironmentMonitor(t, mode, value) ++ if got, want := monitor.requiresV2EnvironmentChecks(), mode == types.DataEngineModeV2Only || value == "true"; got != want { ++ t.Fatalf("mode=%s cachedV2=%s: V2 host checks=%t want=%t", mode, value, got, want) ++ } ++ } ++ } ++} +diff --git a/controller/node_controller.go b/controller/node_controller.go +index e07a79a569a8d2d33e02d80f5b0df4377833faa4..bf9a86e3e505cd688457d2f6d226927506e8d522 100644 +--- a/controller/node_controller.go ++++ b/controller/node_controller.go +@@ -1084,7 +1084,11 @@ func (nc *NodeController) getImTypeDataEngines(node *longhorn.Node) map[longhorn + } + + for _, setting := range []types.SettingName{types.SettingNameV1DataEngine, types.SettingNameV2DataEngine} { +- enabled, err := nc.ds.GetSettingAsBool(setting) ++ dataEngine := longhorn.DataEngineTypeV1 ++ if setting == types.SettingNameV2DataEngine { ++ dataEngine = longhorn.DataEngineTypeV2 ++ } ++ enabled, err := nc.ds.IsDataEngineEnabled(dataEngine) + if err != nil { + log.WithError(err).Warnf("Failed to get %v setting", setting) + continue +@@ -1375,6 +1379,9 @@ func shouldPreserveV2InstanceManagerDuringOfflineUpgrade(im *longhorn.InstanceMa + } + + func (nc *NodeController) createInstanceManager(node *longhorn.Node, imName, imImage string, imType longhorn.InstanceManagerType, dataEngine longhorn.DataEngineType) (*longhorn.InstanceManager, error) { ++ if !nc.ds.DataEngineMode().AllowsDataEngine(dataEngine) { ++ return nil, fmt.Errorf("data engine %q is outside the %q installation capability", dataEngine, nc.ds.DataEngineMode()) ++ } + instanceManager := &longhorn.InstanceManager{ + ObjectMeta: metav1.ObjectMeta{ + Name: imName, +diff --git a/controller/node_controller_test.go b/controller/node_controller_test.go +index 4c5cc4872c24e90afb89a406e35c2fc66359f8cc..680dfcdcc5b7b360b5d572218ba34d53d2a94ae2 100644 +--- a/controller/node_controller_test.go ++++ b/controller/node_controller_test.go +@@ -2606,7 +2606,7 @@ func (s *NodeControllerSuite) initTest(c *C, fixture *NodeControllerFixture) { + + func newTestNodeController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, eventRecorder *record.FakeRecorder, controllerID string) (*NodeController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + nc, err := NewNodeController(logger, ds, scheme.Scheme, kubeClient, TestNamespace, controllerID, TestInstanceManagerImage) +diff --git a/controller/orphan_controller_test.go b/controller/orphan_controller_test.go +index 66c2c8b5910d78d033f58cbd4de420abb7a540e5..f65e99146c7b7ff3eed5d6e779123b7afabb3bdf 100644 +--- a/controller/orphan_controller_test.go ++++ b/controller/orphan_controller_test.go +@@ -21,6 +21,7 @@ import ( + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++ longhorntypes "github.com/longhorn/longhorn-manager/types" + ) + + // OrphanControllerSuite holds the fake clients and indexers needed for testing +@@ -58,7 +59,7 @@ func (s *OrphanControllerSuite) SetUpTest(c *C) { + func newTestOrphanController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*OrphanController, error) { + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, longhorntypes.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + oc, err := NewOrphanController(logger, ds, scheme.Scheme, kubeClient, controllerID, TestNamespace) +diff --git a/controller/setting_controller_test.go b/controller/setting_controller_test.go +index 5f2311cc842c5a0d83911c65f6d2582de4e6c18a..07db7c7e6c5cc7f09324d4c3cc49a3c4095580e2 100644 +--- a/controller/setting_controller_test.go ++++ b/controller/setting_controller_test.go +@@ -204,7 +204,7 @@ func TestUpdateEngineImagePodLivenessProbes(t *testing.T) { + settingIndexer := informerFactories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer() + daemonSetIndexer := informerFactories.KubeNamespaceFilteredInformerFactory.Apps().V1().DaemonSets().Informer().GetIndexer() + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionClient, informerFactories, types.DataEngineModeV1Capable) + sc := &SettingController{ + baseController: newBaseController("longhorn-setting", logrus.StandardLogger()), + ds: ds, +@@ -274,7 +274,7 @@ func TestUpdateEngineImagePodLivenessProbesUsesDefaultValuesOnSettingError(t *te + informerFactories := util.NewInformerFactories(TestNamespace, kubeClient, lhClient, controller.NoResyncPeriodFunc()) + daemonSetIndexer := informerFactories.KubeNamespaceFilteredInformerFactory.Apps().V1().DaemonSets().Informer().GetIndexer() + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionClient, informerFactories, types.DataEngineModeV1Capable) + sc := &SettingController{ + baseController: newBaseController("longhorn-setting", logrus.StandardLogger()), + ds: ds, +diff --git a/controller/shard_group_controller_test.go b/controller/shard_group_controller_test.go +index 412de61743c3e52198b7e8765152bba88ba97151..16c0738646b77518bcba11cb58305ece40292c47 100644 +--- a/controller/shard_group_controller_test.go ++++ b/controller/shard_group_controller_test.go +@@ -76,7 +76,7 @@ func (s *ShardGroupControllerSuite) SetUpTest(c *C) { + func newTestShardGroupController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*ShardGroupController, error) { + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + c, err := NewShardGroupController(logger, ds, scheme.Scheme, kubeClient, controllerID, TestNamespace) +diff --git a/controller/snapshot_controller_test.go b/controller/snapshot_controller_test.go +index 5d6bb0cb21c55b5a6b9d193ad5deaa6824b5fe40..12335e00e52e997bf7e0325f18ec3641cd6ffe25 100644 +--- a/controller/snapshot_controller_test.go ++++ b/controller/snapshot_controller_test.go +@@ -67,7 +67,7 @@ func newSnapshotControllerFixture(c *C) *snapshotControllerFixture { + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint: staticcheck + + informerFactories := util.NewInformerFactories(TestNamespace, kubeClient, lhClient, 0) +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + sc, err := NewSnapshotController( +diff --git a/controller/snapshot_group_controller_test.go b/controller/snapshot_group_controller_test.go +index 901307f7a094aefb942a466bd0d75f017ed56ebf..cf118cf1556228686e5a7069fa3130132e65654f 100644 +--- a/controller/snapshot_group_controller_test.go ++++ b/controller/snapshot_group_controller_test.go +@@ -62,7 +62,7 @@ func newSnapshotGroupControllerFixture(c *C) *snapshotGroupControllerFixture { + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint: staticcheck + + informerFactories := util.NewInformerFactories(TestNamespace, kubeClient, lhClient, 0) +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + ctrl, err := NewSnapshotGroupController(logrus.StandardLogger(), ds, scheme.Scheme, kubeClient, TestNamespace, TestOwnerID1) + c.Assert(err, IsNil) +diff --git a/controller/support_bundle_controller_test.go b/controller/support_bundle_controller_test.go +index 5a319dce5fc093dc0414d11a5dd7d548e2c8f1f0..438dd8b1b24f7a00b5acd79c161d49800209b05f 100644 +--- a/controller/support_bundle_controller_test.go ++++ b/controller/support_bundle_controller_test.go +@@ -230,7 +230,7 @@ func (s *TestSuite) TestReconcileSupportBundle(c *C) { + + func newFakeSupportBundleController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*SupportBundleController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + logrus.SetLevel(logrus.DebugLevel) +diff --git a/controller/system_backup_controller_test.go b/controller/system_backup_controller_test.go +index 3e3fb04f0dba7bee36d6334ae82a50a228b9fbba..dbf029455d0a2bfe1a5dfe5d9e5865fb9393cbbb 100644 +--- a/controller/system_backup_controller_test.go ++++ b/controller/system_backup_controller_test.go +@@ -430,7 +430,7 @@ func (s *TestSuite) TestReconcileSystemBackup(c *C) { + + func newFakeSystemBackupController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*SystemBackupController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + logrus.SetLevel(logrus.DebugLevel) +diff --git a/controller/system_restore_archive.go b/controller/system_restore_archive.go +new file mode 100644 +index 0000000000000000000000000000000000000000..dda0ac13e3ff6740ce92916b03f7b22302419bd2 +--- /dev/null ++++ b/controller/system_restore_archive.go +@@ -0,0 +1,202 @@ ++package controller ++ ++import ( ++ "bufio" ++ "bytes" ++ "encoding/json" ++ "fmt" ++ "io" ++ "os" ++ "path/filepath" ++ ++ "k8s.io/apimachinery/pkg/api/meta" ++ "k8s.io/apimachinery/pkg/runtime" ++ "k8s.io/apimachinery/pkg/runtime/schema" ++ "k8s.io/apimachinery/pkg/runtime/serializer" ++ ++ appsv1 "k8s.io/api/apps/v1" ++ corev1 "k8s.io/api/core/v1" ++ rbacv1 "k8s.io/api/rbac/v1" ++ storagev1 "k8s.io/api/storage/v1" ++ apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ++ utilyaml "k8s.io/apimachinery/pkg/util/yaml" ++ kubernetesscheme "k8s.io/client-go/kubernetes/scheme" ++ ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++type systemRestoreArchiveList struct { ++ directory, filename string ++ gvk schema.GroupVersionKind ++ namespaced bool ++ bind func(runtime.Object) bool ++} ++ ++func archiveList[T runtime.Object](directory, filename string, target *T, gvk schema.GroupVersionKind) systemRestoreArchiveList { ++ return systemRestoreArchiveList{directory: directory, filename: filename, gvk: gvk, namespaced: true, bind: func(obj runtime.Object) bool { ++ list, ok := obj.(T) ++ if ok { ++ *target = list ++ } ++ return ok ++ }} ++} ++ ++func clusterArchiveList[T runtime.Object](directory, filename string, target *T, gvk schema.GroupVersionKind) systemRestoreArchiveList { ++ list := archiveList(directory, filename, target, gvk) ++ list.namespaced = false ++ return list ++} ++ ++func systemRestoreArchiveLists(target *extractedResources) []systemRestoreArchiveList { ++ kube, lh, extensions := types.SystemBackupSubDirKubernetes, types.SystemBackupSubDirLonghorn, types.SystemBackupSubDirAPIExtensions ++ return []systemRestoreArchiveList{ ++ clusterArchiveList(extensions, "customresourcedefinitions.yaml", &target.customResourceDefinitionList, apiextensionsv1.SchemeGroupVersion.WithKind(types.APIExtensionsKindCustomResourceDefinitionList)), ++ // Native producers use these two historical filenames in reverse order. ++ clusterArchiveList(kube, "clusterroles.yaml", &target.clusterRoleBindingList, rbacv1.SchemeGroupVersion.WithKind(types.KubernetesKindClusterRoleBindingList)), ++ clusterArchiveList(kube, "clusterrolebindings.yaml", &target.clusterRoleList, rbacv1.SchemeGroupVersion.WithKind(types.KubernetesKindClusterRoleList)), ++ archiveList(kube, "roles.yaml", &target.roleList, rbacv1.SchemeGroupVersion.WithKind(types.KubernetesKindRoleList)), ++ archiveList(kube, "rolebindings.yaml", &target.roleBindingList, rbacv1.SchemeGroupVersion.WithKind(types.KubernetesKindRoleBindingList)), ++ archiveList(kube, "daemonsets.yaml", &target.daemonSetList, appsv1.SchemeGroupVersion.WithKind(types.KubernetesKindDaemonSetList)), ++ archiveList(kube, "deployments.yaml", &target.deploymentList, appsv1.SchemeGroupVersion.WithKind(types.KubernetesKindDeploymentList)), ++ archiveList(kube, "configmaps.yaml", &target.configMapList, corev1.SchemeGroupVersion.WithKind(types.KubernetesKindConfigMapList)), ++ clusterArchiveList(kube, "persistentvolumes.yaml", &target.persistentVolumeList, corev1.SchemeGroupVersion.WithKind(types.KubernetesKindPersistentVolumeList)), ++ archiveList(kube, "persistentvolumeclaims.yaml", &target.persistentVolumeClaimList, corev1.SchemeGroupVersion.WithKind(types.KubernetesKindPersistentVolumeClaimList)), ++ archiveList(kube, "services.yaml", &target.serviceList, corev1.SchemeGroupVersion.WithKind(types.KubernetesKindServiceList)), ++ archiveList(kube, "serviceaccounts.yaml", &target.serviceAccountList, corev1.SchemeGroupVersion.WithKind(types.KubernetesKindServiceAccountList)), ++ clusterArchiveList(kube, "storageclasses.yaml", &target.storageClassList, storagev1.SchemeGroupVersion.WithKind(types.KubernetesKindStorageClassList)), ++ archiveList(lh, "setting.yaml", &target.settingList, longhorn.SchemeGroupVersion.WithKind(types.LonghornKindSettingList)), ++ archiveList(lh, "engineimages.yaml", &target.engineImageList, longhorn.SchemeGroupVersion.WithKind(types.LonghornKindEngineImageList)), ++ archiveList(lh, "volumes.yaml", &target.volumeList, longhorn.SchemeGroupVersion.WithKind(types.LonghornKindVolumeList)), ++ archiveList(lh, "recurringjobs.yaml", &target.recurringJobList, longhorn.SchemeGroupVersion.WithKind(types.LonghornKindRecurringJobList)), ++ archiveList(lh, "backingimages.yaml", &target.backingImageList, longhorn.SchemeGroupVersion.WithKind(types.LonghornKindBackingImageList)), ++ archiveList(lh, "backuptargets.yaml", &target.backupTargetList, longhorn.SchemeGroupVersion.WithKind(types.LonghornKindBackupTargetList)), ++ } ++} ++ ++// loadV2OnlySystemRestoreArchive is deliberately separate from the legacy ++// permissive loader. No caller cache is published on a partial read. ++func loadV2OnlySystemRestoreArchive(root string) (extractedResources, error) { ++ rootInfo, err := os.Lstat(root) ++ if err != nil || !rootInfo.IsDir() || rootInfo.Mode()&os.ModeSymlink != 0 { ++ return extractedResources{}, fmt.Errorf("missing or invalid archived resource root") ++ } ++ var pending extractedResources ++ lists := systemRestoreArchiveLists(&pending) ++ directories := map[string]map[string]bool{} ++ for _, list := range lists { ++ if directories[list.directory] == nil { ++ directories[list.directory] = map[string]bool{} ++ } ++ directories[list.directory][list.filename] = true ++ } ++ for directory, expected := range directories { ++ path := filepath.Join(root, directory) ++ info, err := os.Lstat(path) ++ if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { ++ return extractedResources{}, fmt.Errorf("missing or invalid archived resource directory") ++ } ++ entries, err := os.ReadDir(path) ++ if err != nil || len(entries) != len(expected) { ++ return extractedResources{}, fmt.Errorf("incomplete archived resource file inventory") ++ } ++ for _, entry := range entries { ++ if !expected[entry.Name()] || !entry.Type().IsRegular() { ++ return extractedResources{}, fmt.Errorf("unexpected archived resource file") ++ } ++ } ++ } ++ scheme := runtime.NewScheme() ++ for _, register := range []func(*runtime.Scheme) error{kubernetesscheme.AddToScheme, apiextensionsv1.SchemeBuilder.AddToScheme, longhorn.SchemeBuilder.AddToScheme} { ++ if err := register(scheme); err != nil { ++ return extractedResources{}, fmt.Errorf("cannot initialize archived resource decoder") ++ } ++ } ++ decoder := serializer.NewCodecFactory(scheme, serializer.EnableStrict).UniversalDeserializer() ++ totalBytes, totalItems := 0, 0 ++ for _, list := range lists { ++ contents, err := readSystemRestoreManifest(filepath.Join(root, list.directory, list.filename)) ++ if err != nil { ++ return extractedResources{}, err ++ } ++ totalBytes += len(contents) ++ if totalBytes > 64<<20 { ++ return extractedResources{}, fmt.Errorf("archived resource byte budget exceeded") ++ } ++ reader := utilyaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(contents))) ++ document, err := reader.Read() ++ if err != nil || len(bytes.TrimSpace(document)) == 0 { ++ return extractedResources{}, fmt.Errorf("invalid archived %s document", list.gvk.Kind) ++ } ++ trailing, err := reader.Read() ++ if err != io.EOF || len(bytes.TrimSpace(trailing)) != 0 { ++ return extractedResources{}, fmt.Errorf("multiple archived %s documents", list.gvk.Kind) ++ } ++ object, gvk, err := decoder.Decode(document, nil, nil) ++ if err != nil || gvk == nil || *gvk != list.gvk || object == nil { ++ return extractedResources{}, fmt.Errorf("invalid archived %s", list.gvk.Kind) ++ } ++ jsonDocument, err := utilyaml.ToJSON(document) ++ if err != nil { ++ return extractedResources{}, fmt.Errorf("invalid archived %s envelope", list.gvk.Kind) ++ } ++ var envelope map[string]json.RawMessage ++ if err := json.Unmarshal(jsonDocument, &envelope); err != nil || envelope["items"] == nil { ++ return extractedResources{}, fmt.Errorf("missing archived %s items", list.gvk.Kind) ++ } ++ metadata, err := meta.ListAccessor(object) ++ if err != nil || metadata.GetContinue() != "" || (metadata.GetRemainingItemCount() != nil && *metadata.GetRemainingItemCount() != 0) { ++ return extractedResources{}, fmt.Errorf("incomplete archived %s", list.gvk.Kind) ++ } ++ items, err := meta.ExtractList(object) ++ if err != nil || len(items) > 10000 { ++ return extractedResources{}, fmt.Errorf("invalid archived %s item inventory", list.gvk.Kind) ++ } ++ totalItems += len(items) ++ if totalItems > 25000 { ++ return extractedResources{}, fmt.Errorf("archived resource item budget exceeded") ++ } ++ seen := map[string]bool{} ++ for _, item := range items { ++ identity, err := meta.Accessor(item) ++ if err != nil || identity.GetName() == "" { ++ return extractedResources{}, fmt.Errorf("unidentified archived %s item", list.gvk.Kind) ++ } ++ if (identity.GetNamespace() != "") != list.namespaced { ++ return extractedResources{}, fmt.Errorf("invalid archived %s item scope", list.gvk.Kind) ++ } ++ key := identity.GetNamespace() + "/" + identity.GetName() ++ if seen[key] { ++ return extractedResources{}, fmt.Errorf("duplicate archived %s item", list.gvk.Kind) ++ } ++ seen[key] = true ++ } ++ if !list.bind(object) { ++ return extractedResources{}, fmt.Errorf("unexpected archived %s concrete type", list.gvk.Kind) ++ } ++ } ++ return pending, nil ++} ++ ++func readSystemRestoreManifest(path string) ([]byte, error) { ++ const maximumBytes = 16 << 20 ++ before, err := os.Lstat(path) ++ if err != nil || !before.Mode().IsRegular() || before.Size() > maximumBytes { ++ return nil, fmt.Errorf("missing or invalid archived resource file") ++ } ++ file, err := os.Open(path) ++ if err != nil { ++ return nil, fmt.Errorf("cannot open archived resource file") ++ } ++ opened, statErr := file.Stat() ++ contents, readErr := io.ReadAll(io.LimitReader(file, maximumBytes+1)) ++ after, afterErr := file.Stat() ++ closeErr := file.Close() ++ if statErr != nil || readErr != nil || afterErr != nil || closeErr != nil || len(contents) > maximumBytes || ++ !os.SameFile(before, opened) || !os.SameFile(opened, after) || opened.Size() != after.Size() || !opened.ModTime().Equal(after.ModTime()) || int64(len(contents)) != after.Size() { ++ return nil, fmt.Errorf("archived resource file changed or could not be read completely") ++ } ++ return contents, nil ++} +diff --git a/controller/system_restore_archive_test.go b/controller/system_restore_archive_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..0fc1ce4b7395e118c3a82f2766f3508c955ac25b +--- /dev/null ++++ b/controller/system_restore_archive_test.go +@@ -0,0 +1,193 @@ ++package controller ++ ++import ( ++ "fmt" ++ "os" ++ "path/filepath" ++ "reflect" ++ "strings" ++ "testing" ++) ++ ++func writeCompleteRestoreArchive(t *testing.T) string { ++ t.Helper() ++ root := t.TempDir() ++ for _, manifest := range systemRestoreArchiveLists(&extractedResources{}) { ++ directory := filepath.Join(root, manifest.directory) ++ if err := os.MkdirAll(directory, 0700); err != nil { ++ t.Fatal(err) ++ } ++ document := fmt.Sprintf("apiVersion: %s\nkind: %s\nmetadata: {}\nitems: []\n", manifest.gvk.GroupVersion(), manifest.gvk.Kind) ++ if err := os.WriteFile(filepath.Join(directory, manifest.filename), []byte(document), 0600); err != nil { ++ t.Fatal(err) ++ } ++ } ++ return root ++} ++ ++func TestV2OnlyRestoreArchiveLoadsEveryNativeListIncludingEmpty(t *testing.T) { ++ root := writeCompleteRestoreArchive(t) ++ bindings, err := os.ReadFile(filepath.Join(root, "kubernetes", "clusterroles.yaml")) ++ if err != nil || !strings.Contains(string(bindings), "kind: ClusterRoleBindingList") { ++ t.Fatal("fixture lost the native cluster RBAC filename mapping") ++ } ++ if _, err := os.Stat(filepath.Join(root, "longhorn", "setting.yaml")); err != nil { ++ t.Fatal("fixture lost the native singular Setting filename") ++ } ++ resources, err := loadV2OnlySystemRestoreArchive(root) ++ if err != nil { ++ t.Fatal(err) ++ } ++ value := reflect.ValueOf(resources) ++ if value.NumField() != 19 { ++ t.Fatal("native restore list inventory changed") ++ } ++ for index := range value.NumField() { ++ if value.Field(index).IsNil() { ++ t.Fatalf("native list %d was treated as absent instead of empty", index) ++ } ++ } ++} ++ ++func TestV2OnlyRestoreArchiveRejectsIncompleteOrAmbiguousPayload(t *testing.T) { ++ for _, test := range []string{"missing", "extra", "symlink", "wrong-kind", "unknown-field", "duplicate-key", "multiple-documents", "missing-items", "continuation", "truncated", "duplicate-items"} { ++ t.Run(test, func(t *testing.T) { ++ root := writeCompleteRestoreArchive(t) ++ path := filepath.Join(root, "longhorn", "setting.yaml") ++ document, err := os.ReadFile(path) ++ if err != nil { ++ t.Fatal(err) ++ } ++ text := string(document) ++ switch test { ++ case "missing": ++ if err := os.Remove(path); err != nil { ++ t.Fatal(err) ++ } ++ case "extra": ++ if err := os.WriteFile(filepath.Join(root, "longhorn", "extra.yaml"), document, 0600); err != nil { ++ t.Fatal(err) ++ } ++ case "symlink": ++ target := filepath.Join(t.TempDir(), "payload") ++ if err := os.WriteFile(target, document, 0600); err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Remove(path); err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Symlink(target, path); err != nil { ++ t.Fatal(err) ++ } ++ case "wrong-kind": ++ text = strings.Replace(text, "kind: SettingList", "kind: VolumeList", 1) ++ case "unknown-field": ++ text += "unrecognized: sentinel-secret\n" ++ case "duplicate-key": ++ text += "kind: VolumeList\n" ++ case "multiple-documents": ++ text += "---\napiVersion: v1\nkind: List\nitems: []\n" ++ case "missing-items": ++ text = strings.Replace(text, "items: []\n", "", 1) ++ case "continuation": ++ text = strings.Replace(text, "metadata: {}", "metadata:\n continue: more", 1) ++ case "truncated": ++ text = text[:len(text)/2] ++ case "duplicate-items": ++ text = strings.Replace(text, "items: []", "items:\n- metadata: {name: duplicate, namespace: longhorn-system}\n value: 'false'\n- metadata: {name: duplicate, namespace: longhorn-system}\n value: 'true'", 1) ++ } ++ if test != "missing" && test != "extra" && test != "symlink" { ++ if err := os.WriteFile(path, []byte(text), 0600); err != nil { ++ t.Fatal(err) ++ } ++ } ++ resources, err := loadV2OnlySystemRestoreArchive(root) ++ if err == nil || strings.Contains(err.Error(), "sentinel-secret") { ++ t.Fatal("partial archive cleared validation or leaked raw decoder diagnostics") ++ } ++ value := reflect.ValueOf(resources) ++ for index := range value.NumField() { ++ if !value.Field(index).IsNil() { ++ t.Fatal("partial archive published a cache") ++ } ++ } ++ }) ++ } ++} ++ ++func TestV2OnlyRestoreArchiveBindsRootAndResourceScope(t *testing.T) { ++ for _, test := range []string{"root-link", "cluster-namespace", "missing-namespace"} { ++ t.Run(test, func(t *testing.T) { ++ root := writeCompleteRestoreArchive(t) ++ switch test { ++ case "root-link": ++ alias := filepath.Join(t.TempDir(), "linked") ++ if err := os.Symlink(root, alias); err != nil { ++ t.Fatal(err) ++ } ++ root = alias ++ case "cluster-namespace": ++ document := "apiVersion: storage.k8s.io/v1\nkind: StorageClassList\nitems:\n- metadata: {name: duplicate, namespace: first}\n- metadata: {name: duplicate, namespace: second}\n" ++ if err := os.WriteFile(filepath.Join(root, "kubernetes", "storageclasses.yaml"), []byte(document), 0600); err != nil { ++ t.Fatal(err) ++ } ++ case "missing-namespace": ++ document := "apiVersion: longhorn.io/v1beta2\nkind: SettingList\nitems:\n- metadata: {name: orphan}\n value: 'false'\n" ++ if err := os.WriteFile(filepath.Join(root, "longhorn", "setting.yaml"), []byte(document), 0600); err != nil { ++ t.Fatal(err) ++ } ++ } ++ if _, err := loadV2OnlySystemRestoreArchive(root); err == nil { ++ t.Fatal("archive root or resource scope was not bound") ++ } ++ }) ++ } ++} ++ ++func TestV2OnlyRestoreArchivePreservesNativeEmptyAndTenantClaims(t *testing.T) { ++ root := writeCompleteRestoreArchive(t) ++ settings := "apiVersion: longhorn.io/v1beta2\nkind: SettingList\nitems: null\n" ++ if err := os.WriteFile(filepath.Join(root, "longhorn", "setting.yaml"), []byte(settings), 0600); err != nil { ++ t.Fatal(err) ++ } ++ claims := "apiVersion: v1\nkind: PersistentVolumeClaimList\nitems:\n- metadata: {name: pending, namespace: tenant}\n spec: {}\n" ++ if err := os.WriteFile(filepath.Join(root, "kubernetes", "persistentvolumeclaims.yaml"), []byte(claims), 0600); err != nil { ++ t.Fatal(err) ++ } ++ resources, err := loadV2OnlySystemRestoreArchive(root) ++ if err != nil || resources.persistentVolumeClaimList.Items[0].Namespace != "tenant" { ++ t.Fatalf("native empty lists or unbound tenant claims were rejected: %v", err) ++ } ++} ++ ++func TestV2OnlyRestoreArchiveRejectsResourceBudgetsAndIncompleteCounts(t *testing.T) { ++ for _, test := range []string{"large-file", "too-many-items", "remaining-count"} { ++ t.Run(test, func(t *testing.T) { ++ root := writeCompleteRestoreArchive(t) ++ path := filepath.Join(root, "longhorn", "setting.yaml") ++ switch test { ++ case "large-file": ++ if err := os.Truncate(path, (16<<20)+1); err != nil { ++ t.Fatal(err) ++ } ++ case "too-many-items": ++ var document strings.Builder ++ document.WriteString("apiVersion: longhorn.io/v1beta2\nkind: SettingList\nitems:\n") ++ for index := range 10001 { ++ fmt.Fprintf(&document, "- metadata: {name: setting-%d, namespace: longhorn-system}\n value: 'false'\n", index) ++ } ++ if err := os.WriteFile(path, []byte(document.String()), 0600); err != nil { ++ t.Fatal(err) ++ } ++ case "remaining-count": ++ document := "apiVersion: longhorn.io/v1beta2\nkind: SettingList\nmetadata: {remainingItemCount: 1}\nitems: []\n" ++ if err := os.WriteFile(path, []byte(document), 0600); err != nil { ++ t.Fatal(err) ++ } ++ } ++ if _, err := loadV2OnlySystemRestoreArchive(root); err == nil { ++ t.Fatal("unbounded or partial archive cleared validation") ++ } ++ }) ++ } ++} +diff --git a/controller/system_restore_controller.go b/controller/system_restore_controller.go +index db74e845a5933e2f838787eb95965fc129bc1aac..dbb15f36a818718400a4020de8316155d3fa7463 100644 +--- a/controller/system_restore_controller.go ++++ b/controller/system_restore_controller.go +@@ -401,6 +401,9 @@ func (c *SystemRestoreController) newSystemRestoreJob(systemRestore *longhorn.Sy + cmd := []string{ + "longhorn-manager", "system-rollout", systemRestore.Name, + } ++ if c.ds.DataEngineMode() == types.DataEngineModeV2Only { ++ cmd = []string{"longhorn-manager", "system-rollout", "--data-engine-mode", string(types.DataEngineModeV2Only), systemRestore.Name} ++ } + + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ +diff --git a/controller/system_restore_controller_test.go b/controller/system_restore_controller_test.go +index 531eed46d5a1dec502d3754240845e3bbacb3038..9502b48eeb360bf7978b830250fc443dd1cb1778 100644 +--- a/controller/system_restore_controller_test.go ++++ b/controller/system_restore_controller_test.go +@@ -182,7 +182,7 @@ func (s *TestSuite) TestReconcileSystemRestore(c *C) { + + func newFakeSystemRestoreController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*SystemRestoreController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + logger := logrus.StandardLogger() + logrus.SetLevel(logrus.DebugLevel) +diff --git a/controller/system_restore_data_engine_mode_test.go b/controller/system_restore_data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..5e30419d21afb7587ff234442be5b3560476eb34 +--- /dev/null ++++ b/controller/system_restore_data_engine_mode_test.go +@@ -0,0 +1,36 @@ ++package controller ++ ++import ( ++ "reflect" ++ "testing" ++ ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ ++ "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" ++ "github.com/longhorn/longhorn-manager/util" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func TestSystemRestoreJobPreservesDataEngineCapability(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ t.Run(string(mode), func(t *testing.T) { ++ lh, kube := lhfake.NewClientset(), kubefake.NewClientset() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := datastore.NewDataStoreForGlobal("longhorn-system", lh, kube, nil, factories, mode) ++ controller := &SystemRestoreController{ds: store} ++ restore := &longhorn.SystemRestore{ObjectMeta: metav1.ObjectMeta{Name: "restore-check"}} ++ job := controller.newSystemRestoreJob(restore, "longhorn-system", "backup-manager-image", "longhorn", "", nil) ++ want := []string{"longhorn-manager", "system-rollout", "restore-check"} ++ if mode == types.DataEngineModeV2Only { ++ want = []string{"longhorn-manager", "system-rollout", "--data-engine-mode", "v2-only", "restore-check"} ++ } ++ if got := job.Spec.Template.Spec.Containers[0].Command; !reflect.DeepEqual(got, want) { ++ t.Fatalf("restore worker lost the installation capability or changed the legacy command: got=%v want=%v", got, want) ++ } ++ }) ++ } ++} +diff --git a/controller/system_restore_manager_profile.go b/controller/system_restore_manager_profile.go +new file mode 100644 +index 0000000000000000000000000000000000000000..44897e60e6ec19a8d00344ac82ee73046d1c2a8b +--- /dev/null ++++ b/controller/system_restore_manager_profile.go +@@ -0,0 +1,149 @@ ++package controller ++ ++import ( ++ "fmt" ++ "reflect" ++ "strings" ++ ++ corev1 "k8s.io/api/core/v1" ++ ++ "github.com/longhorn/longhorn-manager/types" ++) ++ ++// These exceptions are exact native launches, not an allowance for arbitrary ++// shell wrappers. The wait loop comes from chart/templates/deployment-driver.yaml; ++// CSI socket cleanup comes from csi/deployment.go. ++const archivedManagerWaitScript = `while [ $(curl -m 1 -s -o /dev/null -w "%{http_code}" http://longhorn-backend:9500/v1) != "200" ]; do echo waiting; sleep 2; done` ++ ++func isArchivedManagerContainer(container corev1.Container, image string) bool { ++ if container.Image == image || strings.Contains(container.Image, "longhorn-manager") || strings.Contains(container.Name, "longhorn-manager") || container.Name == "longhorn-global-manager" || container.Name == types.CSIPluginName || container.Name == "longhorn-driver-deployer" || strings.Contains(strings.Join(append(append([]string{}, container.Command...), container.Args...), " "), "longhorn-manager") { ++ return true ++ } ++ commands := [][]string{} ++ for _, probe := range []*corev1.Probe{container.StartupProbe, container.LivenessProbe, container.ReadinessProbe} { ++ if probe != nil && probe.Exec != nil { ++ commands = append(commands, probe.Exec.Command) ++ } ++ } ++ if container.Lifecycle != nil { ++ for _, handler := range []*corev1.LifecycleHandler{container.Lifecycle.PostStart, container.Lifecycle.PreStop} { ++ if handler != nil && handler.Exec != nil { ++ commands = append(commands, handler.Exec.Command) ++ } ++ } ++ } ++ for _, command := range commands { ++ if strings.Contains(strings.Join(command, " "), "longhorn-manager") { ++ return true ++ } ++ } ++ return false ++} ++ ++func validateV2OnlyRestorePod(pod corev1.PodSpec, kind, name, image string) error { ++ validate := func(container corev1.Container, init bool) error { ++ if !isArchivedManagerContainer(container, image) { ++ return nil ++ } ++ if container.Image != image { ++ return fmt.Errorf("archived workload contains a different manager image") ++ } ++ command := "" ++ switch { ++ case !init && kind == "DaemonSet" && name == types.LonghornManagerDaemonSetName && container.Name == "longhorn-manager": ++ command = "daemon" ++ case !init && kind == "Deployment" && name == types.LonghornGlobalManagerName && container.Name == "longhorn-global-manager": ++ command = "global" ++ case !init && kind == "DaemonSet" && name == types.CSIPluginName && container.Name == types.CSIPluginName: ++ command = "csi" ++ case !init && kind == "Deployment" && name == "longhorn-driver-deployer" && container.Name == "longhorn-driver-deployer": ++ command = "deploy-driver" ++ case init && kind == "Deployment" && name == "longhorn-driver-deployer" && container.Name == "wait-longhorn-manager" && reflect.DeepEqual(container.Command, []string{"sh", "-c", archivedManagerWaitScript}) && len(container.Args) == 0: ++ command = "wait" ++ default: ++ return fmt.Errorf("archived workload contains an unsupported manager launch") ++ } ++ if command != "wait" { ++ if err := validateV2OnlyManagerLaunch(container, command, image); err != nil { ++ return err ++ } ++ } ++ for _, probe := range []*corev1.Probe{container.StartupProbe, container.LivenessProbe, container.ReadinessProbe} { ++ if probe != nil && probe.Exec != nil { ++ return fmt.Errorf("archived manager has an unsupported executable probe") ++ } ++ } ++ if container.Lifecycle != nil { ++ if handler := container.Lifecycle.PostStart; handler != nil && handler.Exec != nil { ++ return fmt.Errorf("archived manager has an unsupported executable start hook") ++ } ++ if handler := container.Lifecycle.PreStop; handler != nil && handler.Exec != nil { ++ if command != "csi" || !reflect.DeepEqual(handler.Exec.Command, []string{"/bin/sh", "-c", "rm -f /csi//*"}) { ++ return fmt.Errorf("archived manager has an unsupported executable stop hook") ++ } ++ } ++ } ++ if command == "daemon" { ++ flags, err := parseArchivedManagerLaunch(container, command, image) ++ if err != nil || pod.ServiceAccountName == "" || flags["--service-account"] != pod.ServiceAccountName { ++ return fmt.Errorf("archived manager service account is unresolved") ++ } ++ } ++ if command == "global" { ++ for envName, field := range map[string]string{"POD_NAME": "metadata.name", "POD_NAMESPACE": "metadata.namespace"} { ++ count := 0 ++ for _, env := range container.Env { ++ if env.Name == envName { ++ count++ ++ if env.Value != "" || env.ValueFrom == nil || env.ValueFrom.FieldRef == nil || env.ValueFrom.FieldRef.FieldPath != field { ++ return fmt.Errorf("archived global manager runtime identity is unresolved") ++ } ++ } ++ } ++ if count != 1 { ++ return fmt.Errorf("archived global manager runtime identity is incomplete") ++ } ++ } ++ } ++ if command == "deploy-driver" { ++ flags, err := parseArchivedManagerLaunch(container, command, image) ++ if err != nil { ++ return err ++ } ++ for _, suffix := range []string{"attacher", "provisioner", "resizer", "snapshotter", "node-driver-registrar", "liveness-probe"} { ++ if flags["--csi-"+suffix+"-image"] != "" { ++ continue ++ } ++ envName, count := "CSI_"+strings.ToUpper(strings.ReplaceAll(suffix, "-", "_"))+"_IMAGE", 0 ++ for _, env := range container.Env { ++ if env.Name == envName { ++ count++ ++ if env.Value == "" || env.ValueFrom != nil { ++ return fmt.Errorf("archived CSI driver image is unresolved") ++ } ++ } ++ } ++ if count != 1 { ++ return fmt.Errorf("archived CSI driver image is incomplete") ++ } ++ } ++ } ++ return nil ++ } ++ for _, container := range pod.Containers { ++ if err := validate(container, false); err != nil { ++ return err ++ } ++ } ++ for _, container := range pod.InitContainers { ++ if err := validate(container, true); err != nil { ++ return err ++ } ++ } ++ for _, container := range pod.EphemeralContainers { ++ if isArchivedManagerContainer(corev1.Container{Name: container.Name, Image: container.Image, Command: container.Command, Args: container.Args}, image) { ++ return fmt.Errorf("archived workload contains unsupported ephemeral manager execution") ++ } ++ } ++ return nil ++} +diff --git a/controller/system_restore_profile.go b/controller/system_restore_profile.go +new file mode 100644 +index 0000000000000000000000000000000000000000..da2df56ea86290f97e24fad2d7f1db51c35c511e +--- /dev/null ++++ b/controller/system_restore_profile.go +@@ -0,0 +1,366 @@ ++package controller ++ ++import ( ++ "context" ++ "fmt" ++ "io" ++ "reflect" ++ "strings" ++ "time" ++ ++ "go.yaml.in/yaml/v3" ++ ++ "k8s.io/apimachinery/pkg/api/meta" ++ "k8s.io/apimachinery/pkg/runtime" ++ ++ corev1 "k8s.io/api/core/v1" ++ storagev1 "k8s.io/api/storage/v1" ++ sigsyaml "sigs.k8s.io/yaml" ++ ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++func validateV2OnlyManagerLaunch(container corev1.Container, command, managerImage string) error { ++ parsed, err := parseArchivedManagerLaunch(container, command, managerImage) ++ if err != nil { ++ return err ++ } ++ if command == "daemon" || command == "global" { ++ if parsed["--data-engine-mode"] != "v2-only" { ++ return fmt.Errorf("archived manager would change the installation capability") ++ } ++ } ++ if command == "daemon" { ++ for _, flag := range []string{"--engine-image", "--instance-manager-image", "--share-manager-image", "--backing-image-manager-image", "--support-bundle-manager-image", "--manager-image", "--service-account"} { ++ if parsed[flag] == "" { ++ return fmt.Errorf("archived manager lacks a required native launch value") ++ } ++ } ++ } ++ if (command == "daemon" || command == "deploy-driver") && parsed["--manager-image"] != managerImage { ++ return fmt.Errorf("archived manager would change the installation image") ++ } ++ if command == "csi" { ++ if parsed["--drivername"] != types.LonghornDriverName || parsed["--nodeid"] == "" || parsed["--endpoint"] == "" || parsed["--manager-url"] == "" { ++ return fmt.Errorf("archived CSI launch is unresolved") ++ } ++ } ++ if command == "deploy-driver" && parsed["--manager-url"] == "" { ++ return fmt.Errorf("archived driver deployer launch is unresolved") ++ } ++ return nil ++} ++ ++func parseArchivedManagerLaunch(container corev1.Container, command, managerImage string) (map[string]string, error) { ++ if managerImage == "" || container.Image != managerImage || (command != "daemon" && command != "global" && command != "csi" && command != "deploy-driver") { ++ return nil, fmt.Errorf("archived manager image or command is incompatible with v2-only mode") ++ } ++ argv := append([]string{}, container.Command...) ++ argv = append(argv, container.Args...) ++ if len(argv) == 0 || argv[0] != "longhorn-manager" { ++ return nil, fmt.Errorf("archived manager does not use a native launch") ++ } ++ index := 1 ++ rootFlags := map[string]bool{} ++ for index < len(argv) { ++ flag := "" ++ switch argv[index] { ++ case "-d", "--debug": ++ flag = "debug" ++ case "-j", "--log-json": ++ flag = "json" ++ } ++ if flag == "" { ++ break ++ } ++ if rootFlags[flag] { ++ return nil, fmt.Errorf("archived manager has duplicate root flags") ++ } ++ rootFlags[flag] = true ++ index++ ++ } ++ if index >= len(argv) || argv[index] != command { ++ return nil, fmt.Errorf("archived manager subcommand is unresolved") ++ } ++ values := map[string]bool{} ++ booleans := map[string]bool{} ++ if command == "daemon" || command == "global" { ++ values["--data-engine-mode"], values["--kube-config"] = true, true ++ } ++ if command == "daemon" { ++ for _, flag := range []string{"--engine-image", "--instance-manager-image", "--share-manager-image", "--backing-image-manager-image", "--support-bundle-manager-image", "--manager-image", "--service-account"} { ++ values[flag] = true ++ } ++ booleans["--upgrade-version-check"] = true ++ } ++ if command == "csi" { ++ for _, flag := range []string{"--nodeid", "--endpoint", "--drivername", "--manager-url"} { ++ values[flag] = true ++ } ++ booleans["--volume-group-snapshot-enabled"] = true ++ } ++ if command == "deploy-driver" { ++ for _, flag := range []string{"--manager-image", "--manager-url", "--csi-attacher-image", "--csi-provisioner-image", "--csi-resizer-image", "--csi-snapshotter-image", "--csi-node-driver-registrar-image", "--csi-liveness-probe-image"} { ++ values[flag] = true ++ } ++ } ++ seen, parsed := map[string]bool{}, map[string]string{} ++ for index++; index < len(argv); index++ { ++ name, value, assigned := strings.Cut(argv[index], "=") ++ if seen[name] || name == "--" { ++ return nil, fmt.Errorf("archived manager has ambiguous options") ++ } ++ seen[name] = true ++ if booleans[name] { ++ if assigned && value != "true" && value != "false" { ++ return nil, fmt.Errorf("archived manager has an invalid Boolean option") ++ } ++ continue ++ } ++ if !values[name] { ++ return nil, fmt.Errorf("archived manager has an unknown launch option") ++ } ++ if !assigned { ++ index++ ++ if index >= len(argv) || strings.HasPrefix(argv[index], "-") { ++ return nil, fmt.Errorf("archived manager option lacks a literal value") ++ } ++ value = argv[index] ++ } ++ if value == "" { ++ return nil, fmt.Errorf("archived manager has an empty option value") ++ } ++ parsed[name] = value ++ } ++ return parsed, nil ++} ++ ++func (c *SystemRolloutController) prepareV2OnlySystemRestore() error { ++ if c.ds == nil { ++ return fmt.Errorf("missing system restore datastore") ++ } ++ mode, err := types.ParseDataEngineMode(string(c.ds.DataEngineMode())) ++ if err != nil || mode != types.DataEngineModeV2Only { ++ return err ++ } ++ // Restoring may resume in a new process or after a failed attempt. Never ++ // trust the old in-memory cache, including when a fresh read fails. ++ c.extractedResources = extractedResources{} ++ if c.systemRestore == nil || c.downloadPath == "" { ++ return fmt.Errorf("missing system restore archive identity") ++ } ++ resources, err := loadV2OnlySystemRestoreArchive(c.getYAMLDirectory("")) ++ if err != nil { ++ return err ++ } ++ if err := validateV2OnlySystemRestoreProfile(&resources, c.systemRestore.Namespace, c.archiveManagerImage); err != nil { ++ return err ++ } ++ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) ++ defer cancel() ++ if err := c.ds.ValidateDataEngineModeState(ctx); err != nil { ++ return err ++ } ++ if err := c.ds.ValidateDataEngineModeRestoreCRDs(ctx, resources.customResourceDefinitionList); err != nil { ++ return err ++ } ++ if err := ctx.Err(); err != nil { ++ return err ++ } ++ c.extractedResources = resources ++ return nil ++} ++ ++func validateV2OnlySystemRestoreProfile(resources *extractedResources, namespace, managerImage string) error { ++ if resources == nil || namespace == "" || managerImage == "" { ++ return fmt.Errorf("v2-only restore requires an identified archive and installation") ++ } ++ lists := []runtime.Object{ ++ resources.customResourceDefinitionList, resources.clusterRoleList, resources.clusterRoleBindingList, ++ resources.roleList, resources.roleBindingList, resources.daemonSetList, resources.deploymentList, ++ resources.configMapList, resources.persistentVolumeList, resources.persistentVolumeClaimList, ++ resources.serviceList, resources.serviceAccountList, resources.storageClassList, ++ resources.engineImageList, resources.recurringJobList, resources.settingList, ++ resources.volumeList, resources.backingImageList, resources.backupTargetList, ++ } ++ for _, list := range lists { ++ if reflect.ValueOf(list).IsNil() { ++ return fmt.Errorf("v2-only restore requires every native resource list") ++ } ++ items, err := meta.ExtractList(list) ++ if err != nil { ++ return fmt.Errorf("cannot inspect archived resource list") ++ } ++ for _, item := range items { ++ metadata, err := meta.Accessor(item) ++ if err != nil || metadata.GetName() == "" || metadata.GetDeletionTimestamp() != nil { ++ return fmt.Errorf("archived resource identity is unresolved or terminating") ++ } ++ // Native backups may include claims from any tenant namespace, ++ // including pending claims. Other namespaced resources belong to ++ // the source Longhorn installation. ++ if metadata.GetNamespace() != "" && list != resources.persistentVolumeClaimList && metadata.GetNamespace() != namespace { ++ return fmt.Errorf("archived resource belongs to a different installation") ++ } ++ } ++ } ++ settings := map[string]string{} ++ for _, setting := range resources.settingList.Items { ++ if _, duplicate := settings[setting.Name]; duplicate { ++ return fmt.Errorf("duplicate archived Setting") ++ } ++ settings[setting.Name] = setting.Value ++ } ++ if settings[string(types.SettingNameV1DataEngine)] != "false" || settings[string(types.SettingNameV2DataEngine)] != "true" { ++ return fmt.Errorf("archived Settings do not explicitly select v2-only mode") ++ } ++ for _, volume := range resources.volumeList.Items { ++ if volume.Spec.DataEngine != longhorn.DataEngineTypeV2 || volume.Spec.BackingImage != "" { ++ return fmt.Errorf("archived Volume is incompatible with v2-only restore") ++ } ++ } ++ if len(resources.backingImageList.Items) != 0 { ++ return fmt.Errorf("backing images are not supported by the v2-only restore profile") ++ } ++ for _, class := range resources.storageClassList.Items { ++ if err := validateV2OnlyRestoreStorageClass(class); err != nil { ++ return err ++ } ++ } ++ for _, volume := range resources.persistentVolumeList.Items { ++ if volume.Spec.CSI == nil || volume.Spec.CSI.Driver != types.LonghornDriverName || volume.Spec.CSI.VolumeHandle == "" { ++ return fmt.Errorf("archived PersistentVolume lacks a native Longhorn CSI identity") ++ } ++ if engine, present := volume.Spec.CSI.VolumeAttributes["dataEngine"]; present && engine != string(longhorn.DataEngineTypeV2) { ++ return fmt.Errorf("archived PersistentVolume has contradictory engine intent") ++ } ++ } ++ defaultSettingsFound := false ++ for _, configMap := range resources.configMapList.Items { ++ switch configMap.Name { ++ case types.DefaultDefaultSettingConfigMapName: ++ if defaultSettingsFound { ++ return fmt.Errorf("duplicate archived default Settings ConfigMap") ++ } ++ defaultSettingsFound = true ++ raw, present := configMap.Data[types.DefaultSettingYAMLFileName] ++ if !present { ++ return fmt.Errorf("archived default Settings document is missing") ++ } ++ values := map[string]string{} ++ if err := decodeV2OnlyRestoreDocument(raw, &values); err != nil { ++ return err ++ } ++ for _, name := range []string{string(types.SettingNameV1DataEngine), string(types.SettingNameV2DataEngine)} { ++ if value, present := values[name]; present && value != settings[name] { ++ return fmt.Errorf("archived default Settings contradict v2-only mode") ++ } ++ } ++ case types.DefaultStorageClassConfigMapName: ++ raw, present := configMap.Data["storageclass.yaml"] ++ if !present { ++ return fmt.Errorf("archived default StorageClass document is missing") ++ } ++ // Decode through Kubernetes' strict JSON representation, rather ++ // than yaml.v3's field-name rules for Go API structs. ++ class, err := decodeV2OnlyRestoreStorageClass(raw) ++ if err != nil { ++ return err ++ } ++ if class.Provisioner != types.LonghornDriverName { ++ return fmt.Errorf("archived default StorageClass has an unexpected provisioner") ++ } ++ if err := validateV2OnlyRestoreStorageClass(*class); err != nil { ++ return err ++ } ++ } ++ } ++ if !defaultSettingsFound { ++ return fmt.Errorf("archived default Settings ConfigMap is missing") ++ } ++ managerFound, globalFound := false, false ++ for _, daemonSet := range resources.daemonSetList.Items { ++ if err := validateV2OnlyRestorePod(daemonSet.Spec.Template.Spec, "DaemonSet", daemonSet.Name, managerImage); err != nil { ++ return err ++ } ++ if daemonSet.Name == types.LonghornManagerDaemonSetName { ++ if managerFound { ++ return fmt.Errorf("duplicate archived manager DaemonSet") ++ } ++ managerFound = true ++ if err := validateV2OnlyRestoreManagerContainer(daemonSet.Spec.Template.Spec, "longhorn-manager", "daemon", managerImage); err != nil { ++ return err ++ } ++ } ++ } ++ for _, deployment := range resources.deploymentList.Items { ++ if err := validateV2OnlyRestorePod(deployment.Spec.Template.Spec, "Deployment", deployment.Name, managerImage); err != nil { ++ return err ++ } ++ if deployment.Name == types.LonghornGlobalManagerName { ++ if globalFound { ++ return fmt.Errorf("duplicate archived global manager Deployment") ++ } ++ globalFound = true ++ if err := validateV2OnlyRestoreManagerContainer(deployment.Spec.Template.Spec, "longhorn-global-manager", "global", managerImage); err != nil { ++ return err ++ } ++ } ++ } ++ if !managerFound || !globalFound { ++ return fmt.Errorf("archived manager workloads are incomplete") ++ } ++ return nil ++} ++ ++func decodeV2OnlyRestoreDocument(raw string, target interface{}) error { ++ decoder := yaml.NewDecoder(strings.NewReader(raw)) ++ if err := decoder.Decode(target); err != nil && err != io.EOF { ++ return fmt.Errorf("invalid archived configuration document") ++ } ++ var extra interface{} ++ if err := decoder.Decode(&extra); err != io.EOF { ++ return fmt.Errorf("multiple or invalid archived configuration documents") ++ } ++ return nil ++} ++ ++func validateV2OnlyRestoreStorageClass(class storagev1.StorageClass) error { ++ if class.Provisioner == types.LonghornDriverName && class.Parameters["dataEngine"] != string(longhorn.DataEngineTypeV2) { ++ return fmt.Errorf("archived Longhorn StorageClass does not explicitly select v2") ++ } ++ return nil ++} ++ ++func decodeV2OnlyRestoreStorageClass(raw string) (*storagev1.StorageClass, error) { ++ var document map[string]interface{} ++ if err := decodeV2OnlyRestoreDocument(raw, &document); err != nil { ++ return nil, err ++ } ++ var class storagev1.StorageClass ++ if err := sigsyaml.UnmarshalStrict([]byte(raw), &class); err != nil || class.APIVersion != storagev1.SchemeGroupVersion.String() || class.Kind != "StorageClass" || class.Name != types.DefaultStorageClassName || class.Namespace != "" { ++ return nil, fmt.Errorf("invalid archived StorageClass document") ++ } ++ return &class, nil ++} ++ ++func validateV2OnlyRestoreManagerContainer(pod corev1.PodSpec, name, command, image string) error { ++ found := false ++ for _, container := range pod.Containers { ++ if container.Name == name { ++ if found { ++ return fmt.Errorf("duplicate archived manager container") ++ } ++ found = true ++ if err := validateV2OnlyManagerLaunch(container, command, image); err != nil { ++ return err ++ } ++ } ++ } ++ if !found { ++ return fmt.Errorf("archived manager container is missing") ++ } ++ return nil ++} +diff --git a/controller/system_restore_profile_test.go b/controller/system_restore_profile_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..b6564e16ab6c6c21b9294d210803c8c817e6013b +--- /dev/null ++++ b/controller/system_restore_profile_test.go +@@ -0,0 +1,331 @@ ++package controller ++ ++import ( ++ "encoding/json" ++ "os" ++ "path/filepath" ++ "strings" ++ "testing" ++ ++ "github.com/sirupsen/logrus" ++ ++ "k8s.io/apimachinery/pkg/runtime" ++ "k8s.io/client-go/tools/record" ++ ++ appsv1 "k8s.io/api/apps/v1" ++ corev1 "k8s.io/api/core/v1" ++ storagev1 "k8s.io/api/storage/v1" ++ apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ++ extensionsfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ k8stesting "k8s.io/client-go/testing" ++ ++ "github.com/longhorn/go-common-libs/multierr" ++ ++ "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" ++ "github.com/longhorn/longhorn-manager/util" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func TestV2OnlyRestoreBindsNativeManagerLaunch(t *testing.T) { ++ const image = "longhornio/longhorn-manager:fixture" ++ for _, test := range []struct { ++ name, command string ++ argv, args []string ++ valid bool ++ }{ ++ {"native-daemon", "daemon", []string{"longhorn-manager", "-d", "-j", "daemon", "--manager-image", image, "--data-engine-mode", "v2-only"}, nil, true}, ++ {"native-global", "global", []string{"longhorn-manager", "-d", "global", "--data-engine-mode=v2-only"}, nil, true}, ++ {"split-args", "daemon", []string{"longhorn-manager", "-d"}, []string{"daemon", "--manager-image", image, "--data-engine-mode", "v2-only", "--upgrade-version-check=false"}, true}, ++ {"missing-mode", "daemon", []string{"longhorn-manager", "daemon", "--manager-image", image}, nil, false}, ++ {"legacy-mode", "daemon", []string{"longhorn-manager", "daemon", "--manager-image", image, "--data-engine-mode", "v1-capable"}, nil, false}, ++ {"duplicate-mode", "daemon", []string{"longhorn-manager", "daemon", "--manager-image", image, "--data-engine-mode", "v1-capable", "--data-engine-mode", "v2-only"}, nil, false}, ++ {"mode-as-value", "daemon", []string{"longhorn-manager", "daemon", "--engine-image=--data-engine-mode", "v2-only", "--manager-image", image}, nil, false}, ++ {"flag-as-image-value", "daemon", []string{"longhorn-manager", "daemon", "--engine-image", "--data-engine-mode", "v2-only", "--manager-image", image}, nil, false}, ++ {"wrong-manager-image", "daemon", []string{"longhorn-manager", "daemon", "--manager-image", "different", "--data-engine-mode", "v2-only"}, nil, false}, ++ {"wrapper", "daemon", []string{"sh", "-c", "longhorn-manager daemon --data-engine-mode v2-only"}, nil, false}, ++ {"end-of-options", "daemon", []string{"longhorn-manager", "daemon", "--", "--manager-image", image, "--data-engine-mode", "v2-only"}, nil, false}, ++ {"unresolved-mode", "global", []string{"longhorn-manager", "global", "--data-engine-mode", "$(MODE)"}, nil, false}, ++ {"unknown-flag", "global", []string{"longhorn-manager", "global", "--data-engine-mode", "v2-only", "--allow-v1"}, nil, false}, ++ {"wrong-command", "global", []string{"longhorn-manager", "daemon", "--data-engine-mode", "v2-only"}, nil, false}, ++ {"duplicate-root-flag", "global", []string{"longhorn-manager", "-d", "--debug", "global", "--data-engine-mode", "v2-only"}, nil, false}, ++ } { ++ t.Run(test.name, func(t *testing.T) { ++ container := corev1.Container{Image: image, Command: test.argv, Args: test.args} ++ if test.valid && test.command == "daemon" { ++ container.Args = append(container.Args, requiredDaemonFixtureFlags()...) ++ } ++ if err := validateV2OnlyManagerLaunch(container, test.command, image); (err == nil) != test.valid { ++ t.Fatalf("launch validity=%t want=%t err=%v", err == nil, test.valid, err) ++ } ++ }) ++ } ++ if err := validateV2OnlyManagerLaunch(corev1.Container{Image: "different", Command: []string{"longhorn-manager", "global", "--data-engine-mode", "v2-only"}}, "global", image); err == nil { ++ t.Fatal("different archived manager image cleared validation") ++ } ++} ++ ++func requiredDaemonFixtureFlags() []string { ++ return []string{"--engine-image", "engine:fixture", "--instance-manager-image", "instance-manager:fixture", "--share-manager-image", "share-manager:fixture", "--backing-image-manager-image", "backing-image-manager:fixture", "--support-bundle-manager-image", "support-bundle-manager:fixture", "--service-account", "longhorn-service-account"} ++} ++ ++func compatibleV2RestoreProfile(t *testing.T) extractedResources { ++ t.Helper() ++ resources, err := loadV2OnlySystemRestoreArchive(writeCompleteRestoreArchive(t)) ++ if err != nil { ++ t.Fatal(err) ++ } ++ const image = "longhornio/longhorn-manager:fixture" ++ metadata := func(name string) metav1.ObjectMeta { ++ return metav1.ObjectMeta{Name: name, Namespace: "longhorn-system"} ++ } ++ resources.settingList.Items = []longhorn.Setting{ ++ {ObjectMeta: metadata(string(types.SettingNameV1DataEngine)), Value: "false"}, ++ {ObjectMeta: metadata(string(types.SettingNameV2DataEngine)), Value: "true"}, ++ } ++ resources.configMapList.Items = []corev1.ConfigMap{ ++ {ObjectMeta: metadata(types.DefaultDefaultSettingConfigMapName), Data: map[string]string{types.DefaultSettingYAMLFileName: "v1-data-engine: false\nv2-data-engine: true\n"}}, ++ {ObjectMeta: metadata(types.DefaultStorageClassConfigMapName), Data: map[string]string{"storageclass.yaml": "apiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata: {name: longhorn}\nprovisioner: driver.longhorn.io\nparameters: {dataEngine: v2}\n"}}, ++ } ++ resources.daemonSetList.Items = []appsv1.DaemonSet{{ObjectMeta: metadata(types.LonghornManagerDaemonSetName), Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "longhorn-manager", Image: image, Command: []string{"longhorn-manager", "-d", "daemon", "--manager-image", image, "--data-engine-mode", "v2-only"}}}}}}}} ++ resources.deploymentList.Items = []appsv1.Deployment{{ObjectMeta: metadata(types.LonghornGlobalManagerName), Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "longhorn-global-manager", Image: image, Command: []string{"longhorn-manager", "-d", "global", "--data-engine-mode", "v2-only"}}}}}}}} ++ resources.daemonSetList.Items[0].Spec.Template.Spec.ServiceAccountName = "longhorn-service-account" ++ resources.daemonSetList.Items[0].Spec.Template.Spec.Containers[0].Args = requiredDaemonFixtureFlags() ++ resources.deploymentList.Items[0].Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{ ++ {Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}, ++ {Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}}, ++ } ++ resources.volumeList.Items = []longhorn.Volume{{ObjectMeta: metadata("restored-volume"), Spec: longhorn.VolumeSpec{DataEngine: longhorn.DataEngineTypeV2}}} ++ resources.storageClassList.Items = []storagev1.StorageClass{{ObjectMeta: metav1.ObjectMeta{Name: "longhorn-v2"}, Provisioner: types.LonghornDriverName, Parameters: map[string]string{"dataEngine": "v2"}}} ++ resources.customResourceDefinitionList.Items = []apiextensionsv1.CustomResourceDefinition{{ObjectMeta: metav1.ObjectMeta{Name: "volumes.longhorn.io"}, Spec: apiextensionsv1.CustomResourceDefinitionSpec{Group: longhorn.SchemeGroupVersion.Group}}} ++ return resources ++} ++ ++func TestV2OnlyRestoreProfileRejectsLegacyAndUnresolvedResources(t *testing.T) { ++ const image = "longhornio/longhorn-manager:fixture" ++ resources := compatibleV2RestoreProfile(t) ++ if err := validateV2OnlySystemRestoreProfile(&resources, "longhorn-system", image); err != nil { ++ t.Fatal(err) ++ } ++ for _, test := range []struct { ++ name string ++ mutate func(*extractedResources) ++ }{ ++ {"v1-enabled", func(r *extractedResources) { r.settingList.Items[0].Value = "true" }}, ++ {"missing-v2", func(r *extractedResources) { r.settingList.Items = r.settingList.Items[:1] }}, ++ {"foreign-setting", func(r *extractedResources) { r.settingList.Items[0].Namespace = "other" }}, ++ {"legacy-volume", func(r *extractedResources) { r.volumeList.Items[0].Spec.DataEngine = longhorn.DataEngineTypeV1 }}, ++ {"empty-volume-engine", func(r *extractedResources) { r.volumeList.Items[0].Spec.DataEngine = "" }}, ++ {"backing-image-dependency", func(r *extractedResources) { r.volumeList.Items[0].Spec.BackingImage = "legacy-image" }}, ++ {"backing-image", func(r *extractedResources) { r.backingImageList.Items = []longhorn.BackingImage{{}} }}, ++ {"v1-class", func(r *extractedResources) { r.storageClassList.Items[0].Parameters["dataEngine"] = "v1" }}, ++ {"implicit-class", func(r *extractedResources) { delete(r.storageClassList.Items[0].Parameters, "dataEngine") }}, ++ {"non-csi-pv", func(r *extractedResources) { ++ r.persistentVolumeList.Items = []corev1.PersistentVolume{{ObjectMeta: metav1.ObjectMeta{Name: "pv"}}} ++ }}, ++ {"foreign-csi-pv", func(r *extractedResources) { ++ r.persistentVolumeList.Items = []corev1.PersistentVolume{{ObjectMeta: metav1.ObjectMeta{Name: "pv"}, Spec: corev1.PersistentVolumeSpec{PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{Driver: "foreign", VolumeHandle: "volume"}}}}} ++ }}, ++ {"conflicting-cm", func(r *extractedResources) { ++ r.configMapList.Items[0].Data[types.DefaultSettingYAMLFileName] = "v1-data-engine: true" ++ }}, ++ {"malformed-cm", func(r *extractedResources) { ++ r.configMapList.Items[0].Data[types.DefaultSettingYAMLFileName] = "v1-data-engine: [sentinel-secret" ++ }}, ++ {"duplicate-cm-key", func(r *extractedResources) { ++ r.configMapList.Items[0].Data[types.DefaultSettingYAMLFileName] = "v1-data-engine: false\nv1-data-engine: true" ++ }}, ++ {"embedded-v1-class", func(r *extractedResources) { ++ r.configMapList.Items[1].Data["storageclass.yaml"] = "apiVersion: storage.k8s.io/v1\nkind: StorageClass\nmetadata: {name: longhorn}\nprovisioner: driver.longhorn.io\nparameters: {dataEngine: v1}\n" ++ }}, ++ {"embedded-wrong-class-name", func(r *extractedResources) { ++ r.configMapList.Items[1].Data["storageclass.yaml"] = strings.ReplaceAll(r.configMapList.Items[1].Data["storageclass.yaml"], "name: longhorn}", "name: longhorn-v2}") ++ }}, ++ {"manager-mode", func(r *extractedResources) { ++ r.daemonSetList.Items[0].Spec.Template.Spec.Containers[0].Command = []string{"longhorn-manager", "daemon", "--manager-image", image} ++ }}, ++ {"manager-image", func(r *extractedResources) { ++ r.daemonSetList.Items[0].Spec.Template.Spec.Containers[0].Image = "different" ++ }}, ++ {"global-mode", func(r *extractedResources) { ++ r.deploymentList.Items[0].Spec.Template.Spec.Containers[0].Command = []string{"longhorn-manager", "global", "--data-engine-mode", "v1-capable"} ++ }}, ++ {"missing-global", func(r *extractedResources) { r.deploymentList.Items = nil }}, ++ {"missing-list", func(r *extractedResources) { r.roleBindingList = nil }}, ++ } { ++ t.Run(test.name, func(t *testing.T) { ++ candidate := compatibleV2RestoreProfile(t) ++ test.mutate(&candidate) ++ if err := validateV2OnlySystemRestoreProfile(&candidate, "longhorn-system", image); err == nil || strings.Contains(err.Error(), "sentinel-secret") { ++ t.Fatal("unresolved archive cleared the profile or exposed raw configuration") ++ } ++ }) ++ } ++} ++ ++func TestV2OnlyRestorePreparationRevalidatesArchiveAndLiveState(t *testing.T) { ++ resources := compatibleV2RestoreProfile(t) ++ root := writeCompleteRestoreArchive(t) ++ for path, object := range map[string]runtime.Object{ ++ "longhorn/setting.yaml": resources.settingList, "longhorn/volumes.yaml": resources.volumeList, ++ "kubernetes/configmaps.yaml": resources.configMapList, "kubernetes/daemonsets.yaml": resources.daemonSetList, ++ "kubernetes/deployments.yaml": resources.deploymentList, "kubernetes/storageclasses.yaml": resources.storageClassList, ++ "apiextensions/customresourcedefinitions.yaml": resources.customResourceDefinitionList, ++ } { ++ document, err := json.Marshal(object) ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err := os.WriteFile(filepath.Join(root, path), document, 0600); err != nil { ++ t.Fatal(err) ++ } ++ } ++ parent := t.TempDir() ++ if err := os.Mkdir(filepath.Join(parent, "backup"), 0700); err != nil { ++ t.Fatal(err) ++ } ++ if err := os.Rename(root, filepath.Join(parent, "backup", types.SystemBackupSubDirYaml)); err != nil { ++ t.Fatal(err) ++ } ++ settingObjects := []runtime.Object{} ++ for _, setting := range resources.settingList.Items { ++ setting.UID, setting.ResourceVersion = "live-setting", "1" ++ settingObjects = append(settingObjects, setting.DeepCopy()) ++ } ++ lh := lhfake.NewClientset(settingObjects...) ++ kube := kubefake.NewClientset() ++ liveCRD := resources.customResourceDefinitionList.Items[0].DeepCopy() ++ liveCRD.UID, liveCRD.ResourceVersion = "live-crd", "1" ++ extensions := extensionsfake.NewClientset(liveCRD) ++ ds := datastore.NewDataStoreForGlobal("longhorn-system", lh, kube, extensions, util.NewInformerFactories("longhorn-system", kube, lh, 0), types.DataEngineModeV2Only) ++ c := &SystemRolloutController{ds: ds, baseController: &baseController{logger: logrus.New().WithField("test", "v2-only-restore")}, ++ systemRestore: &longhorn.SystemRestore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "longhorn-system"}, Spec: longhorn.SystemRestoreSpec{SystemBackup: "backup"}, Status: longhorn.SystemRestoreStatus{State: longhorn.SystemRestoreStateRestoring}}, ++ downloadPath: filepath.Join(parent, "backup.zip"), archiveManagerImage: "longhornio/longhorn-manager:fixture"} ++ if err := c.prepareV2OnlySystemRestore(); err != nil { ++ t.Fatal(err) ++ } ++ if c.volumeList == nil || len(c.volumeList.Items) != 1 { ++ t.Fatal("compatible preparation lost native resources") ++ } ++ for _, client := range []interface{ Actions() []k8stesting.Action }{lh, kube, extensions} { ++ for _, action := range client.Actions() { ++ if action.GetVerb() != "list" && action.GetVerb() != "get" { ++ t.Fatal("restore preparation wrote archived resources") ++ } ++ } ++ } ++ c.systemRestore.Labels = types.GetSystemRestoreInProgressLabel() ++ if err := ds.SystemRestoreInformer.GetStore().Add(c.systemRestore.DeepCopy()); err != nil { ++ t.Fatal(err) ++ } ++ c.systemRestoreName, c.systemRestoredURL, c.systemRestoredAt = "restore", "s3://fixture", "receipt" ++ c.eventRecorder, c.cacheErrors = record.NewFakeRecorder(100), multierr.NewMultiError() ++ statusWritten := false ++ lh.PrependReactor("update", "systemrestores", func(action k8stesting.Action) (bool, runtime.Object, error) { ++ updated := action.(k8stesting.UpdateAction).GetObject().(*longhorn.SystemRestore) ++ if action.GetSubresource() != "status" || updated.Status.State != longhorn.SystemRestoreStateError { ++ t.Fatal("failed schema binding did not fail the restore status") ++ } ++ statusWritten = true ++ return true, updated, nil ++ }) ++ extensions.PrependReactor("get", "customresourcedefinitions", func(k8stesting.Action) (bool, runtime.Object, error) { ++ changed := liveCRD.DeepCopy() ++ changed.Spec.Group = "changed.io" ++ return true, changed, nil ++ }) ++ if err := c.systemRollout(); err == nil || !statusWritten { ++ t.Fatal("schema drift at the write boundary did not abort the real Restoring phase") ++ } ++ for _, client := range []interface{ Actions() []k8stesting.Action }{lh, kube, extensions} { ++ for _, action := range client.Actions() { ++ if action.GetVerb() == "list" || action.GetVerb() == "get" { ++ continue ++ } ++ if action.GetVerb() != "update" || action.GetResource().Resource != "systemrestores" || action.GetSubresource() != "status" { ++ t.Fatal("schema drift allowed a later archived resource write") ++ } ++ } ++ } ++ // A resumed Restoring phase must read the payload again instead of trusting ++ // the cache populated by an earlier attempt. ++ if err := os.Remove(filepath.Join(parent, "backup", types.SystemBackupSubDirYaml, "longhorn", "volumes.yaml")); err != nil { ++ t.Fatal(err) ++ } ++ if err := c.prepareV2OnlySystemRestore(); err == nil || c.volumeList != nil { ++ t.Fatal("resumed restore trusted a stale or partial archive cache") ++ } ++} ++ ++func TestV2OnlyRestorePreservesNativePVAndPendingTenantClaim(t *testing.T) { ++ resources := compatibleV2RestoreProfile(t) ++ resources.persistentVolumeList.Items = []corev1.PersistentVolume{{ObjectMeta: metav1.ObjectMeta{Name: "pv"}, Spec: corev1.PersistentVolumeSpec{StorageClassName: "static-class", PersistentVolumeSource: corev1.PersistentVolumeSource{CSI: &corev1.CSIPersistentVolumeSource{Driver: types.LonghornDriverName, VolumeHandle: "restored-volume"}}}}} ++ resources.persistentVolumeClaimList.Items = []corev1.PersistentVolumeClaim{{ObjectMeta: metav1.ObjectMeta{Name: "pending", Namespace: "tenant"}}} ++ if err := validateV2OnlySystemRestoreProfile(&resources, "longhorn-system", "longhornio/longhorn-manager:fixture"); err != nil { ++ t.Fatal(err) ++ } ++} ++ ++func TestV2OnlyRestoreRejectsAdditionalManagerExecution(t *testing.T) { ++ const image = "longhornio/longhorn-manager:fixture" ++ legacy := corev1.Container{Name: "extra", Image: image, Command: []string{"longhorn-manager", "daemon"}} ++ for _, test := range []struct { ++ name string ++ mutate func(*extractedResources) ++ }{ ++ {"sidecar", func(r *extractedResources) { ++ r.daemonSetList.Items[0].Spec.Template.Spec.Containers = append(r.daemonSetList.Items[0].Spec.Template.Spec.Containers, legacy) ++ }}, ++ {"init-container", func(r *extractedResources) { ++ r.daemonSetList.Items[0].Spec.Template.Spec.InitContainers = []corev1.Container{legacy} ++ }}, ++ {"other-workload", func(r *extractedResources) { ++ r.deploymentList.Items = append(r.deploymentList.Items, appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "other-manager", Namespace: "longhorn-system"}, Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{legacy}}}}}) ++ }}, ++ {"lifecycle", func(r *extractedResources) { ++ r.daemonSetList.Items[0].Spec.Template.Spec.Containers[0].Lifecycle = &corev1.Lifecycle{PostStart: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: legacy.Command}}} ++ }}, ++ {"probe", func(r *extractedResources) { ++ r.daemonSetList.Items[0].Spec.Template.Spec.Containers[0].StartupProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{Exec: &corev1.ExecAction{Command: legacy.Command}}} ++ }}, ++ {"aliased-lifecycle", func(r *extractedResources) { ++ r.daemonSetList.Items[0].Spec.Template.Spec.Containers = append(r.daemonSetList.Items[0].Spec.Template.Spec.Containers, corev1.Container{Name: "aliased", Image: "unrelated:old", Command: []string{"sleep", "infinity"}, Lifecycle: &corev1.Lifecycle{PostStart: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: legacy.Command}}}}) ++ }}, ++ {"ephemeral", func(r *extractedResources) { ++ r.daemonSetList.Items[0].Spec.Template.Spec.EphemeralContainers = []corev1.EphemeralContainer{{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: legacy.Name, Image: legacy.Image, Command: legacy.Command}}} ++ }}, ++ } { ++ t.Run(test.name, func(t *testing.T) { ++ resources := compatibleV2RestoreProfile(t) ++ test.mutate(&resources) ++ if err := validateV2OnlySystemRestoreProfile(&resources, "longhorn-system", image); err == nil { ++ t.Fatal("additional manager execution cleared restore validation") ++ } ++ }) ++ } ++} ++ ++func TestV2OnlyRestoreSupportsNativeCSIAndDriverLaunches(t *testing.T) { ++ const image = "longhornio/longhorn-manager:fixture" ++ resources := compatibleV2RestoreProfile(t) ++ csi := corev1.Container{Name: types.CSIPluginName, Image: image, ++ Args: []string{"longhorn-manager", "-d", "csi", "--nodeid=$(NODE_ID)", "--endpoint=$(CSI_ENDPOINT)", "--drivername=driver.longhorn.io", "--manager-url=http://longhorn-backend:9500/v1"}, ++ Lifecycle: &corev1.Lifecycle{PreStop: &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"/bin/sh", "-c", "rm -f /csi//*"}}}}} ++ resources.daemonSetList.Items = append(resources.daemonSetList.Items, appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{Name: types.CSIPluginName, Namespace: "longhorn-system"}, Spec: appsv1.DaemonSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{csi}}}}}) ++ driver := corev1.Container{Name: "longhorn-driver-deployer", Image: image, Command: []string{"longhorn-manager", "-d", "deploy-driver", "--manager-image", image, "--manager-url", "http://longhorn-backend:9500/v1"}} ++ for _, suffix := range []string{"ATTACHER", "PROVISIONER", "RESIZER", "SNAPSHOTTER", "NODE_DRIVER_REGISTRAR", "LIVENESS_PROBE"} { ++ driver.Env = append(driver.Env, corev1.EnvVar{Name: "CSI_" + suffix + "_IMAGE", Value: "csi:fixture"}) ++ } ++ resources.deploymentList.Items = append(resources.deploymentList.Items, appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "longhorn-driver-deployer", Namespace: "longhorn-system"}, Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{driver}, InitContainers: []corev1.Container{{Name: "wait-longhorn-manager", Image: image, Command: []string{"sh", "-c", archivedManagerWaitScript}}}}}}}) ++ if err := validateV2OnlySystemRestoreProfile(&resources, "longhorn-system", image); err != nil { ++ t.Fatal(err) ++ } ++ resources.deploymentList.Items[1].Spec.Template.Spec.InitContainers[0].Command[2] += "; longhorn-manager daemon" ++ if err := validateV2OnlySystemRestoreProfile(&resources, "longhorn-system", image); err == nil { ++ t.Fatal("extended wait shell cleared native launch validation") ++ } ++} +diff --git a/controller/system_rollout_controller.go b/controller/system_rollout_controller.go +index b273550315a8b36e86207683b11c83820c2d3ea8..b221fe26fa8cb08eaf88a00929ce7618b5ad6525 100644 +--- a/controller/system_rollout_controller.go ++++ b/controller/system_rollout_controller.go +@@ -1,6 +1,7 @@ + package controller + + import ( ++ "context" + "fmt" + "os" + "os/exec" +@@ -145,8 +146,9 @@ type SystemRolloutController struct { + systemRestoredAt string + systemRestoredURL string + +- downloadPath string +- engineImage string ++ downloadPath string ++ engineImage string ++ archiveManagerImage string + + extractedResources + +@@ -431,7 +433,25 @@ func (c *SystemRolloutController) systemRollout() error { + ) + + case longhorn.SystemRestoreStateRestoring: +- c.restore(types.APIExtensionsKindCustomResourceDefinitionList, c.restoreCustomResourceDefinitions, log) ++ err = c.prepareV2OnlySystemRestore() ++ if err != nil { ++ return errors.Wrap(err, "system restore capability validation failed before archived resource writes") ++ } ++ if c.ds.DataEngineMode() == types.DataEngineModeV2Only { ++ // Schema compatibility is a prerequisite for every later write. ++ // Fail the explicit V2-only operation immediately on drift rather ++ // than waiting through the legacy 24-hour per-kind retry loop. ++ err = c.restoreCustomResourceDefinitions() ++ c.postRestoreHandle(types.APIExtensionsKindCustomResourceDefinitionList, err) ++ if err != nil || len(c.cacheErrors) != 0 { ++ if err == nil { ++ err = errors.New("system restore prerequisite state is unresolved") ++ } ++ return errors.Wrap(err, "current schema binding changed during v2-only restore") ++ } ++ } else { ++ c.restore(types.APIExtensionsKindCustomResourceDefinitionList, c.restoreCustomResourceDefinitions, log) ++ } + c.restore(types.KubernetesKindDaemonSetList, c.restoreDaemonSets, log) + c.restore(types.LonghornKindEngineImageList, c.restoreEngineImages, log) + c.restore(types.LonghornKindSettingList, c.restoreSettings, log) +@@ -562,6 +582,7 @@ func (c *SystemRolloutController) syncController() error { + c.backupTargetCredential = backupTargetClient.Credential + c.backupTargetURL = backupTargetClient.URL + c.engineImage = systemBackupCfg.EngineImage ++ c.archiveManagerImage = systemBackupCfg.ManagerImage + + c.cacheErrors = multierr.NewMultiError() + +@@ -733,6 +754,12 @@ func (c *SystemRolloutController) Unpack(log logrus.FieldLogger) error { + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, "failed to unzip %v", c.downloadPath) + } ++ if mode, err := types.ParseDataEngineMode(string(c.ds.DataEngineMode())); err != nil || mode == types.DataEngineModeV2Only { ++ if err != nil { ++ return err ++ } ++ return c.prepareV2OnlySystemRestore() ++ } + + if err := c.cacheKubernetesResources(); err != nil { + return errors.Wrap(err, "failed to extract Kubernetes resources") +@@ -1015,11 +1042,23 @@ func (c *SystemRolloutController) restoreCustomResourceDefinitions() (err error) + if c.customResourceDefinitionList == nil { + return nil + } ++ var restoreContext context.Context ++ if c.ds.DataEngineMode() == types.DataEngineModeV2Only { ++ var cancel context.CancelFunc ++ restoreContext, cancel = context.WithTimeout(context.Background(), 30*time.Second) ++ defer cancel() ++ } + + for _, restore := range c.customResourceDefinitionList.Items { + log := c.logger.WithField(types.APIExtensionsKindCustomResourceDefinition, restore.Name) + +- exist, err := c.ds.GetCustomResourceDefinition(restore.Name) ++ var exist *apiextensionsv1.CustomResourceDefinition ++ var err error ++ if c.ds.DataEngineMode() == types.DataEngineModeV2Only { ++ exist, err = c.ds.GetDataEngineModeRestoreCRD(restoreContext, &restore) ++ } else { ++ exist, err = c.ds.GetCustomResourceDefinition(restore.Name) ++ } + if err != nil { + if !datastore.ErrorIsNotFound(err) { + return err +@@ -1090,6 +1129,9 @@ func (c *SystemRolloutController) restoreCustomResourceDefinitions() (err error) + if !ok { + return nil, fmt.Errorf(SystemRolloutErrFailedConvertToObjectFmt, exist.GetObjectKind(), types.APIExtensionsKindCustomResourceDefinition) + } ++ if c.ds.DataEngineMode() == types.DataEngineModeV2Only { ++ return c.ds.UpdateDataEngineModeRestoreCRD(restoreContext, obj) ++ } + return c.ds.UpdateCustomResourceDefinition(obj) + } + _, err = c.rolloutResource(updateExist, fnUpdate, isSkipped, log, SystemRolloutMsgSkipIdentical) +@@ -1097,6 +1139,9 @@ func (c *SystemRolloutController) restoreCustomResourceDefinitions() (err error) + return err + } + } ++ if restoreContext != nil { ++ return restoreContext.Err() ++ } + + return nil + } +diff --git a/controller/system_rollout_controller_test.go b/controller/system_rollout_controller_test.go +index ddff8dbb800fada19a8371fdf3b734a7403b6cfe..de80de814a5fbcc13459fb25def114c0769c8dfb 100644 +--- a/controller/system_rollout_controller_test.go ++++ b/controller/system_rollout_controller_test.go +@@ -1012,7 +1012,7 @@ func (s *TestSuite) TestSystemRollout(c *C) { + fakeSystemRolloutVolumes(tc.backupVolumes, c, informerFactories.LhInformerFactory, lhClient) + fakeSystemRolloutBackingImages(tc.backupBackingImages, c, informerFactories.LhInformerFactory, lhClient) + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + doneCh := make(chan struct{}) + if tc.expectState != longhorn.SystemRestoreStateCompleted && tc.expectState != longhorn.SystemRestoreStateError { + doneChs = append(doneChs, doneCh) +diff --git a/controller/volume_attachment_controller_test.go b/controller/volume_attachment_controller_test.go +index 60e41345b8ee1ffafbc49e6748de973426c2c24b..c46c8506f6a93e4d95dfd7828f8a4a5b937614da 100644 +--- a/controller/volume_attachment_controller_test.go ++++ b/controller/volume_attachment_controller_test.go +@@ -430,7 +430,7 @@ func (s *TestSuite) TestIsVolumeAvailableOnNodeV2RequiresReadyEngineFrontend(c * + + informerFactories := util.NewInformerFactories(TestNamespace, kubeClient, lhClient, 0) + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + logger := logrus.StandardLogger() + + volumeIndexer := informerFactories.LhInformerFactory.Longhorn().V1beta2().Volumes().Informer().GetIndexer() +@@ -509,7 +509,7 @@ func (s *TestSuite) runVolumeAttachmentTestCase(c *C, tc *volumeAttachmentTestCa + + informerFactories := util.NewInformerFactories(TestNamespace, kubeClient, lhClient, 0) + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + logger := logrus.StandardLogger() + + volumeIndexer := informerFactories.LhInformerFactory.Longhorn().V1beta2().Volumes().Informer().GetIndexer() +diff --git a/controller/volume_controller_test.go b/controller/volume_controller_test.go +index a9d72c19dc293d273e672c14cf98a1875fe78ab7..0baa90fabafdf79dc2846653f3523a76fd01bc83 100644 +--- a/controller/volume_controller_test.go ++++ b/controller/volume_controller_test.go +@@ -58,7 +58,7 @@ func initSettingsNameValue(name, value string) *longhorn.Setting { + + func newTestVolumeController(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, extensionsClient *apiextensionsfake.Clientset, + informerFactories *util.InformerFactories, controllerID string) (*VolumeController, error) { +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + proxyConnCounter := util.NewAtomicCounter() + +@@ -2939,7 +2939,7 @@ func (s *TestSuite) TestCleanupAutoBalancedReplicasSkipsUnstableNodeIfItWorsensB + c.Assert(informerFactories.KubeInformerFactory.Core().V1().Nodes().Informer().GetIndexer().Add(node), IsNil) + } + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + vc := &VolumeController{ + baseController: newBaseController("test-volume", logrus.StandardLogger()), + ds: ds, +diff --git a/datastore/data_engine_bootstrap.go b/datastore/data_engine_bootstrap.go +new file mode 100644 +index 0000000000000000000000000000000000000000..090c53034b899bdbebfb10e5c7e7b631a1dd3dde +--- /dev/null ++++ b/datastore/data_engine_bootstrap.go +@@ -0,0 +1,84 @@ ++package datastore ++ ++import ( ++ "context" ++ ++ "github.com/cockroachdb/errors" ++ ++ "k8s.io/client-go/kubernetes" ++ ++ apierrors "k8s.io/apimachinery/pkg/api/errors" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhclientset "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned" ++) ++ ++// InitializeDataEngineModeSettings creates missing compatible engine Settings ++// only after an uncached bootstrap preflight. It never rewrites an existing one. ++func InitializeDataEngineModeSettings(ctx context.Context, mode types.DataEngineMode, namespace string, lhClient lhclientset.Interface, kubeClient kubernetes.Interface) error { ++ parsed, err := types.ParseDataEngineMode(string(mode)) ++ if err != nil { ++ return err ++ } ++ if parsed != types.DataEngineModeV2Only { ++ return nil ++ } ++ if ctx == nil || namespace == "" || lhClient == nil || kubeClient == nil { ++ return errors.New("v2-only initialization requires a context, namespace and both API clients") ++ } ++ if err := ctx.Err(); err != nil { ++ return err ++ } ++ known, err := readDataEngineModeSettings(ctx, namespace, lhClient, true) ++ if err != nil { ++ return err ++ } ++ readUnchanged := func(previous [2]dataEngineSettingIdentity) ([2]dataEngineSettingIdentity, error) { ++ current, err := readDataEngineModeSettings(ctx, namespace, lhClient, true) ++ if err != nil { ++ return current, err ++ } ++ for index, identity := range previous { ++ if identity.uid != "" && identity != current[index] { ++ return current, errors.New("engine Setting identity changed during v2-only initialization") ++ } ++ } ++ return current, nil ++ } ++ for index, name := range []types.SettingName{types.SettingNameV1DataEngine, types.SettingNameV2DataEngine} { ++ if err := ValidateDataEngineModeBootstrapState(ctx, parsed, namespace, lhClient, kubeClient); err != nil { ++ return err ++ } ++ known, err = readUnchanged(known) ++ if err != nil { ++ return err ++ } ++ if known[index].uid != "" { ++ continue ++ } ++ created, err := lhClient.LonghornV1beta2().Settings(namespace).Create(ctx, &longhorn.Setting{ ++ ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: namespace}, ++ Value: v2OnlyEngineSettingValue(name), ++ }, metav1.CreateOptions{}) ++ if err != nil && !apierrors.IsAlreadyExists(err) { ++ return errors.Wrapf(err, "failed to create compatible %s Setting", name) ++ } ++ if err == nil { ++ known[index], err = dataEngineModeSettingIdentity(created, name, namespace) ++ if err != nil { ++ return err ++ } ++ } ++ known, err = readUnchanged(known) ++ if err != nil { ++ return err ++ } ++ if known[index].uid == "" { ++ return errors.New("engine Setting creation did not leave a compatible persisted member") ++ } ++ } ++ return ValidateDataEngineModeState(ctx, parsed, namespace, lhClient, kubeClient) ++} +diff --git a/datastore/data_engine_bootstrap_test.go b/datastore/data_engine_bootstrap_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..18aedc0ac08a27ce55744ac94418092549afea4c +--- /dev/null ++++ b/datastore/data_engine_bootstrap_test.go +@@ -0,0 +1,217 @@ ++package datastore ++ ++import ( ++ "context" ++ "errors" ++ "testing" ++ ++ "k8s.io/apimachinery/pkg/runtime" ++ ++ apierrors "k8s.io/apimachinery/pkg/api/errors" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ k8stypes "k8s.io/apimachinery/pkg/types" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ k8stesting "k8s.io/client-go/testing" ++ ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func TestV2OnlyBootstrapRequiresEmptyStorageWhenSettingsAreMissing(t *testing.T) { ++ for _, mask := range []int{0, 1, 2, 3} { ++ for _, engine := range []longhorn.DataEngineType{"none", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2} { ++ objects := []runtime.Object{} ++ for index, name := range []types.SettingName{types.SettingNameV1DataEngine, types.SettingNameV2DataEngine} { ++ if mask&(1< 1000 { ++ return fmt.Errorf("cannot obtain a complete current CRD inventory for v2-only restore") ++ } ++ current := map[string]*apiextensionsv1.CustomResourceDefinition{} ++ for index := range live.Items { ++ crd := &live.Items[index] ++ if crd.Spec.Group != "longhorn.io" { ++ continue ++ } ++ if _, duplicate := current[crd.Name]; duplicate { ++ return fmt.Errorf("duplicate current Longhorn CRD") ++ } ++ current[crd.Name] = crd ++ } ++ if len(current) != len(archive.Items) { ++ return fmt.Errorf("archived and current Longhorn CRD inventories differ") ++ } ++ for _, archived := range archive.Items { ++ live := current[archived.Name] ++ if err := validateV2OnlyRestoreCRD(live, &archived); err != nil { ++ return err ++ } ++ delete(current, archived.Name) ++ } ++ return ctx.Err() ++} ++ ++// GetDataEngineModeRestoreCRD rebinds schema compatibility at the native ++// restore write boundary. Its ResourceVersion is used by the subsequent ++// update, so a concurrent schema change produces a conflict rather than a ++// downgrade. Missing CRDs are never recreated from an archive in v2-only mode. ++func (s *DataStore) GetDataEngineModeRestoreCRD(ctx context.Context, archive *apiextensionsv1.CustomResourceDefinition) (*apiextensionsv1.CustomResourceDefinition, error) { ++ if ctx == nil || s.extensionsClient == nil || archive == nil { ++ return nil, fmt.Errorf("missing v2-only restore CRD binding") ++ } ++ if err := ctx.Err(); err != nil { ++ return nil, err ++ } ++ live, err := s.extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, archive.Name, metav1.GetOptions{}) ++ if err != nil { ++ return nil, fmt.Errorf("cannot rebind current CRD for v2-only restore") ++ } ++ if err := validateV2OnlyRestoreCRD(live, archive); err != nil { ++ return nil, err ++ } ++ return live, ctx.Err() ++} ++ ++func validateV2OnlyRestoreCRD(live, archive *apiextensionsv1.CustomResourceDefinition) error { ++ if live == nil || archive == nil || archive.Name == "" || live.Name != archive.Name || live.Namespace != "" || archive.Namespace != "" || archive.Spec.Group != "longhorn.io" || live.UID == "" || live.ResourceVersion == "" || live.DeletionTimestamp != nil || archive.DeletionTimestamp != nil || !reflect.DeepEqual(live.Spec, archive.Spec) { ++ return fmt.Errorf("archived CRD is missing or incompatible with the current V2 schema") ++ } ++ return nil ++} ++ ++func (s *DataStore) UpdateDataEngineModeRestoreCRD(ctx context.Context, crd *apiextensionsv1.CustomResourceDefinition) (*apiextensionsv1.CustomResourceDefinition, error) { ++ if ctx == nil || s.dataEngineMode != types.DataEngineModeV2Only || s.extensionsClient == nil || crd == nil { ++ return nil, fmt.Errorf("missing bounded v2-only CRD update identity") ++ } ++ if err := ctx.Err(); err != nil { ++ return nil, err ++ } ++ live, err := s.GetDataEngineModeRestoreCRD(ctx, crd) ++ if err != nil { ++ return nil, err ++ } ++ if live.UID != crd.UID || live.ResourceVersion != crd.ResourceVersion { ++ return nil, fmt.Errorf("current CRD identity changed before v2-only restore update") ++ } ++ updated, err := s.extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{}) ++ if err != nil { ++ return nil, fmt.Errorf("cannot update the bound CRD during v2-only restore") ++ } ++ if err := validateV2OnlyRestoreCRD(updated, crd); err != nil || updated.UID != live.UID || !reflect.DeepEqual(updated.Annotations, crd.Annotations) { ++ return nil, fmt.Errorf("v2-only CRD update response is unresolved") ++ } ++ observed, err := s.GetDataEngineModeRestoreCRD(ctx, crd) ++ if err != nil { ++ return nil, err ++ } ++ if observed.UID != updated.UID || observed.ResourceVersion != updated.ResourceVersion || !reflect.DeepEqual(observed.Annotations, updated.Annotations) { ++ return nil, fmt.Errorf("v2-only CRD update readback changed") ++ } ++ return observed, ctx.Err() ++} +diff --git a/datastore/data_engine_mode_restore_test.go b/datastore/data_engine_mode_restore_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..8547f8fa61f27144fd568eea0e94ec8c60087347 +--- /dev/null ++++ b/datastore/data_engine_mode_restore_test.go +@@ -0,0 +1,129 @@ ++package datastore ++ ++import ( ++ "context" ++ "errors" ++ "testing" ++ ++ "k8s.io/apimachinery/pkg/runtime" ++ ++ apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" ++ extensionsfake "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/fake" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ k8stesting "k8s.io/client-go/testing" ++ ++ "github.com/longhorn/longhorn-manager/types" ++) ++ ++func TestV2OnlyRestoreCRDsRequireCompleteCurrentCompatibleSchemas(t *testing.T) { ++ crd := apiextensionsv1.CustomResourceDefinition{ ++ ObjectMeta: metav1.ObjectMeta{Name: "volumes.longhorn.io", UID: "live-crd", ResourceVersion: "7"}, ++ Spec: apiextensionsv1.CustomResourceDefinitionSpec{Group: "longhorn.io", Names: apiextensionsv1.CustomResourceDefinitionNames{Plural: "volumes"}, Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{Name: "v1beta2", Served: true, Storage: true}}}, ++ } ++ archive := &apiextensionsv1.CustomResourceDefinitionList{Items: []apiextensionsv1.CustomResourceDefinition{*crd.DeepCopy()}} ++ for _, test := range []string{"compatible", "missing", "changed-schema", "deleting", "missing-identity", "incomplete", "duplicate-archive", "foreign-archive", "extra-live", "empty-archive"} { ++ t.Run(test, func(t *testing.T) { ++ candidate, live := archive.DeepCopy(), crd.DeepCopy() ++ switch test { ++ case "missing": ++ live.Name = "other.longhorn.io" ++ case "changed-schema": ++ live.Spec.Versions[0].Served = false ++ case "deleting": ++ now := metav1.Now() ++ live.DeletionTimestamp = &now ++ case "missing-identity": ++ live.UID = "" ++ case "duplicate-archive": ++ candidate.Items = append(candidate.Items, crd) ++ case "foreign-archive": ++ candidate.Items[0].Spec.Group = "other.io" ++ case "empty-archive": ++ candidate.Items = nil ++ } ++ objects := []runtime.Object{live} ++ if test == "extra-live" { ++ extra := crd.DeepCopy() ++ extra.Name = "settings.longhorn.io" ++ extra.Spec.Names.Plural = "settings" ++ objects = append(objects, extra) ++ } ++ client := extensionsfake.NewClientset(objects...) ++ if test == "incomplete" { ++ client.PrependReactor("list", "customresourcedefinitions", func(k8stesting.Action) (bool, runtime.Object, error) { ++ return true, &apiextensionsv1.CustomResourceDefinitionList{ListMeta: metav1.ListMeta{Continue: "next"}, Items: []apiextensionsv1.CustomResourceDefinition{*live}}, nil ++ }) ++ } ++ ds := &DataStore{dataEngineMode: types.DataEngineModeV2Only, extensionsClient: client} ++ if err := ds.ValidateDataEngineModeRestoreCRDs(t.Context(), candidate); (err == nil) != (test == "compatible") { ++ t.Fatalf("unexpected CRD compatibility result: %v", err) ++ } ++ for _, action := range client.Actions() { ++ if action.GetVerb() != "list" && action.GetVerb() != "get" { ++ t.Fatal("CRD validation wrote to the cluster") ++ } ++ } ++ }) ++ } ++} ++ ++func TestV2OnlyRestoreCRDUpdateBindsDeadlineSchemaAndReadback(t *testing.T) { ++ for _, scenario := range []string{"compatible", "expired", "schema-drift", "update-error", "readback-error", "readback-identity-drift", "stripped-receipt"} { ++ t.Run(scenario, func(t *testing.T) { ++ crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: "volumes.longhorn.io", UID: "uid", ResourceVersion: "1", Annotations: map[string]string{"restore-receipt": "expected"}}, Spec: apiextensionsv1.CustomResourceDefinitionSpec{Group: "longhorn.io"}} ++ live := crd.DeepCopy() ++ if scenario == "schema-drift" { ++ live.Spec.Group = "changed.io" ++ } ++ client := extensionsfake.NewClientset(live) ++ gets := 0 ++ client.PrependReactor("get", "customresourcedefinitions", func(k8stesting.Action) (bool, runtime.Object, error) { ++ gets++ ++ if gets == 2 && scenario == "readback-error" { ++ return true, nil, errors.New("unavailable") ++ } ++ if gets == 2 && scenario == "readback-identity-drift" { ++ changed := crd.DeepCopy() ++ changed.UID = "replacement" ++ return true, changed, nil ++ } ++ return false, nil, nil ++ }) ++ if scenario == "update-error" { ++ client.PrependReactor("update", "customresourcedefinitions", func(k8stesting.Action) (bool, runtime.Object, error) { ++ return true, nil, errors.New("conflict") ++ }) ++ } ++ if scenario == "stripped-receipt" { ++ client.PrependReactor("update", "customresourcedefinitions", func(action k8stesting.Action) (bool, runtime.Object, error) { ++ changed := action.(k8stesting.UpdateAction).GetObject().(*apiextensionsv1.CustomResourceDefinition).DeepCopy() ++ changed.Annotations = nil ++ if err := client.Tracker().Update(action.GetResource(), changed, ""); err != nil { ++ t.Fatal(err) ++ } ++ return true, changed, nil ++ }) ++ } ++ ctx, cancel := context.WithCancel(t.Context()) ++ defer cancel() ++ if scenario == "expired" { ++ cancel() ++ } ++ ds := &DataStore{dataEngineMode: types.DataEngineModeV2Only, extensionsClient: client} ++ _, err := ds.UpdateDataEngineModeRestoreCRD(ctx, crd) ++ if (err == nil) != (scenario == "compatible") { ++ t.Fatalf("unexpected bounded CRD update result: %v", err) ++ } ++ if scenario == "expired" && len(client.Actions()) != 0 { ++ t.Fatal("expired CRD update contacted the API") ++ } ++ if scenario == "schema-drift" { ++ for _, action := range client.Actions() { ++ if action.GetVerb() == "update" { ++ t.Fatal("schema drift permitted a CRD write") ++ } ++ } ++ } ++ }) ++ } ++} +diff --git a/datastore/data_engine_mode_settings.go b/datastore/data_engine_mode_settings.go +new file mode 100644 +index 0000000000000000000000000000000000000000..81b8f1b0cbb098e53600f11fca302e17c21a2016 +--- /dev/null ++++ b/datastore/data_engine_mode_settings.go +@@ -0,0 +1,111 @@ ++package datastore ++ ++import ( ++ "context" ++ "io" ++ "strings" ++ "time" ++ ++ "github.com/cockroachdb/errors" ++ "go.yaml.in/yaml/v3" ++ ++ corev1 "k8s.io/api/core/v1" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ ++ "github.com/longhorn/longhorn-manager/types" ++) ++ ++// ValidateDataEngineModeStartup checks raw chart intent and uncached storage ++// before upgrades or default-Setting writes. Missing members remain a separate ++// create-only bootstrap phase. ++func (s *DataStore) ValidateDataEngineModeStartup(ctx context.Context) error { ++ mode, err := types.ParseDataEngineMode(string(s.dataEngineMode)) ++ if err != nil { ++ return err ++ } ++ if mode != types.DataEngineModeV2Only { ++ return nil ++ } ++ if ctx == nil || s.namespace == "" || s.lhClient == nil || s.kubeClient == nil { ++ return errors.New("v2-only startup requires a context, namespace and both API clients") ++ } ++ ctx, cancel := context.WithTimeout(ctx, 30*time.Second) ++ defer cancel() ++ readIntent := func() (*corev1.ConfigMap, error) { ++ configMap, err := s.kubeClient.CoreV1().ConfigMaps(s.namespace).Get(ctx, types.DefaultDefaultSettingConfigMapName, metav1.GetOptions{}) ++ if err != nil { ++ return nil, errors.Wrap(err, "failed to read raw default Settings for v2-only startup") ++ } ++ if err := s.validateDataEngineModeCustomizedSettings(configMap); err != nil { ++ return nil, err ++ } ++ return configMap, nil ++ } ++ before, err := readIntent() ++ if err != nil { ++ return err ++ } ++ if err := ValidateDataEngineModeBootstrapState(ctx, mode, s.namespace, s.lhClient, s.kubeClient); err != nil { ++ return err ++ } ++ after, err := readIntent() ++ if err != nil { ++ return err ++ } ++ if before.UID != after.UID || before.ResourceVersion != after.ResourceVersion { ++ return errors.New("default Settings ConfigMap changed during v2-only startup validation") ++ } ++ return ctx.Err() ++} ++ ++func (s *DataStore) InitializeDataEngineModeSettings(ctx context.Context) error { ++ return InitializeDataEngineModeSettings(ctx, s.dataEngineMode, s.namespace, s.lhClient, s.kubeClient) ++} ++ ++func isDataEngineSetting(name types.SettingName) bool { ++ return name == types.SettingNameV1DataEngine || name == types.SettingNameV2DataEngine ++} ++ ++func (s *DataStore) validateDataEngineModeSetting(name types.SettingName, value string) error { ++ mode, err := types.ParseDataEngineMode(string(s.dataEngineMode)) ++ if err != nil { ++ return err ++ } ++ if mode == types.DataEngineModeV2Only && isDataEngineSetting(name) && value != v2OnlyEngineSettingValue(name) { ++ return errors.New("v2-only capability requires v1-data-engine=false and v2-data-engine=true") ++ } ++ return nil ++} ++ ++func (s *DataStore) validateDataEngineModeCustomizedSettings(configMap *corev1.ConfigMap) error { ++ mode, err := types.ParseDataEngineMode(string(s.dataEngineMode)) ++ if err != nil { ++ return err ++ } ++ if mode != types.DataEngineModeV2Only { ++ return nil ++ } ++ if configMap == nil || configMap.Name != types.DefaultDefaultSettingConfigMapName || configMap.Namespace != s.namespace || configMap.UID == "" || configMap.ResourceVersion == "" || configMap.DeletionTimestamp != nil { ++ return errors.New("v2-only mode requires an identified default Settings ConfigMap") ++ } ++ // The general default-Settings parser logs malformed input. Validate this ++ // capability boundary without echoing an archived or live ConfigMap body, ++ // and reject a second document rather than silently reading only the first. ++ raw := map[string]string{} ++ decoder := yaml.NewDecoder(strings.NewReader(configMap.Data[types.DefaultSettingYAMLFileName])) ++ if err := decoder.Decode(&raw); err != nil && err != io.EOF { ++ return errors.New("invalid raw default Settings document for v2-only mode") ++ } ++ var extra interface{} ++ if err := decoder.Decode(&extra); err != io.EOF { ++ return errors.New("multiple or invalid raw default Settings documents for v2-only mode") ++ } ++ for _, name := range []types.SettingName{types.SettingNameV1DataEngine, types.SettingNameV2DataEngine} { ++ if value, present := raw[string(name)]; present { ++ if err := s.validateDataEngineModeSetting(name, value); err != nil { ++ return errors.Wrap(err, "invalid raw engine intent in default Settings ConfigMap") ++ } ++ } ++ } ++ return nil ++} +diff --git a/datastore/data_engine_mode_settings_test.go b/datastore/data_engine_mode_settings_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..0d0a82dfe421eb06621c990f462b0ddec6be2a2d +--- /dev/null ++++ b/datastore/data_engine_mode_settings_test.go +@@ -0,0 +1,173 @@ ++package datastore ++ ++import ( ++ "bytes" ++ "context" ++ "strings" ++ "testing" ++ ++ "github.com/sirupsen/logrus" ++ ++ "k8s.io/apimachinery/pkg/runtime" ++ ++ corev1 "k8s.io/api/core/v1" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ k8stesting "k8s.io/client-go/testing" ++ ++ "github.com/longhorn/longhorn-manager/types" ++ "github.com/longhorn/longhorn-manager/util" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++func TestV2OnlyStartupBindsRawConfigMapAcrossBootstrapInventory(t *testing.T) { ++ for _, replace := range []bool{false, true} { ++ lh, kube := bootstrapClients() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, types.DataEngineModeV2Only) ++ configMap := &corev1.ConfigMap{ ++ ObjectMeta: metav1.ObjectMeta{Name: types.DefaultDefaultSettingConfigMapName, Namespace: "longhorn-system", UID: "config-uid", ResourceVersion: "1"}, ++ Data: map[string]string{types.DefaultSettingYAMLFileName: "v1-data-engine: 'false'\nv2-data-engine: 'true'"}, ++ } ++ if err := kube.Tracker().Add(configMap); err != nil { ++ t.Fatal(err) ++ } ++ if replace { ++ kube.PrependReactor("list", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { ++ changed := configMap.DeepCopy() ++ changed.ResourceVersion = "2" ++ if err := kube.Tracker().Update(corev1.SchemeGroupVersion.WithResource("configmaps"), changed, changed.Namespace); err != nil { ++ t.Fatal(err) ++ } ++ return false, nil, nil ++ }) ++ } ++ err := store.ValidateDataEngineModeStartup(t.Context()) ++ if (err == nil) == replace { ++ t.Fatalf("startup intent continuity: replaced=%t err=%v", replace, err) ++ } ++ for _, action := range lh.Actions() { ++ if action.GetVerb() != "get" && action.GetVerb() != "list" { ++ t.Fatal("startup preflight mutated engine state") ++ } ++ } ++ } ++} ++ ++func TestV2OnlyRawDefaultsRejectAmbiguityWithoutLoggingPayload(t *testing.T) { ++ previous := logrus.StandardLogger().Out ++ var output bytes.Buffer ++ logrus.SetOutput(&output) ++ defer logrus.SetOutput(previous) ++ store := &DataStore{namespace: "longhorn-system", dataEngineMode: types.DataEngineModeV2Only} ++ for _, raw := range []string{"v1-data-engine: [sentinel-secret", "v1-data-engine: 'false'\nv1-data-engine: 'true'", "v1-data-engine: 'false'\n---\nv1-data-engine: 'true'\n"} { ++ configMap := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: types.DefaultDefaultSettingConfigMapName, Namespace: "longhorn-system", UID: "uid", ResourceVersion: "1"}, Data: map[string]string{types.DefaultSettingYAMLFileName: raw}} ++ err := store.validateDataEngineModeCustomizedSettings(configMap) ++ if err == nil || strings.Contains(err.Error(), "sentinel-secret") || strings.Contains(output.String(), "sentinel-secret") { ++ t.Fatal("ambiguous raw engine defaults cleared validation or leaked the payload") ++ } ++ } ++} ++ ++func TestDefaultStartupPreservesNoAdditionalAPIChecks(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable} { ++ store := &DataStore{dataEngineMode: mode} ++ if err := store.ValidateDataEngineModeStartup(context.Background()); err != nil { ++ t.Fatal(err) ++ } ++ } ++} ++ ++func TestV2OnlySettingValidationRejectsUnchangedContradictions(t *testing.T) { ++ lh, kube := modeStateClients() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, types.DataEngineModeV2Only) ++ for name, value := range map[types.SettingName]string{types.SettingNameV1DataEngine: "true", types.SettingNameV2DataEngine: "false"} { ++ setting := &longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: "longhorn-system"}, Value: value} ++ if err := factories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer().Add(setting); err != nil { ++ t.Fatal(err) ++ } ++ if err := store.ValidateSetting(string(name), value); err == nil || types.ErrorIsNotSupport(err) { ++ t.Fatalf("unchanged contradictory Setting bypassed the capability ceiling: name=%s err=%v", name, err) ++ } ++ } ++} ++ ++func TestV2OnlyGenericSettingSyncCannotCreateOrUpsertEngineTuple(t *testing.T) { ++ lh, kube := modeStateClients() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, types.DataEngineModeV2Only) ++ indexer := factories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer() ++ for _, name := range types.SettingNameList { ++ if name == types.SettingNameV1DataEngine || name == types.SettingNameV2DataEngine { ++ continue ++ } ++ if err := indexer.Add(&longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: "longhorn-system"}, Value: "present"}); err != nil { ++ t.Fatal(err) ++ } ++ } ++ if err := store.createNonExistingSettingCRsWithDefaultSetting("config-version"); err != nil { ++ t.Fatal(err) ++ } ++ customized := map[string]string{string(types.SettingNameV1DataEngine): "false", string(types.SettingNameV2DataEngine): "true"} ++ if err := store.syncSettingCRsWithCustomizedDefaultSettings(customized, "config-version"); err != nil { ++ t.Fatal(err) ++ } ++ if result := store.filterCustomizedDefaultSettings(customized, "config-version"); len(result) != 0 { ++ t.Fatal("generic customized Settings changed engine definitions") ++ } ++ if len(lh.Actions()) != 0 { ++ t.Fatal("generic Settings sync touched the create-only engine tuple") ++ } ++} ++ ++func TestDefaultSettingValidationPreservesMutableEngineSettings(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable} { ++ lh, kube := modeStateClients() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, mode) ++ for name, value := range map[types.SettingName]string{types.SettingNameV1DataEngine: "true", types.SettingNameV2DataEngine: "false"} { ++ setting := &longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: "longhorn-system"}, Value: value} ++ if err := factories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer().Add(setting); err != nil { ++ t.Fatal(err) ++ } ++ if err := store.ValidateSetting(string(name), value); err != nil { ++ t.Fatalf("default mode lost mutable engine Settings: name=%s err=%v", name, err) ++ } ++ } ++ } ++} ++ ++func TestV2OnlySettingValidationAllowsCompatibleBootstrapSingleton(t *testing.T) { ++ for name, value := range map[types.SettingName]string{types.SettingNameV1DataEngine: "false", types.SettingNameV2DataEngine: "true"} { ++ lh, kube := bootstrapClients() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, types.DataEngineModeV2Only) ++ if err := store.ValidateSetting(string(name), value); err != nil { ++ t.Fatalf("compatible singleton blocked on absent sibling: name=%s err=%v", name, err) ++ } ++ } ++} ++ ++func TestV2OnlyCustomizedEngineIntentFailsBeforeAnyWrite(t *testing.T) { ++ for _, raw := range []string{"v1-data-engine: 'true'", "v2-data-engine: 'false'", "v1-data-engine: invalid-bool"} { ++ lh, kube := modeStateClients() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, types.DataEngineModeV2Only) ++ configMap := &corev1.ConfigMap{ ++ ObjectMeta: metav1.ObjectMeta{Name: types.DefaultDefaultSettingConfigMapName, Namespace: "longhorn-system", UID: "config-uid", ResourceVersion: "1"}, ++ Data: map[string]string{types.DefaultSettingYAMLFileName: raw}, ++ } ++ if err := factories.KubeNamespaceFilteredInformerFactory.Core().V1().ConfigMaps().Informer().GetIndexer().Add(configMap); err != nil { ++ t.Fatal(err) ++ } ++ if err := store.UpdateCustomizedSettings(map[types.SettingName]string{types.SettingNameDefaultEngineImage: "must-not-write"}); err == nil { ++ t.Fatalf("invalid raw engine intent cleared customized settings: %q", raw) ++ } ++ for _, action := range lh.Actions() { ++ if action.GetVerb() != "get" && action.GetVerb() != "list" { ++ t.Fatalf("invalid raw engine intent permitted mutation before validation: %q %s", raw, action.GetVerb()) ++ } ++ } ++ } ++} +diff --git a/datastore/data_engine_mode_test.go b/datastore/data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..4bcdf7ab890818d35c57b983444470761d8c8775 +--- /dev/null ++++ b/datastore/data_engine_mode_test.go +@@ -0,0 +1,327 @@ ++package datastore ++ ++import ( ++ "context" ++ "errors" ++ "testing" ++ ++ "k8s.io/apimachinery/pkg/runtime" ++ ++ corev1 "k8s.io/api/core/v1" ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ k8stypes "k8s.io/apimachinery/pkg/types" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ k8stesting "k8s.io/client-go/testing" ++ ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func modeStateClients(objects ...runtime.Object) (*lhfake.Clientset, *kubefake.Clientset) { ++ settings := []runtime.Object{} ++ for name, value := range map[types.SettingName]string{types.SettingNameV1DataEngine: "false", types.SettingNameV2DataEngine: "true"} { ++ settings = append(settings, &longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: "longhorn-system", UID: k8stypes.UID(name), ResourceVersion: "1"}, Value: value}) ++ } ++ return lhfake.NewClientset(append(settings, objects...)...), kubefake.NewClientset() ++} ++ ++func TestV2OnlyStateChecksUncachedStorageWithoutNodes(t *testing.T) { ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll, "unrecognized"} { ++ for _, kind := range []string{"volume", "engine", "replica", "instance-manager", "deleting-volume"} { ++ t.Run(kind+"/"+string(engine), func(t *testing.T) { ++ metadata := metav1.ObjectMeta{Name: "retained-without-node", Namespace: "longhorn-system"} ++ var resource runtime.Object ++ switch kind { ++ case "volume", "deleting-volume": ++ if kind == "deleting-volume" { ++ now := metav1.Now() ++ metadata.DeletionTimestamp = &now ++ } ++ resource = &longhorn.Volume{ObjectMeta: metadata, Spec: longhorn.VolumeSpec{DataEngine: engine}} ++ case "engine": ++ resource = &longhorn.Engine{ObjectMeta: metadata, Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: engine}}} ++ case "replica": ++ resource = &longhorn.Replica{ObjectMeta: metadata, Spec: longhorn.ReplicaSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: engine}}} ++ case "instance-manager": ++ resource = &longhorn.InstanceManager{ObjectMeta: metadata, Spec: longhorn.InstanceManagerSpec{DataEngine: engine}} ++ } ++ lh, kube := modeStateClients(resource) ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err == nil { ++ t.Fatal("uncached legacy, unknown or deleting storage cleared V2-only startup") ++ } ++ }) ++ } ++ } ++} ++ ++func TestV2OnlyStateRequiresExplicitSettings(t *testing.T) { ++ for _, name := range []types.SettingName{types.SettingNameV1DataEngine, types.SettingNameV2DataEngine} { ++ for _, value := range []string{"missing", "", "unexpected", map[types.SettingName]string{types.SettingNameV1DataEngine: "true", types.SettingNameV2DataEngine: "false"}[name]} { ++ t.Run(string(name)+"/"+value, func(t *testing.T) { ++ lh, kube := modeStateClients() ++ lh.PrependReactor("get", "settings", func(action k8stesting.Action) (bool, runtime.Object, error) { ++ if action.(k8stesting.GetAction).GetName() != string(name) { ++ return false, nil, nil ++ } ++ if value == "missing" { ++ return true, nil, errors.New("not found") ++ } ++ return true, &longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: "longhorn-system", UID: k8stypes.UID(name), ResourceVersion: "1"}, Value: value}, nil ++ }) ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err == nil { ++ t.Fatal("absent or contradictory settings cleared V2-only startup") ++ } ++ }) ++ } ++ } ++} ++ ++func TestV2OnlyStateRequiresCompleteSuccessfulInventories(t *testing.T) { ++ for _, resource := range []string{"volumes", "engines", "replicas", "instancemanagers", "pods"} { ++ for _, partial := range []bool{false, true} { ++ t.Run(resource+map[bool]string{false: "/read-error", true: "/incomplete-page"}[partial], func(t *testing.T) { ++ lh, kube := modeStateClients() ++ read := func(k8stesting.Action) (bool, runtime.Object, error) { ++ if !partial { ++ return true, nil, errors.New("inventory unavailable") ++ } ++ metadata := metav1.ListMeta{Continue: "another-page"} ++ switch resource { ++ case "volumes": ++ return true, &longhorn.VolumeList{ListMeta: metadata}, nil ++ case "engines": ++ return true, &longhorn.EngineList{ListMeta: metadata}, nil ++ case "replicas": ++ return true, &longhorn.ReplicaList{ListMeta: metadata}, nil ++ case "instancemanagers": ++ return true, &longhorn.InstanceManagerList{ListMeta: metadata}, nil ++ default: ++ return true, &corev1.PodList{ListMeta: metadata}, nil ++ } ++ } ++ if resource == "pods" { ++ kube.PrependReactor("list", resource, read) ++ } else { ++ lh.PrependReactor("list", resource, read) ++ } ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err == nil { ++ t.Fatal("failed or incomplete uncached inventory cleared V2-only startup") ++ } ++ }) ++ } ++ } ++} ++ ++func TestV2OnlyStateRejectsOrphanInstanceManagerRuntime(t *testing.T) { ++ for _, labels := range []map[string]string{ ++ nil, ++ {types.GetLonghornLabelComponentKey(): types.LonghornLabelInstanceManager}, ++ {types.GetLonghornLabelComponentKey(): types.LonghornLabelInstanceManager, types.GetLonghornLabelKey(types.LonghornLabelDataEngine): "v1"}, ++ {types.GetLonghornLabelComponentKey(): types.LonghornLabelInstanceManager, types.GetLonghornLabelKey(types.LonghornLabelDataEngine): "unrecognized"}, ++ } { ++ lh, kube := modeStateClients() ++ _, err := kube.CoreV1().Pods("longhorn-system").Create(context.Background(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "instance-manager-orphan", Namespace: "longhorn-system", Labels: labels}}, metav1.CreateOptions{}) ++ if err != nil { ++ t.Fatal(err) ++ } ++ if err = ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err == nil { ++ t.Fatal("orphan runtime without a Node or instance-manager CR cleared V2-only startup") ++ } ++ } ++} ++ ++func TestV1CapableStateRetainsDefaultBehavior(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable} { ++ if err := ValidateDataEngineModeState(context.Background(), mode, "", nil, nil); err != nil { ++ t.Fatal("default mode acquired a new storage-state requirement") ++ } ++ } ++ if err := ValidateDataEngineModeState(context.Background(), "invalid", "", nil, nil); err == nil { ++ t.Fatal("unknown installation mode cleared startup") ++ } ++} ++ ++func TestV2OnlyStateAcceptsEmptyQualifiedInstallation(t *testing.T) { ++ lh, kube := modeStateClients() ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err != nil { ++ t.Fatal(err) ++ } ++} ++ ++func TestV2OnlyStateAcceptsExplicitV2StorageAndRuntime(t *testing.T) { ++ metadata := metav1.ObjectMeta{Name: "instance-manager-v2", Namespace: "longhorn-system"} ++ manager := modeStateV2Manager() ++ lh, kube := modeStateClients( ++ &longhorn.Volume{ObjectMeta: metadata, Spec: longhorn.VolumeSpec{DataEngine: longhorn.DataEngineTypeV2}}, ++ &longhorn.Engine{ObjectMeta: metadata, Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: longhorn.DataEngineTypeV2}}}, ++ &longhorn.Replica{ObjectMeta: metadata, Spec: longhorn.ReplicaSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: longhorn.DataEngineTypeV2}}}, ++ manager, ++ ) ++ if _, err := kube.CoreV1().Pods(metadata.Namespace).Create(context.Background(), modeStateV2Pod(manager), metav1.CreateOptions{}); err != nil { ++ t.Fatal(err) ++ } ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err != nil { ++ t.Fatal(err) ++ } ++} ++ ++func modeStateV2Manager() *longhorn.InstanceManager { ++ return &longhorn.InstanceManager{ObjectMeta: metav1.ObjectMeta{Name: "instance-manager-v2", Namespace: "longhorn-system", UID: "manager-uid"}, Spec: longhorn.InstanceManagerSpec{DataEngine: longhorn.DataEngineTypeV2, NodeID: "node", Image: "longhornio/longhorn-instance-manager:tested"}} ++} ++ ++// Source-derived native launch profile from createInstanceManagerPodSpec. ++func modeStateV2Pod(manager *longhorn.InstanceManager) *corev1.Pod { ++ return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: manager.Name, Namespace: manager.Namespace, UID: "pod-uid", OwnerReferences: GetOwnerReferencesForInstanceManager(manager), Labels: types.GetInstanceManagerLabels(manager.Spec.NodeID, manager.Spec.Image, longhorn.InstanceManagerTypeAllInOne, longhorn.DataEngineTypeV2)}, Spec: corev1.PodSpec{ ++ NodeName: manager.Spec.NodeID, ++ Containers: []corev1.Container{{Name: "instance-manager", Image: manager.Spec.Image, Args: []string{ ++ "start-spdk-tgt", "--spdk-log", "", "--spdk-cpumask", "0x3", "--spdk-core-number", "2", "--spdk-memory-size", "2048", ++ "--longhorn-control-path", types.DefaultControlPath, "--enable-spdk", "--debug", "daemon", "--spdk-enabled", "--listen", "0.0.0.0:8500", ++ }, Env: []corev1.EnvVar{{Name: types.EnvDataEngine, Value: "v2"}}}}, ++ }} ++} ++ ++func TestV2OnlyStateRejectsContradictoryObservedProcesses(t *testing.T) { ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll, "unrecognized"} { ++ for _, kind := range []string{"engine", "replica", "frontend", "shard", "shard-group"} { ++ t.Run(kind+"/"+string(engine), func(t *testing.T) { ++ manager := modeStateV2Manager() ++ processes := map[string]longhorn.InstanceProcess{"observed-process": {Spec: longhorn.InstanceProcessSpec{Name: "observed-process", DataEngine: engine}}} ++ switch kind { ++ case "engine": ++ manager.Status.InstanceEngines = processes ++ case "replica": ++ manager.Status.InstanceReplicas = processes ++ case "frontend": ++ manager.Status.InstanceEngineFrontends = processes ++ case "shard": ++ manager.Status.InstanceShards = processes ++ case "shard-group": ++ manager.Status.InstanceShardGroups = processes ++ } ++ lh, kube := modeStateClients(manager) ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err == nil { ++ t.Fatal("requested V2 spec concealed a non-V2 observed process") ++ } ++ }) ++ } ++ } ++} ++ ++func TestV2OnlyStateRejectsRelabelledOrUnboundRuntime(t *testing.T) { ++ for name, change := range map[string]func(*corev1.Pod){ ++ "V1 launch": func(p *corev1.Pod) { ++ p.Spec.Containers[0].Args = []string{"instance-manager", "--debug", "daemon", "--listen", ":8500"} ++ }, ++ "command override": func(p *corev1.Pod) { p.Spec.Containers[0].Command = []string{"/bin/sh", "-c"} }, ++ "foreign image": func(p *corev1.Pod) { p.Spec.Containers[0].Image = "another-image" }, ++ "foreign owner": func(p *corev1.Pod) { p.OwnerReferences[0].UID = "another-manager" }, ++ "no owner": func(p *corev1.Pod) { p.OwnerReferences = nil }, ++ "another node": func(p *corev1.Pod) { p.Spec.NodeName = "another-node" }, ++ "another mode": func(p *corev1.Pod) { p.Spec.Containers[0].Env[0].Value = "v1" }, ++ "unresolved environment": func(p *corev1.Pod) { ++ p.Spec.Containers[0].Env[0].ValueFrom = &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}} ++ }, ++ "missing daemon SPDK enablement": func(p *corev1.Pod) { p.Spec.Containers[0].Args[14] = "--debug" }, ++ "flag hidden in option value": func(p *corev1.Pod) { ++ p.Spec.Containers[0].Args[4] = "--enable-spdk" ++ p.Spec.Containers[0].Args = append(p.Spec.Containers[0].Args[:11], p.Spec.Containers[0].Args[12:]...) ++ }, ++ "unknown argument": func(p *corev1.Pod) { ++ p.Spec.Containers[0].Args = append(p.Spec.Containers[0].Args, "--unknown-launch-mode") ++ }, ++ } { ++ t.Run(name, func(t *testing.T) { ++ manager := modeStateV2Manager() ++ pod := modeStateV2Pod(manager) ++ change(pod) ++ lh, kube := modeStateClients(manager) ++ if _, err := kube.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{}); err != nil { ++ t.Fatal(err) ++ } ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err == nil { ++ t.Fatal("V2 labels concealed contradictory or unresolved runtime") ++ } ++ }) ++ } ++ manager := modeStateV2Manager() ++ pod := modeStateV2Pod(manager) ++ lh, kube := modeStateClients() ++ if _, err := kube.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{}); err != nil { ++ t.Fatal(err) ++ } ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err == nil { ++ t.Fatal("an orphan carrying V2 labels cleared runtime inventory") ++ } ++} ++ ++func TestV2OnlyStateAcceptsNativeOptionalSPDKProfiles(t *testing.T) { ++ for _, runtimePinning := range []bool{false, true} { ++ manager := modeStateV2Manager() ++ pod := modeStateV2Pod(manager) ++ args := pod.Spec.Containers[0].Args ++ if runtimePinning { ++ args[4] = "" ++ } ++ withPools := append([]string(nil), args[:13]...) ++ withPools = append(withPools, "--spdk-iobuf-large-pool-size", "1048576", "--spdk-iobuf-small-pool-size", "2097152") ++ withPools = append(withPools, args[13:]...) ++ pod.Spec.Containers[0].Args = append(withPools, "--spdk-interrupt-mode", "--enable-irq-affinity", "--enable-workqueue-affinity", "--enable-rps", "--spdk-no-hugepage") ++ lh, kube := modeStateClients(manager) ++ if _, err := kube.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{}); err != nil { ++ t.Fatal(err) ++ } ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err != nil { ++ t.Fatal(err) ++ } ++ } ++} ++ ++func TestV2OnlyStateRejectsSettingsThatChangeDuringInventory(t *testing.T) { ++ for _, replace := range []bool{false, true} { ++ lh, kube := modeStateClients() ++ reads := 0 ++ lh.PrependReactor("get", "settings", func(action k8stesting.Action) (bool, runtime.Object, error) { ++ if action.(k8stesting.GetAction).GetName() != string(types.SettingNameV1DataEngine) { ++ return false, nil, nil ++ } ++ reads++ ++ if reads == 1 { ++ return false, nil, nil ++ } ++ metadata := metav1.ObjectMeta{Name: string(types.SettingNameV1DataEngine), Namespace: "longhorn-system", UID: k8stypes.UID(types.SettingNameV1DataEngine), ResourceVersion: "1"} ++ if replace { ++ metadata.UID = "replaced-setting" ++ } else { ++ metadata.ResourceVersion = "2" ++ } ++ return true, &longhorn.Setting{ObjectMeta: metadata, Value: "false"}, nil ++ }) ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, kube); err == nil { ++ t.Fatal("changed or replaced Settings cleared the inventory observation") ++ } ++ } ++} ++ ++func TestV2OnlyStateRefusesMissingInputsAndCanceledReads(t *testing.T) { ++ lh, kube := modeStateClients() ++ for _, ctx := range []context.Context{nil, func() context.Context { ++ canceled, cancel := context.WithCancel(context.Background()) ++ cancel() ++ return canceled ++ }()} { ++ if err := ValidateDataEngineModeState(ctx, types.DataEngineModeV2Only, "longhorn-system", lh, kube); err == nil { ++ t.Fatal("missing or canceled context cleared startup") ++ } ++ } ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "", lh, kube); err == nil { ++ t.Fatal("empty namespace cleared startup") ++ } ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", nil, kube); err == nil { ++ t.Fatal("missing Longhorn client cleared startup") ++ } ++ if err := ValidateDataEngineModeState(context.Background(), types.DataEngineModeV2Only, "longhorn-system", lh, nil); err == nil { ++ t.Fatal("missing Kubernetes client cleared startup") ++ } ++} +diff --git a/datastore/datastore.go b/datastore/datastore.go +index eb7c858a0153e6150fd810037c6ff9692560eabe..c7d3e309244dab8b73366275fec3179b64d9e8f2 100644 +--- a/datastore/datastore.go ++++ b/datastore/datastore.go +@@ -1,6 +1,7 @@ + package datastore + + import ( ++ "context" + "errors" + "time" + +@@ -43,7 +44,8 @@ var ( + + // DataStore object + type DataStore struct { +- namespace string ++ namespace string ++ dataEngineMode types.DataEngineMode + + cacheSyncs []cache.InformerSynced + +@@ -144,11 +146,19 @@ type DataStore struct { + extensionsClient apiextensionsclientset.Interface + } + ++func (s *DataStore) DataEngineMode() types.DataEngineMode { ++ return s.dataEngineMode ++} ++ ++func (s *DataStore) ValidateDataEngineModeState(ctx context.Context) error { ++ return ValidateDataEngineModeState(ctx, s.dataEngineMode, s.namespace, s.lhClient, s.kubeClient) ++} ++ + // NewDataStoreForGlobal creates a DataStore whose Pod informer watches + // cluster-wide. It is used by the longhorn-global-manager process and by the + // uninstall and system-rollout entry points. +-func NewDataStoreForGlobal(namespace string, lhClient lhclientset.Interface, kubeClient clientset.Interface, extensionsClient apiextensionsclientset.Interface, informerFactories *util.InformerFactories) *DataStore { +- ds := newDataStoreCommon(namespace, lhClient, kubeClient, extensionsClient, informerFactories) ++func NewDataStoreForGlobal(namespace string, lhClient lhclientset.Interface, kubeClient clientset.Interface, extensionsClient apiextensionsclientset.Interface, informerFactories *util.InformerFactories, mode types.DataEngineMode) *DataStore { ++ ds := newDataStoreCommon(namespace, lhClient, kubeClient, extensionsClient, informerFactories, mode) + pi := informerFactories.KubeInformerFactory.Core().V1().Pods() + attachPodInformer(ds, pi.Lister(), pi.Informer()) + return ds +@@ -157,8 +167,8 @@ func NewDataStoreForGlobal(namespace string, lhClient lhclientset.Interface, kub + // NewDataStoreForNodeLocal creates a DataStore whose Pod informer is + // namespace-filtered (longhorn-system), which suffices for every Pod consumer + // in the longhorn-manager DaemonSet. +-func NewDataStoreForNodeLocal(namespace string, lhClient lhclientset.Interface, kubeClient clientset.Interface, extensionsClient apiextensionsclientset.Interface, informerFactories *util.InformerFactories) *DataStore { +- ds := newDataStoreCommon(namespace, lhClient, kubeClient, extensionsClient, informerFactories) ++func NewDataStoreForNodeLocal(namespace string, lhClient lhclientset.Interface, kubeClient clientset.Interface, extensionsClient apiextensionsclientset.Interface, informerFactories *util.InformerFactories, mode types.DataEngineMode) *DataStore { ++ ds := newDataStoreCommon(namespace, lhClient, kubeClient, extensionsClient, informerFactories, mode) + pi := informerFactories.KubeNamespaceFilteredInformerFactory.Core().V1().Pods() + attachPodInformer(ds, pi.Lister(), pi.Informer()) + return ds +@@ -174,7 +184,7 @@ func attachPodInformer(ds *DataStore, podLister corelisters.PodLister, podShared + + // newDataStoreCommon builds the DataStore with every informer except + // PodInformer, which the public wrappers attach afterward. +-func newDataStoreCommon(namespace string, lhClient lhclientset.Interface, kubeClient clientset.Interface, extensionsClient apiextensionsclientset.Interface, informerFactories *util.InformerFactories) *DataStore { ++func newDataStoreCommon(namespace string, lhClient lhclientset.Interface, kubeClient clientset.Interface, extensionsClient apiextensionsclientset.Interface, informerFactories *util.InformerFactories, mode types.DataEngineMode) *DataStore { + cacheSyncs := []cache.InformerSynced{} + + // Longhorn Informers +@@ -272,7 +282,8 @@ func newDataStoreCommon(namespace string, lhClient lhclientset.Interface, kubeCl + cacheSyncs = append(cacheSyncs, leaseInformer.Informer().HasSynced) + + return &DataStore{ +- namespace: namespace, ++ namespace: namespace, ++ dataEngineMode: mode, + + cacheSyncs: cacheSyncs, + +diff --git a/datastore/kubernetes_test.go b/datastore/kubernetes_test.go +index 36a6996c42028229e8ffb80339c3277ecda30522..da9a1c4e939fe44326b84aa23613f7814611ec2f 100644 +--- a/datastore/kubernetes_test.go ++++ b/datastore/kubernetes_test.go +@@ -19,6 +19,7 @@ import ( + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++ longhorntypes "github.com/longhorn/longhorn-manager/types" + ) + + func TestNewPVCManifestForVolume(t *testing.T) { +@@ -130,7 +131,7 @@ func TestLeaseInformerIsNamespaceScoped(t *testing.T) { + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint: staticcheck + + informerFactories := util.NewInformerFactories(longhornNamespace, kubeClient, lhClient, 0) +- ds := NewDataStoreForGlobal(longhornNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := NewDataStoreForGlobal(longhornNamespace, lhClient, kubeClient, extensionsClient, informerFactories, longhorntypes.DataEngineModeV1Capable) + + stopCh := make(chan struct{}) + defer close(stopCh) +diff --git a/datastore/longhorn.go b/datastore/longhorn.go +index 4f3a96efe03b9069e0f93389e5c00ed4570abaf7..942fa87f5b78f5fb941cb0635ebaeb37f9660951 100644 +--- a/datastore/longhorn.go ++++ b/datastore/longhorn.go +@@ -60,6 +60,16 @@ func (s *DataStore) UpdateCustomizedSettings(defaultImages map[types.SettingName + if err != nil { + return err + } ++ if err := s.validateDataEngineModeCustomizedSettings(defaultSettingCM); err != nil { ++ return err ++ } ++ if s.dataEngineMode == types.DataEngineModeV2Only { ++ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) ++ defer cancel() ++ if err := s.ValidateDataEngineModeState(ctx); err != nil { ++ return err ++ } ++ } + + customizedDefaultSettings, err := types.GetCustomizedDefaultSettings(defaultSettingCM) + if err != nil { +@@ -105,6 +115,9 @@ func (s *DataStore) UpdateCustomizedSettings(defaultImages map[types.SettingName + + func (s *DataStore) createNonExistingSettingCRsWithDefaultSetting(configMapResourceVersion string) error { + for _, sName := range types.SettingNameList { ++ if s.dataEngineMode == types.DataEngineModeV2Only && isDataEngineSetting(sName) { ++ continue ++ } + if _, err := s.GetSettingExactRO(sName); err != nil && apierrors.IsNotFound(err) { + definition, ok := types.GetSettingDefinition(sName) + if !ok { +@@ -132,6 +145,9 @@ func (s *DataStore) filterCustomizedDefaultSettings(customizedDefaultSettings ma + availableCustomizedDefaultSettings := make(map[string]string) + + for name, value := range customizedDefaultSettings { ++ if s.dataEngineMode == types.DataEngineModeV2Only && isDataEngineSetting(types.SettingName(name)) { ++ continue ++ } + if !s.shouldApplyCustomizedSettingValue(types.SettingName(name), value, defaultSettingCMResourceVersion) { + continue + } +@@ -518,6 +534,9 @@ func convertDataEngineValuesToJSONString(values map[longhorn.DataEngineType]any) + + func (s *DataStore) syncSettingCRsWithCustomizedDefaultSettings(customizedDefaultSettings map[string]string, defaultSettingCMResourceVersion string) error { + for _, sName := range types.SettingNameList { ++ if s.dataEngineMode == types.DataEngineModeV2Only && isDataEngineSetting(sName) { ++ continue ++ } + definition, ok := types.GetSettingDefinition(sName) + if !ok { + return fmt.Errorf("BUG: setting %v is not defined", sName) +@@ -620,6 +639,9 @@ func (s *DataStore) ValidateSetting(name, value string) (err error) { + if err := types.ValidateSetting(name, value); err != nil { + return err + } ++ if err := s.validateDataEngineModeSetting(types.SettingName(name), value); err != nil { ++ return err ++ } + + switch types.SettingName(name) { + case types.SettingNamePriorityClass: +@@ -7526,8 +7548,11 @@ func IsSupportedVolumeSize(dataEngine longhorn.DataEngineType, fsType string, vo + return true + } + +-// IsDataEngineEnabled returns true if the given dataEngine is enabled ++// IsDataEngineEnabled applies the process capability before mutable enablement. + func (s *DataStore) IsDataEngineEnabled(dataEngine longhorn.DataEngineType) (bool, error) { ++ if !s.dataEngineMode.AllowsDataEngine(dataEngine) { ++ return false, nil ++ } + dataEngineSetting := types.SettingNameV1DataEngine + if types.IsDataEngineV2(dataEngine) { + dataEngineSetting = types.SettingNameV2DataEngine +@@ -7535,7 +7560,7 @@ func (s *DataStore) IsDataEngineEnabled(dataEngine longhorn.DataEngineType) (boo + + enabled, err := s.GetSettingAsBool(dataEngineSetting) + if err != nil { +- return true, errors.Wrapf(err, "failed to get %v setting", dataEngineSetting) ++ return s.dataEngineMode != types.DataEngineModeV2Only, errors.Wrapf(err, "failed to get %v setting", dataEngineSetting) + } + + return enabled, nil +@@ -7545,18 +7570,20 @@ func (s *DataStore) GetDataEngines() map[longhorn.DataEngineType]struct{} { + dataEngines := map[longhorn.DataEngineType]struct{}{} + + for _, setting := range []types.SettingName{types.SettingNameV1DataEngine, types.SettingNameV2DataEngine} { ++ dataEngine := longhorn.DataEngineTypeV1 ++ if setting == types.SettingNameV2DataEngine { ++ dataEngine = longhorn.DataEngineTypeV2 ++ } ++ if !s.dataEngineMode.AllowsDataEngine(dataEngine) { ++ continue ++ } + dataEngineEnabled, err := s.GetSettingAsBool(setting) + if err != nil { + logrus.WithError(err).Warnf("Failed to get setting %v", setting) + continue + } + if dataEngineEnabled { +- switch setting { +- case types.SettingNameV1DataEngine: +- dataEngines[longhorn.DataEngineTypeV1] = struct{}{} +- case types.SettingNameV2DataEngine: +- dataEngines[longhorn.DataEngineTypeV2] = struct{}{} +- } ++ dataEngines[dataEngine] = struct{}{} + } + } + +diff --git a/datastore/longhorn_test.go b/datastore/longhorn_test.go +index 8ab4f3c448b76ebfe07fca2c35477b606cc6ecc6..4b2b9d28e326ba3246ea806076e7ce4fdb7cdfff 100644 +--- a/datastore/longhorn_test.go ++++ b/datastore/longhorn_test.go +@@ -225,7 +225,7 @@ func TestGetVolumeCurrentEngineFrontendReturnsErrorWhenMissing(t *testing.T) { + kubeClient := fake.NewSimpleClientset() // nolint: staticcheck + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint: staticcheck + informerFactories := util.NewInformerFactories(testNamespace, kubeClient, lhClient, 0) +- ds := NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + stopCh := make(chan struct{}) + defer close(stopCh) +@@ -363,7 +363,7 @@ func TestValidateSettingBlocksV2IMUpgradeStartTimeWhenActiveIMUExistsWithoutIMUC + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint:staticcheck + informerFactories := util.NewInformerFactories(testNamespace, kubeClient, lhClient, 0) + +- ds := NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + imuIndexer := informerFactories.LhInformerFactory.Longhorn().V1beta2().InstanceManagerUpgrades().Informer().GetIndexer() + + imu := &longhorn.InstanceManagerUpgrade{ +@@ -396,7 +396,7 @@ func TestValidateSettingRejectsUnsupportedV2InstanceManagerLiveUpgrade(t *testin + kubeClient := fake.NewSimpleClientset() // nolint:staticcheck + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint:staticcheck + informerFactories := util.NewInformerFactories(testNamespace, kubeClient, lhClient, 0) +- ds := NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + settingIndexer := informerFactories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer() + + currentVersion := &longhorn.Setting{ +@@ -427,7 +427,7 @@ func TestGetSettingValueExistedByDataEngineAllowsEmptyValue(t *testing.T) { + kubeClient := fake.NewSimpleClientset() // nolint:staticcheck + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint:staticcheck + informerFactories := util.NewInformerFactories(testNamespace, kubeClient, lhClient, 0) +- ds := NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + settingIndexer := informerFactories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer() + + setting := &longhorn.Setting{ +diff --git a/manager/data_engine_mode_test.go b/manager/data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..93ea5c5c4af1d82fdc32a0f70fdcd352dff8cd8a +--- /dev/null ++++ b/manager/data_engine_mode_test.go +@@ -0,0 +1,55 @@ ++package manager ++ ++import ( ++ "strings" ++ "testing" ++ ++ "k8s.io/apimachinery/pkg/runtime" ++ ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ k8stesting "k8s.io/client-go/testing" ++ ++ "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" ++ "github.com/longhorn/longhorn-manager/util" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func TestCreateResolvesCapabilityBeforeBackingImageOrStorageWrites(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2} { ++ lh, kube := lhfake.NewClientset(), kubefake.NewClientset() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := datastore.NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, mode) ++ manager := &VolumeManager{ds: store} ++ lh.PrependReactor("create", "volumes", func(action k8stesting.Action) (bool, runtime.Object, error) { ++ volume := action.(k8stesting.CreateAction).GetObject().(*longhorn.Volume).DeepCopy() ++ volume.Namespace, volume.UID, volume.ResourceVersion = "longhorn-system", "created-volume", "1" ++ if err := factories.LhInformerFactory.Longhorn().V1beta2().Volumes().Informer().GetIndexer().Add(volume); err != nil { ++ t.Fatal(err) ++ } ++ return true, volume, nil ++ }) ++ spec := &longhorn.VolumeSpec{DataEngine: engine, Size: 4 << 20} ++ if mode == types.DataEngineModeV2Only && engine == longhorn.DataEngineTypeV1 { ++ spec.BackingImage = "must-not-be-restored" ++ } ++ created, err := manager.Create("new-volume", spec, nil) ++ if mode == types.DataEngineModeV2Only && engine == longhorn.DataEngineTypeV1 { ++ if err == nil || !strings.Contains(err.Error(), "capability") || len(lh.Actions()) != 0 { ++ t.Fatalf("V1 creation reached backing-image or storage processing: err=%v actions=%v", err, lh.Actions()) ++ } ++ continue ++ } ++ want := engine ++ if mode == types.DataEngineModeV2Only { ++ want = longhorn.DataEngineTypeV2 ++ } ++ if err != nil || created.Spec.DataEngine != want || spec.DataEngine != engine { ++ t.Fatalf("mode=%s input=%q: returned=%v inputChanged=%t err=%v", mode, engine, created, spec.DataEngine != engine, err) ++ } ++ } ++ } ++} +diff --git a/manager/volume.go b/manager/volume.go +index 59138650455e98da42f74052a7b52ed0fe859869..950dc29731a2d06b760e219b7d91f1d7caf97833 100644 +--- a/manager/volume.go ++++ b/manager/volume.go +@@ -51,6 +51,10 @@ func (m *VolumeManager) GetCurrentNodeID() string { + return m.currentNodeID + } + ++func (m *VolumeManager) GetDataEngineMode() types.DataEngineMode { ++ return m.ds.DataEngineMode() ++} ++ + func (m *VolumeManager) Node2APIAddress(nodeID string) (string, error) { + nodeIPMap, err := m.ds.GetManagerNodeIPMap() + if err != nil { +@@ -154,6 +158,22 @@ func (m *VolumeManager) Create(name string, spec *longhorn.VolumeSpec, recurring + logrus.Errorf("manager: unable to create volume %v: %+v: %v", name, spec, err) + } + }() ++ if spec == nil || m.ds == nil { ++ return nil, fmt.Errorf("volume creation requires a specification and datastore capability") ++ } ++ mode, err := types.ParseDataEngineMode(string(m.ds.DataEngineMode())) ++ if err != nil { ++ return nil, err ++ } ++ if mode == types.DataEngineModeV2Only { ++ spec = spec.DeepCopy() ++ if spec.DataEngine == "" { ++ spec.DataEngine = longhorn.DataEngineTypeV2 ++ } ++ if !mode.AllowsDataEngine(spec.DataEngine) { ++ return nil, fmt.Errorf("data engine %q is outside the v2-only installation capability", spec.DataEngine) ++ } ++ } + + labels := map[string]string{} + for _, job := range recurringJobSelector { +diff --git a/manager/volume_test.go b/manager/volume_test.go +index 7365b2e0ae43b3ad248938863efb5c07b0c4b20f..abedce7cae5a27a130d30504eb39d24fbc88f47e 100644 +--- a/manager/volume_test.go ++++ b/manager/volume_test.go +@@ -15,6 +15,7 @@ import ( + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++ longhorntypes "github.com/longhorn/longhorn-manager/types" + ) + + // TestCreatePreservesSpecFields guards the Create spec rebuild: fields plumbed +@@ -26,7 +27,7 @@ func TestCreatePreservesSpecFields(t *testing.T) { + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint: staticcheck + informerFactories := util.NewInformerFactories("default", kubeClient, lhClient, controller.NoResyncPeriodFunc()) + +- ds := datastore.NewDataStoreForGlobal("default", lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal("default", lhClient, kubeClient, extensionsClient, informerFactories, longhorntypes.DataEngineModeV1Capable) + + // Start the informers so the datastore's create-then-verify lister sees + // objects created through the fake clientset. +@@ -100,7 +101,7 @@ func TestExpandDefersAffectedV2VolumeDuringLiveUpgrade(t *testing.T) { + t.Fatalf("failed to create instance manager upgrade: %v", err) + } + +- ds := datastore.NewDataStoreForGlobal("default", lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal("default", lhClient, kubeClient, extensionsClient, informerFactories, longhorntypes.DataEngineModeV1Capable) + stop := make(chan struct{}) + defer close(stop) + informerFactories.LhInformerFactory.Start(stop) +@@ -160,7 +161,7 @@ func TestV2VolumeExpansionIgnoresTerminalLiveUpgrade(t *testing.T) { + } + } + +- ds := datastore.NewDataStoreForGlobal("default", lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal("default", lhClient, kubeClient, extensionsClient, informerFactories, longhorntypes.DataEngineModeV1Capable) + stop := make(chan struct{}) + defer close(stop) + informerFactories.LhInformerFactory.Start(stop) +diff --git a/scheduler/replica_scheduler_test.go b/scheduler/replica_scheduler_test.go +index 4f8b463a5ef1b4f147fa9b3a1e01b9e36f0dac20..4f318226bdbea439177eba03ca735e329d950f40 100644 +--- a/scheduler/replica_scheduler_test.go ++++ b/scheduler/replica_scheduler_test.go +@@ -70,7 +70,7 @@ func newReplicaScheduler(lhClient *lhfake.Clientset, kubeClient *fake.Clientset, + informerFactories *util.InformerFactories) *ReplicaScheduler { + fmt.Printf("testing NewReplicaScheduler\n") + +- ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + rcs := NewReplicaScheduler(ds) + rcs.nowHandler = getTestNow +diff --git a/scheduler/shard_scheduler_test.go b/scheduler/shard_scheduler_test.go +index 4ea897d1a4ec2cb88914c8c2c169c89843e8a2ce..9b9afe0c8cffca0a5ae471623ca796abeea8b861 100644 +--- a/scheduler/shard_scheduler_test.go ++++ b/scheduler/shard_scheduler_test.go +@@ -68,8 +68,8 @@ func (s *TestSuite) TestScheduleShardTopologyRequirement(c *C) { + addSetting(types.SettingNameStorageMinimalAvailablePercentage, "10") + + ss := &ShardScheduler{ +- ds: datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories), +- rcs: NewReplicaScheduler(datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories)), ++ ds: datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable), ++ rcs: NewReplicaScheduler(datastore.NewDataStoreForGlobal(TestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable)), + } + + nodes := map[string]*longhorn.Node{ +diff --git a/types/data_engine_mode.go b/types/data_engine_mode.go +new file mode 100644 +index 0000000000000000000000000000000000000000..0cc09767fa0ef3b17b73900787a555d446bc0390 +--- /dev/null ++++ b/types/data_engine_mode.go +@@ -0,0 +1,43 @@ ++package types ++ ++import ( ++ "fmt" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++// DataEngineMode is the process's installation capability, not a mutable engine ++// setting. A supported engine still needs its existing enablement setting. ++type DataEngineMode string ++ ++const ( ++ DataEngineModeV1Capable DataEngineMode = "v1-capable" ++ DataEngineModeV2Only DataEngineMode = "v2-only" ++) ++ ++func ParseDataEngineMode(value string) (DataEngineMode, error) { ++ switch mode := DataEngineMode(value); mode { ++ case "", DataEngineModeV1Capable: ++ return DataEngineModeV1Capable, nil ++ case DataEngineModeV2Only: ++ return mode, nil ++ default: ++ return "", fmt.Errorf("unsupported data engine mode %q: expected v1-capable or v2-only", value) ++ } ++} ++ ++func (mode DataEngineMode) RequiresISCSI() bool { ++ return mode != DataEngineModeV2Only ++} ++ ++func (mode DataEngineMode) AllowsDataEngine(engine longhorn.DataEngineType) bool { ++ switch mode { ++ case "", DataEngineModeV1Capable: ++ // Preserve legacy empty V1 fields and combined instance managers. ++ return engine == "" || engine == longhorn.DataEngineTypeV1 || engine == longhorn.DataEngineTypeV2 || engine == longhorn.DataEngineTypeAll ++ case DataEngineModeV2Only: ++ return engine == longhorn.DataEngineTypeV2 ++ default: ++ return false ++ } ++} +diff --git a/types/data_engine_mode_test.go b/types/data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..e74368016d3fbacabae01ce200f3b7a0db134b66 +--- /dev/null ++++ b/types/data_engine_mode_test.go +@@ -0,0 +1,59 @@ ++package types ++ ++import ( ++ "testing" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++) ++ ++func TestParseDataEngineMode(t *testing.T) { ++ for _, test := range []struct { ++ input string ++ want DataEngineMode ++ valid bool ++ }{ ++ {"", DataEngineModeV1Capable, true}, ++ {"v1-capable", DataEngineModeV1Capable, true}, ++ {"v2-only", DataEngineModeV2Only, true}, ++ {"v2", "", false}, ++ {"V2-ONLY", "", false}, ++ {" v2-only", "", false}, ++ {"v2-only ", "", false}, ++ {"disabled", "", false}, ++ } { ++ t.Run(test.input, func(t *testing.T) { ++ got, err := ParseDataEngineMode(test.input) ++ if (err == nil) != test.valid || got != test.want { ++ t.Fatalf("mode %q: got %q, error %v", test.input, got, err) ++ } ++ }) ++ } ++} ++ ++func TestDataEngineModeCapabilityCeiling(t *testing.T) { ++ for _, test := range []struct { ++ mode DataEngineMode ++ iscsi bool ++ allowed []longhorn.DataEngineType ++ }{ ++ {"", true, []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2, longhorn.DataEngineTypeAll}}, ++ {DataEngineModeV1Capable, true, []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2, longhorn.DataEngineTypeAll}}, ++ {DataEngineModeV2Only, false, []longhorn.DataEngineType{longhorn.DataEngineTypeV2}}, ++ {"invalid-mode", true, nil}, ++ } { ++ t.Run(string(test.mode), func(t *testing.T) { ++ if test.mode.RequiresISCSI() != test.iscsi { ++ t.Fatal("iSCSI prerequisite did not follow the installation capability") ++ } ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2, longhorn.DataEngineTypeAll, "unrecognized"} { ++ want := false ++ for _, allowed := range test.allowed { ++ want = want || engine == allowed ++ } ++ if test.mode.AllowsDataEngine(engine) != want { ++ t.Fatalf("mode %q, engine %q: wrong capability ceiling", test.mode, engine) ++ } ++ } ++ }) ++ } ++} +diff --git a/util/client/client.go b/util/client/client.go +index 5ffa209ad34caa4550b2f99e53ff9d4afda54492..207af43c6350835e4b5a23a74405a0acd40a502a 100644 +--- a/util/client/client.go ++++ b/util/client/client.go +@@ -39,18 +39,23 @@ type Clients struct { + + // NewClients constructs a Clients whose datastore, when needDataStore is + // true, uses the cluster-wide Pod informer. +-func NewClients(kubeconfigPath string, needDataStore bool, stopCh <-chan struct{}) (*Clients, error) { +- return newClients(kubeconfigPath, needDataStore, false, stopCh) ++func NewClients(kubeconfigPath string, needDataStore bool, mode types.DataEngineMode, stopCh <-chan struct{}) (*Clients, error) { ++ return newClients(kubeconfigPath, needDataStore, false, mode, stopCh) + } + + // NewClientsForNodeLocal constructs a Clients whose datastore uses the + // namespace-filtered (longhorn-system) Pod informer; the cluster-wide Pod + // watch lives only in the longhorn-global-manager Deployment. +-func NewClientsForNodeLocal(kubeconfigPath string, stopCh <-chan struct{}) (*Clients, error) { +- return newClients(kubeconfigPath, true, true, stopCh) ++func NewClientsForNodeLocal(kubeconfigPath string, mode types.DataEngineMode, stopCh <-chan struct{}) (*Clients, error) { ++ return newClients(kubeconfigPath, true, true, mode, stopCh) + } + +-func newClients(kubeconfigPath string, needDataStore, nodeLocalOnly bool, stopCh <-chan struct{}) (*Clients, error) { ++func newClients(kubeconfigPath string, needDataStore, nodeLocalOnly bool, mode types.DataEngineMode, stopCh <-chan struct{}) (*Clients, error) { ++ mode, err := types.ParseDataEngineMode(string(mode)) ++ if err != nil { ++ return nil, err ++ } ++ + namespace := os.Getenv(types.EnvPodNamespace) + if namespace == "" { + logrus.Warnf("Cannot detect pod namespace, environment variable %v is missing, using default namespace", types.EnvPodNamespace) +@@ -110,9 +115,9 @@ func newClients(kubeconfigPath string, needDataStore, nodeLocalOnly bool, stopCh + // some refs to look at: https://github.com/kubernetes-sigs/controller-runtime/issues/521 + informerFactories := util.NewInformerFactories(namespace, clients.K8s, lhClient, 30*time.Second) + if nodeLocalOnly { +- ds = datastore.NewDataStoreForNodeLocal(namespace, lhClient, clients.K8s, extensionsClient, informerFactories) ++ ds = datastore.NewDataStoreForNodeLocal(namespace, lhClient, clients.K8s, extensionsClient, informerFactories, mode) + } else { +- ds = datastore.NewDataStoreForGlobal(namespace, lhClient, clients.K8s, extensionsClient, informerFactories) ++ ds = datastore.NewDataStoreForGlobal(namespace, lhClient, clients.K8s, extensionsClient, informerFactories, mode) + } + + informerFactories.Start(stopCh) +diff --git a/util/client/data_engine_mode_test.go b/util/client/data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..7cd7d9607c41959196343133aa011d02ff1eacef +--- /dev/null ++++ b/util/client/data_engine_mode_test.go +@@ -0,0 +1,22 @@ ++package client ++ ++import ( ++ "strings" ++ "testing" ++ ++ "github.com/longhorn/longhorn-manager/types" ++) ++ ++func TestInvalidDataEngineCapabilityRejectedBeforeClientConfiguration(t *testing.T) { ++ for _, nodeLocal := range []bool{false, true} { ++ var err error ++ if nodeLocal { ++ _, err = NewClientsForNodeLocal("/must-not-be-read", types.DataEngineMode("unknown"), t.Context().Done()) ++ } else { ++ _, err = NewClients("/must-not-be-read", true, types.DataEngineMode("unknown"), t.Context().Done()) ++ } ++ if err == nil || !strings.Contains(err.Error(), "unsupported data engine mode") { ++ t.Fatalf("node-local=%t: invalid capability reached client setup: %v", nodeLocal, err) ++ } ++ } ++} +diff --git a/webhook/common/common.go b/webhook/common/common.go +index 0f2f2948d0690d1250be451eb96a4c7ee8805297..547e9b01b45f17d9181e038f64c1a370d6674198 100644 +--- a/webhook/common/common.go ++++ b/webhook/common/common.go +@@ -95,6 +95,9 @@ func GetLonghornLabelsPatchOp(obj runtime.Object, requiredLabels, removingLabels + } + + func ValidateRequiredDataEngineEnabled(ds *datastore.DataStore, dataEngine longhorn.DataEngineType) error { ++ if err := ValidateDataEngineCapability(ds, dataEngine); err != nil { ++ return err ++ } + dataEngineSetting := types.SettingNameV1DataEngine + if types.IsDataEngineV2(dataEngine) { + dataEngineSetting = types.SettingNameV2DataEngine +diff --git a/webhook/common/data_engine_mode.go b/webhook/common/data_engine_mode.go +new file mode 100644 +index 0000000000000000000000000000000000000000..621964c3eec012b9e820f70d315df7dbb0c921fa +--- /dev/null ++++ b/webhook/common/data_engine_mode.go +@@ -0,0 +1,167 @@ ++package common ++ ++import ( ++ "fmt" ++ "reflect" ++ "slices" ++ ++ "k8s.io/apimachinery/pkg/api/meta" ++ "k8s.io/apimachinery/pkg/runtime" ++ ++ "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ werror "github.com/longhorn/longhorn-manager/webhook/error" ++) ++ ++func admissionDataEngineMode(ds *datastore.DataStore) (types.DataEngineMode, error) { ++ if ds == nil { ++ return "", werror.NewInternalError("missing installation data engine capability") ++ } ++ mode, err := types.ParseDataEngineMode(string(ds.DataEngineMode())) ++ if err != nil { ++ return "", werror.NewInternalError(err.Error()) ++ } ++ return mode, nil ++} ++ ++func ValidateDataEngineCapability(ds *datastore.DataStore, engine longhorn.DataEngineType) error { ++ mode, err := admissionDataEngineMode(ds) ++ if err != nil { ++ return err ++ } ++ if mode == types.DataEngineModeV2Only && engine != longhorn.DataEngineTypeV2 { ++ return werror.NewForbiddenError(fmt.Sprintf("data engine %q is outside the v2-only installation capability", engine)) ++ } ++ return nil ++} ++ ++// ValidateDataEngineCapabilityUpdate reports whether the request is a strictly ++// bounded finalizer cleanup that must bypass mutation and activation checks. ++func ValidateDataEngineCapabilityUpdate(ds *datastore.DataStore, oldObj, newObj runtime.Object) (bool, error) { ++ mode, err := admissionDataEngineMode(ds) ++ if err != nil { ++ return false, err ++ } ++ if mode != types.DataEngineModeV2Only { ++ return false, nil ++ } ++ oldEngine, err := resourceDataEngine(oldObj) ++ if err != nil { ++ return false, werror.NewInvalidError(err.Error(), "") ++ } ++ newEngine, err := resourceDataEngine(newObj) ++ if err != nil { ++ return false, werror.NewInvalidError(err.Error(), "") ++ } ++ if reflect.TypeOf(oldObj) != reflect.TypeOf(newObj) { ++ return false, werror.NewInvalidError("data engine update requires matching Longhorn resource types", "") ++ } ++ cleanup, err := IsLonghornFinalizerOnlyRemoval(oldObj, newObj) ++ if err != nil { ++ return false, werror.NewInvalidError(err.Error(), "") ++ } ++ if cleanup { ++ return true, nil ++ } ++ oldMeta, err := meta.Accessor(oldObj) ++ if err != nil { ++ return false, werror.NewInvalidError(err.Error(), "") ++ } ++ newMeta, err := meta.Accessor(newObj) ++ if err != nil { ++ return false, werror.NewInvalidError(err.Error(), "") ++ } ++ countFinalizer := func(finalizers []string) int { ++ count := 0 ++ for _, finalizer := range finalizers { ++ if finalizer == longhornFinalizerKey { ++ count++ ++ } ++ } ++ return count ++ } ++ if countFinalizer(oldMeta.GetFinalizers()) > countFinalizer(newMeta.GetFinalizers()) { ++ return false, werror.NewInvalidError("Longhorn finalizer removal must preserve the entire deleting resource", "metadata.finalizers") ++ } ++ if err := ValidateDataEngineCapability(ds, oldEngine); err != nil { ++ return false, err ++ } ++ return false, ValidateDataEngineCapability(ds, newEngine) ++} ++ ++func IsLonghornFinalizerOnlyRemoval(oldObj, newObj runtime.Object) (bool, error) { ++ if _, err := resourceDataEngine(oldObj); err != nil { ++ return false, err ++ } ++ if _, err := resourceDataEngine(newObj); err != nil { ++ return false, err ++ } ++ if reflect.TypeOf(oldObj) != reflect.TypeOf(newObj) { ++ return false, fmt.Errorf("finalizer cleanup requires matching resource types") ++ } ++ oldMeta, err := meta.Accessor(oldObj) ++ if err != nil { ++ return false, err ++ } ++ newMeta, err := meta.Accessor(newObj) ++ if err != nil { ++ return false, err ++ } ++ if oldMeta.GetName() == "" || oldMeta.GetNamespace() == "" || oldMeta.GetUID() == "" || oldMeta.GetResourceVersion() == "" || ++ oldMeta.GetName() != newMeta.GetName() || oldMeta.GetNamespace() != newMeta.GetNamespace() || oldMeta.GetUID() != newMeta.GetUID() || ++ oldMeta.GetDeletionTimestamp() == nil || newMeta.GetDeletionTimestamp() == nil || !oldMeta.GetDeletionTimestamp().Equal(newMeta.GetDeletionTimestamp()) { ++ return false, nil ++ } ++ seen := map[string]bool{} ++ expected := []string{} ++ for _, finalizer := range oldMeta.GetFinalizers() { ++ if seen[finalizer] { ++ return false, nil ++ } ++ seen[finalizer] = true ++ if finalizer != longhornFinalizerKey { ++ expected = append(expected, finalizer) ++ } ++ } ++ if !seen[longhornFinalizerKey] || !slices.Equal(expected, newMeta.GetFinalizers()) { ++ return false, nil ++ } ++ oldCopy, newCopy := oldObj.DeepCopyObject(), newObj.DeepCopyObject() ++ oldCopyMeta, err := meta.Accessor(oldCopy) ++ if err != nil { ++ return false, err ++ } ++ newCopyMeta, err := meta.Accessor(newCopy) ++ if err != nil { ++ return false, err ++ } ++ oldCopyMeta.SetFinalizers(nil) ++ newCopyMeta.SetFinalizers(nil) ++ oldCopyMeta.SetManagedFields(nil) ++ newCopyMeta.SetManagedFields(nil) ++ return reflect.DeepEqual(oldCopy, newCopy), nil ++} ++ ++func resourceDataEngine(obj runtime.Object) (longhorn.DataEngineType, error) { ++ switch object := obj.(type) { ++ case *longhorn.Volume: ++ if object != nil { ++ return object.Spec.DataEngine, nil ++ } ++ case *longhorn.Engine: ++ if object != nil { ++ return object.Spec.DataEngine, nil ++ } ++ case *longhorn.Replica: ++ if object != nil { ++ return object.Spec.DataEngine, nil ++ } ++ case *longhorn.InstanceManager: ++ if object != nil { ++ return object.Spec.DataEngine, nil ++ } ++ } ++ return "", fmt.Errorf("expected a non-nil Longhorn data engine resource, got %T", obj) ++} +diff --git a/webhook/common/data_engine_mode_test.go b/webhook/common/data_engine_mode_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..ab414388298d98e30293cae3cecf6948f5f7d725 +--- /dev/null ++++ b/webhook/common/data_engine_mode_test.go +@@ -0,0 +1,213 @@ ++package common ++ ++import ( ++ "net/http" ++ "reflect" ++ "testing" ++ "time" ++ ++ "k8s.io/apimachinery/pkg/api/meta" ++ "k8s.io/apimachinery/pkg/runtime" ++ ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ ++ "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" ++ "github.com/longhorn/longhorn-manager/util" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func capabilityTestStore(t *testing.T, mode types.DataEngineMode) *datastore.DataStore { ++ return capabilityTestStoreSettings(t, mode, "true", "true") ++} ++ ++func capabilityTestStoreSettings(t *testing.T, mode types.DataEngineMode, v1, v2 string) *datastore.DataStore { ++ t.Helper() ++ lh, kube := lhfake.NewClientset(), kubefake.NewClientset() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := datastore.NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, mode) ++ for name, value := range map[types.SettingName]string{types.SettingNameV1DataEngine: v1, types.SettingNameV2DataEngine: v2} { ++ if err := factories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer().Add(&longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: "longhorn-system"}, Value: value}); err != nil { ++ t.Fatal(err) ++ } ++ } ++ return store ++} ++ ++func capabilityObject(kind int, engine longhorn.DataEngineType) runtime.Object { ++ metadata := metav1.ObjectMeta{Name: "retained", Namespace: "longhorn-system", UID: "retained-uid", ResourceVersion: "10", Finalizers: []string{"foreign", "longhorn.io"}} ++ switch kind { ++ case 0: ++ return &longhorn.Volume{ObjectMeta: metadata, Spec: longhorn.VolumeSpec{DataEngine: engine}} ++ case 1: ++ return &longhorn.Engine{ObjectMeta: metadata, Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: engine}}} ++ case 2: ++ return &longhorn.Replica{ObjectMeta: metadata, Spec: longhorn.ReplicaSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: engine}}} ++ default: ++ return &longhorn.InstanceManager{ObjectMeta: metadata, Spec: longhorn.InstanceManagerSpec{DataEngine: engine}} ++ } ++} ++ ++func TestV2OnlyCommonSettingGateCannotWidenCapability(t *testing.T) { ++ store := capabilityTestStore(t, types.DataEngineModeV2Only) ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll, "unknown"} { ++ if err := ValidateRequiredDataEngineEnabled(store, engine); err == nil { ++ t.Fatalf("mutable V1=true admitted unsupported engine %q", engine) ++ } ++ } ++ if err := ValidateRequiredDataEngineEnabled(store, longhorn.DataEngineTypeV2); err != nil { ++ t.Fatal(err) ++ } ++} ++ ++func TestV2OnlyUpdateRejectsLegacyRelabeling(t *testing.T) { ++ store := capabilityTestStore(t, types.DataEngineModeV2Only) ++ for kind := range 4 { ++ for _, oldEngine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2, longhorn.DataEngineTypeAll} { ++ for _, newEngine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2, longhorn.DataEngineTypeAll} { ++ cleanup, err := ValidateDataEngineCapabilityUpdate(store, capabilityObject(kind, oldEngine), capabilityObject(kind, newEngine)) ++ allowed := oldEngine == longhorn.DataEngineTypeV2 && newEngine == longhorn.DataEngineTypeV2 ++ if cleanup || (err == nil) != allowed { ++ t.Fatalf("kind=%d old=%q new=%q: cleanup=%t err=%v", kind, oldEngine, newEngine, cleanup, err) ++ } ++ } ++ } ++ } ++} ++ ++func TestV2OnlyStrictFinalizerCleanupPreservesTheWholeObject(t *testing.T) { ++ store := capabilityTestStore(t, types.DataEngineModeV2Only) ++ for kind := range 4 { ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2} { ++ oldObj := capabilityObject(kind, engine) ++ oldMeta, _ := meta.Accessor(oldObj) ++ deleted := metav1.Now() ++ oldMeta.SetDeletionTimestamp(&deleted) ++ newObj := oldObj.DeepCopyObject() ++ newMeta, _ := meta.Accessor(newObj) ++ newMeta.SetFinalizers([]string{"foreign"}) ++ oldBefore, newBefore := oldObj.DeepCopyObject(), newObj.DeepCopyObject() ++ if cleanup, err := ValidateDataEngineCapabilityUpdate(store, oldObj, newObj); err != nil || !cleanup { ++ t.Fatalf("kind=%d engine=%q blocked exact cleanup: cleanup=%t err=%v", kind, engine, cleanup, err) ++ } ++ if !reflect.DeepEqual(oldObj, oldBefore) || !reflect.DeepEqual(newObj, newBefore) { ++ t.Fatal("cleanup validation mutated an admission object") ++ } ++ for _, mutate := range []func(metav1.Object){ ++ func(m metav1.Object) { m.SetName("replacement") }, ++ func(m metav1.Object) { m.SetNamespace("different") }, ++ func(m metav1.Object) { m.SetUID("different") }, ++ func(m metav1.Object) { m.SetResourceVersion("11") }, ++ func(m metav1.Object) { m.SetGeneration(2) }, ++ func(m metav1.Object) { m.SetLabels(map[string]string{"activation": "changed"}) }, ++ func(m metav1.Object) { m.SetAnnotations(map[string]string{"activation": "changed"}) }, ++ func(m metav1.Object) { ++ m.SetOwnerReferences([]metav1.OwnerReference{{Name: "replacement", UID: "different"}}) ++ }, ++ func(m metav1.Object) { m.SetFinalizers([]string{"substituted"}) }, ++ func(m metav1.Object) { ++ changed := metav1.NewTime(deleted.Add(time.Second)) ++ m.SetDeletionTimestamp(&changed) ++ }, ++ } { ++ changed := newObj.DeepCopyObject() ++ metadata, _ := meta.Accessor(changed) ++ mutate(metadata) ++ if cleanup, err := ValidateDataEngineCapabilityUpdate(store, oldObj, changed); err == nil || cleanup { ++ t.Fatalf("kind=%d engine=%q cleanup hid another metadata change", kind, engine) ++ } ++ } ++ for _, mutate := range []func(runtime.Object){ ++ func(object runtime.Object) { ++ switch object := object.(type) { ++ case *longhorn.Volume: ++ object.Spec.NodeID = "activate" ++ case *longhorn.Engine: ++ object.Spec.DesireState = longhorn.InstanceStateRunning ++ case *longhorn.Replica: ++ object.Spec.DesireState = longhorn.InstanceStateRunning ++ case *longhorn.InstanceManager: ++ object.Spec.NodeID = "activate" ++ } ++ }, ++ func(object runtime.Object) { ++ switch object := object.(type) { ++ case *longhorn.Volume: ++ object.Status.CurrentNodeID = "activate" ++ case *longhorn.Engine: ++ object.Status.CurrentState = longhorn.InstanceStateRunning ++ case *longhorn.Replica: ++ object.Status.CurrentState = longhorn.InstanceStateRunning ++ case *longhorn.InstanceManager: ++ object.Status.CurrentState = longhorn.InstanceManagerStateRunning ++ } ++ }, ++ } { ++ changed := newObj.DeepCopyObject() ++ mutate(changed) ++ if cleanup, err := ValidateDataEngineCapabilityUpdate(store, oldObj, changed); err == nil || cleanup { ++ t.Fatalf("kind=%d engine=%q cleanup hid a spec or status change", kind, engine) ++ } ++ } ++ oldMeta.SetFinalizers([]string{"foreign", "longhorn.io", "longhorn.io"}) ++ if cleanup, err := ValidateDataEngineCapabilityUpdate(store, oldObj, newObj); err == nil || cleanup { ++ t.Fatal("duplicate Longhorn finalizers cleared cleanup") ++ } ++ } ++ } ++} ++ ++func TestCapabilityGatePrecedesSettingsAndPreservesEnablement(t *testing.T) { ++ store := capabilityTestStoreSettings(t, types.DataEngineModeV2Only, "invalid", "true") ++ err := ValidateRequiredDataEngineEnabled(store, longhorn.DataEngineTypeV1) ++ status, ok := err.(interface{ AsResult() *metav1.Status }) ++ if !ok || status.AsResult().Code != http.StatusForbidden { ++ t.Fatalf("V1 capability denial read the invalid mutable Setting first: %v", err) ++ } ++ for _, value := range []string{"false", "invalid"} { ++ store = capabilityTestStoreSettings(t, types.DataEngineModeV2Only, "false", value) ++ err = ValidateRequiredDataEngineEnabled(store, longhorn.DataEngineTypeV2) ++ if err == nil { ++ t.Fatalf("capability enabled V2 despite mutable Setting %q", value) ++ } ++ } ++ for _, mode := range []types.DataEngineMode{"", types.DataEngineModeV1Capable} { ++ store = capabilityTestStore(t, mode) ++ for kind := range 4 { ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2, longhorn.DataEngineTypeAll} { ++ if cleanup, err := ValidateDataEngineCapabilityUpdate(store, capabilityObject(kind, engine), capabilityObject(kind, engine)); err != nil || cleanup { ++ t.Fatalf("default mode update behavior changed: %v", err) ++ } ++ } ++ } ++ } ++} ++ ++func TestFinalizerCleanupRejectsInvalidObjectsWithoutPanic(t *testing.T) { ++ for _, invalid := range []runtime.Object{nil, (*longhorn.Volume)(nil), &metav1.Status{}} { ++ if cleanup, err := IsLonghornFinalizerOnlyRemoval(invalid, capabilityObject(0, longhorn.DataEngineTypeV1)); err == nil || cleanup { ++ t.Fatal("invalid cleanup object was accepted") ++ } ++ } ++} ++ ++func TestStrictCleanupAllowsOnlyAPIBookkeepingAndSoleFinalizerRemoval(t *testing.T) { ++ store := capabilityTestStore(t, types.DataEngineModeV2Only) ++ for kind := range 4 { ++ oldObj := capabilityObject(kind, longhorn.DataEngineTypeV1) ++ oldMeta, _ := meta.Accessor(oldObj) ++ deleted := metav1.Now() ++ oldMeta.SetDeletionTimestamp(&deleted) ++ oldMeta.SetFinalizers([]string{"longhorn.io"}) ++ newObj := oldObj.DeepCopyObject() ++ newMeta, _ := meta.Accessor(newObj) ++ newMeta.SetFinalizers(nil) ++ newMeta.SetManagedFields([]metav1.ManagedFieldsEntry{{Manager: "garbage-collector", Operation: metav1.ManagedFieldsOperationUpdate, APIVersion: longhorn.SchemeGroupVersion.String()}}) ++ if cleanup, err := ValidateDataEngineCapabilityUpdate(store, oldObj, newObj); err != nil || !cleanup { ++ t.Fatalf("kind=%d rejected sole-finalizer cleanup or API bookkeeping: cleanup=%t err=%v", kind, cleanup, err) ++ } ++ } ++} +diff --git a/webhook/data_engine_mode_integration_test.go b/webhook/data_engine_mode_integration_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..409f6d09b1604ac5a2b53f180a0f1f51a221b7d7 +--- /dev/null ++++ b/webhook/data_engine_mode_integration_test.go +@@ -0,0 +1,125 @@ ++package webhook_test ++ ++import ( ++ "net/http" ++ "testing" ++ ++ "k8s.io/apimachinery/pkg/api/meta" ++ "k8s.io/apimachinery/pkg/runtime" ++ ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ ++ "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" ++ "github.com/longhorn/longhorn-manager/util" ++ "github.com/longhorn/longhorn-manager/webhook/admission" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++ enginewebhook "github.com/longhorn/longhorn-manager/webhook/resources/engine" ++ imwebhook "github.com/longhorn/longhorn-manager/webhook/resources/instancemanager" ++ replicawebhook "github.com/longhorn/longhorn-manager/webhook/resources/replica" ++ volumewebhook "github.com/longhorn/longhorn-manager/webhook/resources/volume" ++) ++ ++type capabilityWebhook struct { ++ name string ++ mutator func(*datastore.DataStore) admission.Mutator ++ validator func(*datastore.DataStore) admission.Validator ++ object func(longhorn.DataEngineType) runtime.Object ++} ++ ++func capabilityWebhooks() []capabilityWebhook { ++ metadata := func() metav1.ObjectMeta { ++ return metav1.ObjectMeta{Name: "retained", Namespace: "longhorn-system", UID: "source-uid", ResourceVersion: "10", Finalizers: []string{"foreign", "longhorn.io"}} ++ } ++ return []capabilityWebhook{ ++ {"volume", volumewebhook.NewMutator, func(ds *datastore.DataStore) admission.Validator { return volumewebhook.NewValidator(ds, "node") }, func(e longhorn.DataEngineType) runtime.Object { ++ return &longhorn.Volume{ObjectMeta: metadata(), Spec: longhorn.VolumeSpec{DataEngine: e}} ++ }}, ++ {"engine", enginewebhook.NewMutator, enginewebhook.NewValidator, func(e longhorn.DataEngineType) runtime.Object { ++ return &longhorn.Engine{ObjectMeta: metadata(), Spec: longhorn.EngineSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: e}}} ++ }}, ++ {"replica", replicawebhook.NewMutator, replicawebhook.NewValidator, func(e longhorn.DataEngineType) runtime.Object { ++ return &longhorn.Replica{ObjectMeta: metadata(), Spec: longhorn.ReplicaSpec{InstanceSpec: longhorn.InstanceSpec{DataEngine: e}}} ++ }}, ++ {"instance-manager", imwebhook.NewMutator, imwebhook.NewValidator, func(e longhorn.DataEngineType) runtime.Object { ++ return &longhorn.InstanceManager{ObjectMeta: metadata(), Spec: longhorn.InstanceManagerSpec{DataEngine: e}} ++ }}, ++ } ++} ++ ++func capabilityWebhookStore(t *testing.T) *datastore.DataStore { ++ t.Helper() ++ lh, kube := lhfake.NewClientset(), kubefake.NewClientset() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := datastore.NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, types.DataEngineModeV2Only) ++ for _, name := range []types.SettingName{types.SettingNameV1DataEngine, types.SettingNameV2DataEngine} { ++ if err := factories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer().Add(&longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: "longhorn-system"}, Value: "true"}); err != nil { ++ t.Fatal(err) ++ } ++ } ++ return store ++} ++ ++func requireCapabilityDenial(t *testing.T, err error) { ++ t.Helper() ++ status, ok := err.(interface{ AsResult() *metav1.Status }) ++ if !ok || status.AsResult().Code != http.StatusForbidden { ++ t.Fatalf("request reached native processing instead of the capability boundary: %v", err) ++ } ++} ++ ++func TestV2OnlyNativeWebhookCreationBoundaries(t *testing.T) { ++ for _, webhook := range capabilityWebhooks() { ++ t.Run(webhook.name, func(t *testing.T) { ++ store := capabilityWebhookStore(t) ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll, "unknown"} { ++ if webhook.name != "volume" || engine != "" { ++ _, err := webhook.mutator(store).Create(nil, webhook.object(engine)) ++ requireCapabilityDenial(t, err) ++ } ++ requireCapabilityDenial(t, webhook.validator(store).Create(nil, webhook.object(engine))) ++ } ++ }) ++ } ++} ++ ++func TestV2OnlyNativeWebhooksRejectLegacyRelabeling(t *testing.T) { ++ for _, webhook := range capabilityWebhooks() { ++ t.Run(webhook.name, func(t *testing.T) { ++ store := capabilityWebhookStore(t) ++ for _, oldEngine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeAll} { ++ oldObj, newObj := webhook.object(oldEngine), webhook.object(longhorn.DataEngineTypeV2) ++ _, err := webhook.mutator(store).Update(nil, oldObj, newObj) ++ requireCapabilityDenial(t, err) ++ requireCapabilityDenial(t, webhook.validator(store).Update(nil, oldObj, newObj)) ++ } ++ }) ++ } ++} ++ ++func TestV2OnlyNativeWebhookCleanupHasZeroPatches(t *testing.T) { ++ for _, webhook := range capabilityWebhooks() { ++ t.Run(webhook.name, func(t *testing.T) { ++ store := capabilityWebhookStore(t) ++ for _, engine := range []longhorn.DataEngineType{"", longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2} { ++ oldObj := webhook.object(engine) ++ oldMeta, _ := meta.Accessor(oldObj) ++ deleted := metav1.Now() ++ oldMeta.SetDeletionTimestamp(&deleted) ++ newObj := oldObj.DeepCopyObject() ++ newMeta, _ := meta.Accessor(newObj) ++ newMeta.SetFinalizers([]string{"foreign"}) ++ patches, err := webhook.mutator(store).Update(nil, oldObj, newObj) ++ if err != nil || len(patches) != 0 { ++ t.Fatalf("cleanup changed the object or required native reads: engine=%q patches=%v err=%v", engine, patches, err) ++ } ++ if err := webhook.validator(store).Update(nil, oldObj, newObj); err != nil { ++ t.Fatalf("cleanup blocked: engine=%q err=%v", engine, err) ++ } ++ } ++ }) ++ } ++} +diff --git a/webhook/resources/engine/mutator.go b/webhook/resources/engine/mutator.go +index 5459a621a5ca26106ac5afa1df0df4634c2f8a61..7e2445c39d5a08130a415f135d116179fef5bed6 100644 +--- a/webhook/resources/engine/mutator.go ++++ b/webhook/resources/engine/mutator.go +@@ -41,10 +41,24 @@ func (e *engineMutator) Resource() admission.Resource { + } + + func (e *engineMutator) Create(request *admission.Request, newObj runtime.Object) (admission.PatchOps, error) { ++ engine, ok := newObj.(*longhorn.Engine) ++ if !ok || engine == nil { ++ return nil, werror.NewInvalidError("expected a non-nil Longhorn Engine", "") ++ } ++ if err := common.ValidateDataEngineCapability(e.ds, engine.Spec.DataEngine); err != nil { ++ return nil, err ++ } + return mutate(newObj) + } + + func (e *engineMutator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) (admission.PatchOps, error) { ++ cleanup, err := common.ValidateDataEngineCapabilityUpdate(e.ds, oldObj, newObj) ++ if err != nil { ++ return nil, err ++ } ++ if cleanup { ++ return nil, nil ++ } + return mutate(newObj) + } + +diff --git a/webhook/resources/engine/validator.go b/webhook/resources/engine/validator.go +index 016d51de079f6ddff1fb3bd994f8bbba0358e21e..92e56622c486697f51454d781e8e79f7f606e8b2 100644 +--- a/webhook/resources/engine/validator.go ++++ b/webhook/resources/engine/validator.go +@@ -43,9 +43,12 @@ func (e *engineValidator) Resource() admission.Resource { + + func (e *engineValidator) Create(request *admission.Request, newObj runtime.Object) error { + engine, ok := newObj.(*longhorn.Engine) +- if !ok { ++ if !ok || engine == nil { + return werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.Engine", newObj), "") + } ++ if err := wcommon.ValidateDataEngineCapability(e.ds, engine.Spec.DataEngine); err != nil { ++ return err ++ } + + volume, err := e.ds.GetVolume(engine.Spec.VolumeName) + if err != nil { +@@ -65,6 +68,13 @@ func (e *engineValidator) Create(request *admission.Request, newObj runtime.Obje + } + + func (e *engineValidator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) error { ++ cleanup, err := wcommon.ValidateDataEngineCapabilityUpdate(e.ds, oldObj, newObj) ++ if err != nil { ++ return err ++ } ++ if cleanup { ++ return nil ++ } + oldEngine, ok := oldObj.(*longhorn.Engine) + if !ok { + return werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.Engine", oldObj), "") +diff --git a/webhook/resources/instancemanager/mutator.go b/webhook/resources/instancemanager/mutator.go +index 0db96ea1bd58e4d6d4773eeb5015daf707ec960a..732f92450ea0e3369aab36f2b5bb2c2f13b0c8cb 100644 +--- a/webhook/resources/instancemanager/mutator.go ++++ b/webhook/resources/instancemanager/mutator.go +@@ -6,12 +6,14 @@ import ( + + "github.com/cockroachdb/errors" + +- admissionregv1 "k8s.io/api/admissionregistration/v1" + "k8s.io/apimachinery/pkg/runtime" + ++ admissionregv1 "k8s.io/api/admissionregistration/v1" ++ + "github.com/longhorn/longhorn-manager/datastore" + "github.com/longhorn/longhorn-manager/types" + "github.com/longhorn/longhorn-manager/webhook/admission" ++ "github.com/longhorn/longhorn-manager/webhook/common" + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + werror "github.com/longhorn/longhorn-manager/webhook/error" +@@ -42,9 +44,12 @@ func (i *instanceManagerMutator) Resource() admission.Resource { + + func (i *instanceManagerMutator) Create(request *admission.Request, newObj runtime.Object) (admission.PatchOps, error) { + im, ok := newObj.(*longhorn.InstanceManager) +- if !ok { ++ if !ok || im == nil { + return nil, werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.InstanceManager", newObj), "") + } ++ if err := common.ValidateDataEngineCapability(i.ds, im.Spec.DataEngine); err != nil { ++ return nil, err ++ } + + patchOps := mutate(im) + +@@ -78,6 +83,13 @@ func (i *instanceManagerMutator) Create(request *admission.Request, newObj runti + } + + func (i *instanceManagerMutator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) (admission.PatchOps, error) { ++ cleanup, err := common.ValidateDataEngineCapabilityUpdate(i.ds, oldObj, newObj) ++ if err != nil { ++ return nil, err ++ } ++ if cleanup { ++ return nil, nil ++ } + newIm, ok := newObj.(*longhorn.InstanceManager) + if !ok { + return nil, werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.InstanceManager", newObj), "") +diff --git a/webhook/resources/instancemanager/validator.go b/webhook/resources/instancemanager/validator.go +index 8200d5beadc2bf4ddeb7dedd1c299f1bfdbad7fe..e336dd0cc839e89daf060795bb7e5130078b05f5 100644 +--- a/webhook/resources/instancemanager/validator.go ++++ b/webhook/resources/instancemanager/validator.go +@@ -9,6 +9,7 @@ import ( + + "github.com/longhorn/longhorn-manager/datastore" + "github.com/longhorn/longhorn-manager/webhook/admission" ++ "github.com/longhorn/longhorn-manager/webhook/common" + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + werror "github.com/longhorn/longhorn-manager/webhook/error" +@@ -39,9 +40,12 @@ func (i *instanceManagerValidator) Resource() admission.Resource { + + func (i *instanceManagerValidator) Create(request *admission.Request, newObj runtime.Object) error { + im, ok := newObj.(*longhorn.InstanceManager) +- if !ok { ++ if !ok || im == nil { + return werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.InstanceManager", newObj), "") + } ++ if err := common.ValidateDataEngineCapability(i.ds, im.Spec.DataEngine); err != nil { ++ return err ++ } + if err := i.validate(im); err != nil { + return werror.NewInvalidError(err.Error(), "") + } +@@ -50,6 +54,13 @@ func (i *instanceManagerValidator) Create(request *admission.Request, newObj run + } + + func (i *instanceManagerValidator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) error { ++ cleanup, err := common.ValidateDataEngineCapabilityUpdate(i.ds, oldObj, newObj) ++ if err != nil { ++ return err ++ } ++ if cleanup { ++ return nil ++ } + newIm, ok := newObj.(*longhorn.InstanceManager) + if !ok { + return werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.InstanceManager", newObj), "") +diff --git a/webhook/resources/node/validator_test.go b/webhook/resources/node/validator_test.go +index 3840e6680b7bd351733d01f47125523b4e6d2ee2..0d195fbc10de99f58be1e1eb4d1d789d0b942681 100644 +--- a/webhook/resources/node/validator_test.go ++++ b/webhook/resources/node/validator_test.go +@@ -16,6 +16,7 @@ import ( + + longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" + lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++ longhorntypes "github.com/longhorn/longhorn-manager/types" + managerutil "github.com/longhorn/longhorn-manager/util" + ) + +@@ -221,7 +222,7 @@ func newNodeValidatorForTest(kubeClient *kubefake.Clientset) *nodeValidator { + lhClient := lhfake.NewClientset() + extensionsClient := apiextensionsfake.NewSimpleClientset() + informerFactories := managerutil.NewInformerFactories(nodeValidatorTestNamespace, kubeClient, lhClient, 0) +- ds := datastore.NewDataStoreForGlobal(nodeValidatorTestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(nodeValidatorTestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, longhorntypes.DataEngineModeV1Capable) + return &nodeValidator{ds: ds} + } + +diff --git a/webhook/resources/replica/mutator.go b/webhook/resources/replica/mutator.go +index 9f4c6723ebefc9dd6af5899c72aebc50515a8c87..e4f19920994a45e42aad8e8089fc0518a398423a 100644 +--- a/webhook/resources/replica/mutator.go ++++ b/webhook/resources/replica/mutator.go +@@ -41,10 +41,24 @@ func (e *replicaMutator) Resource() admission.Resource { + } + + func (e *replicaMutator) Create(request *admission.Request, newObj runtime.Object) (admission.PatchOps, error) { ++ replica, ok := newObj.(*longhorn.Replica) ++ if !ok || replica == nil { ++ return nil, werror.NewInvalidError("expected a non-nil Longhorn Replica", "") ++ } ++ if err := common.ValidateDataEngineCapability(e.ds, replica.Spec.DataEngine); err != nil { ++ return nil, err ++ } + return mutate(newObj) + } + + func (e *replicaMutator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) (admission.PatchOps, error) { ++ cleanup, err := common.ValidateDataEngineCapabilityUpdate(e.ds, oldObj, newObj) ++ if err != nil { ++ return nil, err ++ } ++ if cleanup { ++ return nil, nil ++ } + return mutate(newObj) + } + +diff --git a/webhook/resources/replica/validator.go b/webhook/resources/replica/validator.go +index b22409b99ae2b60b5f926020b2b0aea678977891..d68b448a55caee43ba5f23f6dbe7dd7ec28e87e2 100644 +--- a/webhook/resources/replica/validator.go ++++ b/webhook/resources/replica/validator.go +@@ -45,7 +45,7 @@ func (r *replicaValidator) Resource() admission.Resource { + + func (r *replicaValidator) Create(request *admission.Request, newObj runtime.Object) error { + replica, ok := newObj.(*longhorn.Replica) +- if !ok { ++ if !ok || replica == nil { + return werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.Replica", newObj), "") + } + +@@ -58,6 +58,13 @@ func (r *replicaValidator) Create(request *admission.Request, newObj runtime.Obj + } + + func (r *replicaValidator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) error { ++ cleanup, err := wcommon.ValidateDataEngineCapabilityUpdate(r.ds, oldObj, newObj) ++ if err != nil { ++ return err ++ } ++ if cleanup { ++ return nil ++ } + oldReplica, ok := oldObj.(*longhorn.Replica) + if !ok { + return werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.Replica", oldObj), "") +diff --git a/webhook/resources/snapshotgroup/mutator_test.go b/webhook/resources/snapshotgroup/mutator_test.go +index 2b89b4c26f7737962d5ddd3652b9eca7320434a3..0e2746f3921c6c988c27f451a8f859f5ecb1ef13 100644 +--- a/webhook/resources/snapshotgroup/mutator_test.go ++++ b/webhook/resources/snapshotgroup/mutator_test.go +@@ -30,7 +30,7 @@ func newTestDataStore(t *testing.T, volumes ...*longhorn.Volume) *datastore.Data + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint: staticcheck + + informerFactories := util.NewInformerFactories(testNamespace, kubeClient, lhClient, 0) +- ds := datastore.NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(testNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + volumeIndexer := informerFactories.LhInformerFactory.Longhorn().V1beta2().Volumes().Informer().GetIndexer() + for _, volume := range volumes { +diff --git a/webhook/resources/volume/mutator.go b/webhook/resources/volume/mutator.go +index 172590847ae140583df2cb248710ec38542d6f4c..5fc9d43156251c06ea1f681364ccc088028195dd 100644 +--- a/webhook/resources/volume/mutator.go ++++ b/webhook/resources/volume/mutator.go +@@ -87,10 +87,18 @@ func (v *volumeMutator) getActiveInstanceManagerImage(defaultInstanceManagerImag + + func (v *volumeMutator) Create(request *admission.Request, newObj runtime.Object) (admission.PatchOps, error) { + volume, ok := newObj.(*longhorn.Volume) +- if !ok { ++ if !ok || volume == nil { + return nil, werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.Volume", newObj), "") + } + var patchOps admission.PatchOps ++ if v.ds != nil && v.ds.DataEngineMode() == types.DataEngineModeV2Only && volume.Spec.DataEngine == "" { ++ volume = volume.DeepCopy() ++ volume.Spec.DataEngine = longhorn.DataEngineTypeV2 ++ patchOps = append(patchOps, `{"op": "add", "path": "/spec/dataEngine", "value": "v2"}`) ++ } ++ if err := common.ValidateDataEngineCapability(v.ds, volume.Spec.DataEngine); err != nil { ++ return nil, err ++ } + + name := util.AutoCorrectName(volume.Name, datastore.NameMaximumLength) + if name != volume.Name { +@@ -373,7 +381,7 @@ func (v *volumeMutator) Create(request *admission.Request, newObj runtime.Object + + var patchOpsInCommon admission.PatchOps + var err error +- if patchOpsInCommon, err = v.mutate(newObj, moreLabels); err != nil { ++ if patchOpsInCommon, err = v.mutate(volume, moreLabels); err != nil { + return nil, err + } + patchOps = append(patchOps, patchOpsInCommon...) +@@ -382,6 +390,13 @@ func (v *volumeMutator) Create(request *admission.Request, newObj runtime.Object + } + + func (v *volumeMutator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) (admission.PatchOps, error) { ++ cleanup, capabilityErr := common.ValidateDataEngineCapabilityUpdate(v.ds, oldObj, newObj) ++ if capabilityErr != nil { ++ return nil, capabilityErr ++ } ++ if cleanup { ++ return nil, nil ++ } + oldVolume, ok := oldObj.(*longhorn.Volume) + if !ok { + return nil, werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.Volume", oldObj), "") +diff --git a/webhook/resources/volume/mutator_test.go b/webhook/resources/volume/mutator_test.go +new file mode 100644 +index 0000000000000000000000000000000000000000..d41dacb0e80cd137934581509f02135e59aee622 +--- /dev/null ++++ b/webhook/resources/volume/mutator_test.go +@@ -0,0 +1,68 @@ ++package volume ++ ++import ( ++ "encoding/json" ++ "strings" ++ "testing" ++ ++ jsonpatch "github.com/evanphx/json-patch" ++ ++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ++ kubefake "k8s.io/client-go/kubernetes/fake" ++ ++ "github.com/longhorn/longhorn-manager/datastore" ++ "github.com/longhorn/longhorn-manager/types" ++ "github.com/longhorn/longhorn-manager/util" ++ ++ longhorn "github.com/longhorn/longhorn-manager/k8s/pkg/apis/longhorn/v1beta2" ++ lhfake "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned/fake" ++) ++ ++func TestCreateSelectsEngineBeforeEngineSpecificDefaults(t *testing.T) { ++ for _, mode := range []types.DataEngineMode{types.DataEngineModeV1Capable, types.DataEngineModeV2Only} { ++ lh, kube := lhfake.NewClientset(), kubefake.NewClientset() ++ factories := util.NewInformerFactories("longhorn-system", kube, lh, 0) ++ store := datastore.NewDataStoreForNodeLocal("longhorn-system", lh, kube, nil, factories, mode) ++ for name, value := range map[types.SettingName]string{ ++ types.SettingNameDefaultReplicaCount: `{"v1":"1","v2":"2"}`, ++ types.SettingNameDefaultEngineImage: "engine-v1", ++ types.SettingNameDefaultInstanceManagerImage: "im-v2", ++ } { ++ if err := factories.LhInformerFactory.Longhorn().V1beta2().Settings().Informer().GetIndexer().Add(&longhorn.Setting{ObjectMeta: metav1.ObjectMeta{Name: string(name), Namespace: "longhorn-system"}, Value: value}); err != nil { ++ t.Fatal(err) ++ } ++ } ++ input := &longhorn.Volume{ObjectMeta: metav1.ObjectMeta{Name: "new-volume", Namespace: "longhorn-system", Labels: map[string]string{"fixture": "preserve"}, Finalizers: []string{"foreign"}}, Spec: longhorn.VolumeSpec{Size: 4 << 20, Encrypted: true}} ++ wantEngine, wantImage, wantReplicas := longhorn.DataEngineTypeV2, "im-v2", 2 ++ if mode == types.DataEngineModeV1Capable { ++ input.Spec.DataEngine = longhorn.DataEngineTypeV1 ++ wantEngine, wantImage, wantReplicas = longhorn.DataEngineTypeV1, "engine-v1", 1 ++ } ++ before, err := json.Marshal(input) ++ if err != nil { ++ t.Fatal(err) ++ } ++ patches, err := NewMutator(store).Create(nil, input.DeepCopy()) ++ if err != nil { ++ t.Fatalf("mode=%s creation failed: %v", mode, err) ++ } ++ patch, err := jsonpatch.DecodePatch([]byte("[" + strings.Join(patches, ",") + "]")) ++ if err != nil { ++ t.Fatal(err) ++ } ++ after, err := patch.Apply(before) ++ if err != nil { ++ t.Fatal(err) ++ } ++ var applied longhorn.Volume ++ if err := json.Unmarshal(after, &applied); err != nil { ++ t.Fatal(err) ++ } ++ if applied.Spec.DataEngine != wantEngine || applied.Spec.NumberOfReplicas != wantReplicas || applied.Spec.Image != wantImage { ++ t.Fatalf("mode=%s selected defaults before resolving the engine: engine=%s replicas=%d image=%s", mode, applied.Spec.DataEngine, applied.Spec.NumberOfReplicas, applied.Spec.Image) ++ } ++ if mode == types.DataEngineModeV2Only && (applied.Labels[types.LonghornLabelV2EncryptedVolumeWithLuksHeader] != longhorn.TrueValue || applied.Spec.UnmapMarkSnapChainRemoved != longhorn.UnmapMarkSnapChainRemovedDisabled) { ++ t.Fatalf("V2 encryption or shared defaults were lost: labels=%v unmap=%s", applied.Labels, applied.Spec.UnmapMarkSnapChainRemoved) ++ } ++ } ++} +diff --git a/webhook/resources/volume/validator.go b/webhook/resources/volume/validator.go +index ce1f31caa1e8cbdf3dee09e7dc00d98c535df290..8d12534d90a442d9040b74af303d18bba073fef1 100644 +--- a/webhook/resources/volume/validator.go ++++ b/webhook/resources/volume/validator.go +@@ -57,9 +57,12 @@ func (v *volumeValidator) Resource() admission.Resource { + + func (v *volumeValidator) Create(request *admission.Request, newObj runtime.Object) error { + volume, ok := newObj.(*longhorn.Volume) +- if !ok { ++ if !ok || volume == nil { + return werror.NewInvalidError(fmt.Sprintf("%v is not a *longhorn.Volume", newObj), "") + } ++ if err := wcommon.ValidateDataEngineCapability(v.ds, volume.Spec.DataEngine); err != nil { ++ return err ++ } + + if !util.ValidateName(volume.Name) { + return werror.NewInvalidError(fmt.Sprintf("invalid name %v", volume.Name), "") +@@ -240,6 +243,13 @@ func (v *volumeValidator) Create(request *admission.Request, newObj runtime.Obje + } + + func (v *volumeValidator) Update(request *admission.Request, oldObj runtime.Object, newObj runtime.Object) error { ++ cleanup, err := wcommon.ValidateDataEngineCapabilityUpdate(v.ds, oldObj, newObj) ++ if err != nil { ++ return err ++ } ++ if cleanup { ++ return nil ++ } + oldVolume, ok := oldObj.(*longhorn.Volume) + if !ok { + return werror.NewInvalidError(fmt.Sprintf("oldObj %v is not a longhorn.Volume", oldObj), "") +diff --git a/webhook/resources/volume/validator_test.go b/webhook/resources/volume/validator_test.go +index 9b2f7ecf63da23ae32552331a975901460e1ef72..662b559e37dc9fe547db081a8f227cf3c2c8eb1e 100644 +--- a/webhook/resources/volume/validator_test.go ++++ b/webhook/resources/volume/validator_test.go +@@ -184,7 +184,7 @@ func newLinkedCloneTestDataStore(t *testing.T, volumes []*longhorn.Volume, snaps + extensionsClient := apiextensionsfake.NewSimpleClientset() // nolint: staticcheck + + informerFactories := util.NewInformerFactories(linkedCloneTestNamespace, kubeClient, lhClient, 0) +- ds := datastore.NewDataStoreForGlobal(linkedCloneTestNamespace, lhClient, kubeClient, extensionsClient, informerFactories) ++ ds := datastore.NewDataStoreForGlobal(linkedCloneTestNamespace, lhClient, kubeClient, extensionsClient, informerFactories, types.DataEngineModeV1Capable) + + volumeIndexer := informerFactories.LhInformerFactory.Longhorn().V1beta2().Volumes().Informer().GetIndexer() + for _, volume := range volumes { diff --git a/scripts/build-longhorn-v2-only.sh b/scripts/build-longhorn-v2-only.sh new file mode 100755 index 000000000..57aa44c5f --- /dev/null +++ b/scripts/build-longhorn-v2-only.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +build_files="$root/k8s/providers/hetzner/infrastructure/controllers/longhorn" +release=v1.13.0 +manager_revision=20d9c0790312b45aafd5668674e73dbf2eba1832 +manager_port=cbf776c32bc406304545ca36a7291730b95148ff +chart_revision=c3d88832bd8cec4f38dec65e79436984871dc313 +manager_patch=1861d01f016889dd622f479978a4a06a410a0e9984f3211084d81cd90a797356 +chart_patch=7aa94c54ab661cf7e3c5680bf380dead18d0ff06d3f6a781489aa5a055ab408b + +if [[ $# != 2 || "$1" != /* || -e "$1" || -L "$1" || ! "$2" =~ ^[1-9][0-9]{0,11}$ ]]; then + printf '%s\n' 'Usage: build-longhorn-v2-only.sh NEW_ABSOLUTE_OUTPUT_DIR PUBLISH_RUN_NUMBER' >&2 + exit 1 +fi +output="$1" +chart_version="1.13.0-platform.$2" +source_dir="$(mktemp -d)" +trap 'rm -rf -- "$source_dir"' EXIT + +checkout_release() { + local repository="$1" target="$2" revision="$3" + git init --quiet "$target" + git -C "$target" fetch --quiet --no-tags --depth=1 "$repository" "+refs/tags/$release:refs/tags/$release" + if [[ "$(git -C "$target" rev-parse "refs/tags/$release^{commit}")" != "$revision" ]]; then + printf '%s\n' 'The upstream release no longer resolves to its reviewed revision.' >&2 + exit 1 + fi + git -C "$target" checkout --quiet --detach "$revision" +} + +check_patch() { + printf '%s %s\n' "$1" "$2" | sha256sum --check +} + +manager_source="$source_dir/manager" +chart_source="$source_dir/chart" +manager_diff="$build_files/longhorn-manager-v1.13.0-v2-only.patch" +chart_diff="$build_files/longhorn-chart-v1.13.0-v2-only.patch" +check_patch "$manager_patch" "$manager_diff" +check_patch "$chart_patch" "$chart_diff" +checkout_release https://github.com/longhorn/longhorn-manager.git "$manager_source" "$manager_revision" +checkout_release https://github.com/longhorn/longhorn.git "$chart_source" "$chart_revision" + +helm template longhorn "$chart_source/chart" --namespace longhorn-system --kube-version 1.34.0 >"$source_dir/stock.yaml" +git -C "$chart_source" apply --check --whitespace=error-all "$chart_diff" +git -C "$chart_source" apply --whitespace=error-all "$chart_diff" +CHART_DEFAULT_BASELINE="$source_dir/stock.yaml" bash "$chart_source/scripts/test-data-engine-mode-chart.sh" +helm lint --strict "$chart_source/chart" + +git -C "$manager_source" apply --check --whitespace=error-all "$manager_diff" +git -C "$manager_source" apply --whitespace=error-all "$manager_diff" +mkdir -- "$output" +mkdir -- "$output/manager" "$output/chart" +( + cd "$manager_source" + export GOENV=off GOFLAGS='' GOWORK=off GOTOOLCHAIN=local + go test -race -count=1 -timeout=8m -run 'DataEngineMode|V2Only|Capability' \ + ./api ./app ./controller ./controller/monitor ./datastore ./manager ./types ./util/client ./webhook/... + build_date="$(git show -s --format=%cI "$manager_revision")" + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false \ + -ldflags="-s -w -X github.com/longhorn/longhorn-manager/meta.Version=$release -X github.com/longhorn/longhorn-manager/meta.GitCommit=$manager_port -X github.com/longhorn/longhorn-manager/meta.BuildDate=$build_date" \ + -o "$output/manager/longhorn-manager" . +) +helm package --version "$chart_version" --destination "$output/chart" "$chart_source/chart" +( + cd "$output" + sha256sum manager/longhorn-manager "chart/longhorn-$chart_version.tgz" >SHA256SUMS +)