diff --git a/.github/workflows/contract.yml b/.github/workflows/contract.yml index 670f4c60..ae29fd63 100644 --- a/.github/workflows/contract.yml +++ b/.github/workflows/contract.yml @@ -17,6 +17,10 @@ on: # merges still exercise contract tests. push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 197442fd..3d68099e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -16,6 +16,10 @@ on: # merges still exercise the envtest harness. push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 69a8a32f..3a8fd1cc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,6 +7,10 @@ name: Lint on: push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 00000000..4d54f85e --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,84 @@ +name: Deploy site to Pages + +# blockstor.org is the Hugo and Docsy site under site/. Netlify builds the +# deploy previews from the same sources (netlify.toml). +on: + push: + branches: [main] + paths: + - 'site/**' + - '.github/workflows/pages.yml' + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +# One deployment at a time. A queued run is skipped for the newest one, but +# a deployment in progress is allowed to finish. +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + env: + HUGO_VERSION: 0.164.0 + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Install Hugo + run: | + wget -q -O "${RUNNER_TEMP}/hugo.deb" \ + "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb" + sudo dpkg -i "${RUNNER_TEMP}/hugo.deb" + + - name: Setup Go + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version: "1.23" + cache: false + + - name: Setup Node.js + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + # Hugo runs Node asset tools with "node --permission", which needs + # Node 22.13 or later. + node-version: "22" + + - name: Setup Pages + id: pages + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + + - name: Install Node.js dependencies + working-directory: site + run: npm ci + + - name: Build with Hugo + working-directory: site + env: + HUGO_ENVIRONMENT: production + HUGO_ENV: production + BASE_URL: ${{ steps.pages.outputs.base_url }} + run: hugo --gc --minify --baseURL "${BASE_URL}/" + + - name: Upload artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: ./site/public + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index d97aedb8..8417d2ac 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -26,6 +26,10 @@ name: Pull Request on: pull_request: types: [opened, synchronize, reopened] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' # Cancel in-flight runs for the same PR when a new push arrives — saves # runner minutes on rapid force-push iterations. diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index ed62a118..82493ae4 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -7,6 +7,10 @@ name: E2E Tests on: push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab968f7a..5084e743 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,10 @@ name: Tests on: push: branches: [main] + # The site under site/ is published on its own and has no tests. + paths-ignore: + - 'site/**' + - 'netlify.toml' permissions: {} diff --git a/.gitignore b/.gitignore index bebae282..32e8dace 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,9 @@ .claude/ pkg/satellite/pvc-dispatch-adjust.res linstor-migrate + +# blockstor.org site build output +site/public/ +site/resources/ +site/node_modules/ +site/.hugo_build.lock diff --git a/docs/bug-42-investigation.md b/docs/bug-42-investigation.md index ac3281d0..e257f818 100644 --- a/docs/bug-42-investigation.md +++ b/docs/bug-42-investigation.md @@ -2,7 +2,7 @@ ## TL;DR -The premise in `known-issues.md` ("satellite pod IPs come from outside the pod CIDR" / "netInterfaces[0].address=") is wrong. On both e2e-iptables and e2e-quorum the stored `Node.spec.netInterfaces[0].address` is the host InternalIP (10.27.0.x / 10.161.0.x). What actually differs between the two stands is **who owns the Node CRD's `spec` once piraeus-operator finishes reconciling**: +The premise in the Bug 42 entry of `known-issues.md` (since removed) ("satellite pod IPs come from outside the pod CIDR" / "netInterfaces[0].address=") is wrong. On both e2e-iptables and e2e-quorum the stored `Node.spec.netInterfaces[0].address` is the host InternalIP (10.27.0.x / 10.161.0.x). What actually differs between the two stands is **who owns the Node CRD's `spec` once piraeus-operator finishes reconciling**: - **e2e-quorum** — piraeus runs against its own bundled `linstor-controller` (LinstorCluster `spec: {}`). Piraeus never touches the blockstor Node CRD. Spec stays exactly as `install-blockstor.sh` left it: one interface `default` with the k8s InternalIP, `generation: 1`, no `Aux/piraeus.io/*` props. - **e2e-iptables** — `LinstorCluster.spec.externalController.url = http://blockstor-apiserver.blockstor-system.svc:3370` was set (by an `observability-*` e2e scenario, see `tests/e2e/observability-three-way.sh:78-84` and `tests/e2e/observability-capacity-correlation.sh:240-244`). The operator now drives blockstor-apiserver as its LINSTOR backend and rewrites the Node CRD via `PUT /v1/nodes/{node}/net-interfaces/default-ipv4`: interface name changes from `default` to `default-ipv4`, `satellitePort=3366` + `satelliteEncryptionType=PLAIN` are added, and `Aux/piraeus.io/last-applied` / `Aux/piraeus.io/configured-interfaces` props appear in `spec.props`. Generation climbs to 9. @@ -116,5 +116,5 @@ Both `tests/e2e/observability-three-way.sh:81-84` and `tests/e2e/observability-c ## Recommended next steps 1. Apply fix (A) + (B), each as its own commit. -2. Rewrite `docs/known-issues.md` Bug 42 entry to match what's actually happening (interface-name + props churn from externalController-mode piraeus), drop the unsupported pod-CIDR claim, and adjust severity (P3 — cosmetic / fragility, not a DRBD outage). +2. Rewrite the Bug 42 entry (the page has since been removed) to match what's actually happening (interface-name + props churn from externalController-mode piraeus), drop the unsupported pod-CIDR claim, and adjust severity (P3 — cosmetic / fragility, not a DRBD outage). 3. Optional follow-up: add a contract test asserting that `snapshot_fetcher.peerAddr()` resolves correctly when the only NetInterface is named `default-ipv4`. That pins fix (A) and would have caught this earlier. diff --git a/docs/cli-parity-known-deltas.md b/docs/cli-parity-known-deltas.md index 698670d0..d84fdeef 100644 --- a/docs/cli-parity-known-deltas.md +++ b/docs/cli-parity-known-deltas.md @@ -60,6 +60,7 @@ Row IDs match the command-catalogue indexes used by `cli-parity-refresh.sh` (see | 42 | `r d ` (idempotent resource delete) | WIRE_SHAPE | permanent | BLOCKSTOR idempotent-delete envelope. Deleting a resource placement on a node that holds none returns `SUCCESS: resource already absent: on `; upstream LINSTOR returns `WARNING: Node: , Resource: not found`. Both exit 0. Deliberate: a CSI `DeleteVolume` / operator retry on an already-removed placement MUST be idempotent. Same family as the snapshot-delete idempotency in row 33. Pinned by the resource-delete REST handler's already-absent path (`pkg/rest/resources.go`). | | 40 | `n c --node-type Satellite` (node-create success envelope) | WIRE_SHAPE | permanent | BLOCKSTOR envelope shape on satellite node registration. BS emits `node created: ` + a `SUCCESS: No active connection to satellite ''` line; upstream emits `New node '' registered.` (with a UUID detail) + a `WARNING: No active connection to satellite ''` line whose Details explain the controller will (re-)establish the connection. Both register the node and exit 0; the operator-visible outcome (node exists, awaiting satellite handshake) is identical. The "no active connection" notice is INFO/SUCCESS-class in BS vs WARNING-class upstream, and BS does not surface the volatile node UUID inline. Envelope-shape only; no behavioural divergence. | | 16 | `ps l` (physical-storage list — `size` field) | WIRE_SHAPE | 2026-12-31 | BS `/v1/physical-storage` omits the per-device `size` key that python-linstor's `show_physical_storage` reads unconditionally (`linstor/responses.py` `devices.size`), so `linstor ps l` raises `KeyError: 'size'` and exits 2 against BS where upstream renders the table and exits 0. `ps l` is a hardware-discovery convenience surface; linstor-csi / piraeus-operator never call it, and the device-pool creation path (`ps create-device-pool`, cli-matrix `ps-cdp-*`) is unaffected. Tracked as a missing wire field to populate on the physical-storage DTO; whitelisted until then. | +| 86 | `node interface modify --active` | BEHAVIOR | permanent | blockstor has no satellite-to-controller connection to re-target: the satellite talks only to the Kubernetes API through the in-cluster Service. `NodeNetInterface` carries no `Active` field, and the REST `is_active` flag is synthesised on read as "first interface", so `--active` changes nothing. To make another interface active, reorder `spec.netInterfaces` so it comes first; a changed address reaches the node's DRBD configuration after its satellite pod restarts. Pinned by `TestSatelliteFlagsLackControllerBindAddress` and `TestNodeCRDHasNoActiveField` in `pkg/satellite/stream/redial_spec_test.go`. | ## Open (block merge until addressed) diff --git a/docs/csi-sanity-classification.md b/docs/csi-sanity-classification.md index c7150a1e..2c884779 100644 --- a/docs/csi-sanity-classification.md +++ b/docs/csi-sanity-classification.md @@ -10,7 +10,7 @@ Fix (this PR): relax the apiserver validator to mirror upstream, AND make the Re The residual failures are unchanged from the table below (all class (a) Node/satellite-convergence + class (b) apiserver scale-out cache race + Bug 33 satellite-snapshot finalizer drain in AfterEach teardown — the latter affects lowercase snapshots too). -Prior baseline (pre-audit): **56/74 specs pass** (74 ran; 17 skipped; 1 pending) after seven wire-shape gaps and three concurrency races were closed. The same job-config is used here. The audit collected 9 hard failures during the first 16 seconds of the run before the harness wedged on a snapshot-data-plane test (`sanity-controller-source-vol-*`) waiting for a satellite-side reconcile that never converged on the e2e-lc1 stand (Bug 33 territory — see `docs/known-issues.md`). Wire-shape and routing observations cover all 18 expected failures from the prior baseline. +Prior baseline (pre-audit): **56/74 specs pass** (74 ran; 17 skipped; 1 pending) after seven wire-shape gaps and three concurrency races were closed. The same job-config is used here. The audit collected 9 hard failures during the first 16 seconds of the run before the harness wedged on a snapshot-data-plane test (`sanity-controller-source-vol-*`) waiting for a satellite-side reconcile that never converged on the e2e-lc1 stand (Bug 33 territory, since closed by the orphan DRBD sweeper). Wire-shape and routing observations cover all 18 expected failures from the prior baseline. ## Failure-class legend @@ -60,6 +60,6 @@ So the **post-fix expected pass rate stays at 56/74**. The new finding from this 2. **CreateSnapshot empty-name validation** (c, #12-#13) — single check, flips two FAILs to PASS. Surfaces a real bug-class (silent slug-of-empty). 3. **DeleteSnapshot folds unknown-RD into success** (b, #14) — one-liner, flips one FAIL to PASS, closes a CSI idempotence corner. 4. **ListSnapshots empty-result envelope + exact-fit pagination** (c, #15-#16) — small handler tweaks, flips two FAILs to PASS. -5. **Bug 33 satellite-finalizer storm** — not a csi-sanity gap per se, but the audit blocker for re-running the suite cleanly. Already tracked in `docs/known-issues.md` (Bug 33 — closed by 5.34 sweeper; the storm seen on this run was a residual from the controller crashloop before Bug 41 RBAC was applied to the live stand). +5. **Bug 33 satellite-finalizer storm** — not a csi-sanity gap per se, but the audit blocker for re-running the suite cleanly. Tracked as Bug 33 (closed by 5.34 sweeper; the storm seen on this run was a residual from the controller crashloop before Bug 41 RBAC was applied to the live stand). After (1)–(4) the expected pass rate is **64/74**. The remaining 10 failures are all (a) (kernel-mount Node Service + satellite-convergence snapshot/clone tests) and are out of scope for the single-pod csi-sanity harness; running them needs a real piraeus-operator deployment with a CSI Node sidecar per worker and a stand with reliable satellite reconcile. diff --git a/docs/e2e-audit.md b/docs/e2e-audit.md index 5cce8a35..a032c9b2 100644 --- a/docs/e2e-audit.md +++ b/docs/e2e-audit.md @@ -38,7 +38,7 @@ Each row names the Tier 2 test(s) that exercise the same REST/CRD/reconciler pat | `placement-label-sync.sh` | NodeLabelSyncReconciler mirroring k8s labels → `Aux/...` props; envtest IS real k8s API. | Group A `TestGroupANodeAuxLabelSync` | | `recovery-false-diskless.sh` | Today asserts 404 on `r mkavail` (handler not wired). Pure wire-shape probe — no kernel-state contract. | Group F wire-shape probes; will move to Tier 4 when the handler lands and the test becomes "adopt existing on-disk state" | -`lc-rd-delete-churn.sh` is **kept** despite touching the same handlers — it runs 10 iterations and watches `dmesg`/`zfs list` for orphan ZVOLs and `node_id of X instead of Y` kernel warnings (cited by `docs/known-issues.md` as the validation source for the storage-sweeper). That assertion needs a live cluster. +`lc-rd-delete-churn.sh` is **kept** despite touching the same handlers — it runs 10 iterations and watches `dmesg`/`zfs list` for orphan ZVOLs and `node_id of X instead of Y` kernel warnings (the validation source for the storage-sweeper). That assertion needs a live cluster. --- diff --git a/docs/known-issues.md b/docs/known-issues.md deleted file mode 100644 index 899333d0..00000000 --- a/docs/known-issues.md +++ /dev/null @@ -1,323 +0,0 @@ -# Known Issues - -This document tracks outstanding bugs and observations surfaced during the scenario/e2e validation session. Each entry references the scenario or test where it was first observed, plus the closing commit if any. - -Severity legend: - -- **P0** — data corruption, split-brain, lost volumes, or cluster-wide unavailability -- **P1** — feature unusable, retry/cancel paths missing, regression vs upstream LINSTOR -- **P2** — cosmetic, log noise, doc/RBAC drift, ergonomic gaps - -## Bug 32: force-strip ZVOL cleanup race - -**Status**: open (observation, not a defect per se) -**Severity**: P2 -**Scenario reference**: tests/scenarios/05-finalizers.md §5.34 -**Surfaced by**: scenario -**Reproduction steps**: - -1. Create a Resource with a ZFS-backed VolumeDefinition on a single node. -2. Force-delete the Resource CR (`kubectl delete --grace-period=0 --force`) while the satellite is still mid-`drbdsetup down`. -3. Observe the underlying ZVOL on the host (`zfs list -t volume`). - -**Expected behaviour**: ZVOL is destroyed in lock-step with the DRBD resource teardown; no stray dataset survives the CR removal. - -**Actual behaviour**: A short window exists where the CR is gone but the ZVOL still resides on disk because the satellite finalizer was bypassed. The next reconcile or operator restart eventually GCs it, but for a few seconds the storage pool reports phantom capacity. - -**Recommended fix**: No code change required. Document the window and rely on Bug 33 (DRBD orphan sweeper) plus Bug 43 (storage orphan sweeper) to handle the residual cleanup deterministically. - -**Related commits / tests**: see Bug 33 (DRBD sweeper) and Bug 43 (storage sweeper). - -## Bug 33: orphan kernel DRBD-Diskless after force-strip - -**Status**: closed (by Bug 5.34 sweeper) -**Severity**: P1 -**Scenario reference**: tests/scenarios/05-finalizers.md §5.34 -**Surfaced by**: scenario -**Reproduction steps**: - -1. Create a 3-replica Resource (2 disk + 1 diskless tiebreaker). -2. Force-delete the tiebreaker Resource CR before the satellite finalizer drains. -3. Inspect `drbdsetup status` on the former diskless node. - -**Expected behaviour**: Kernel DRBD device on the diskless node is fully removed; `drbdsetup status` no longer lists it. - -**Actual behaviour**: Kernel resource remained in `Diskless` state with no corresponding CR, preventing subsequent recreation on the same node (port/minor collision). - -**Recommended fix**: Periodic sweeper on the satellite enumerates kernel DRBD resources and reconciles against the local CR cache; any kernel resource without a matching CR is `drbdsetup down`-ed. - -**Related commits / tests**: 5.34 sweeper landed in the satellite controller; e2e covers via scenario 05 force-strip path. - -## Bug 34: migrate-disk endpoint missing - -**Status**: closed (by 05b8709); Option B follow-up open -**Severity**: P1 -**Scenario reference**: tests/scenarios/07-toggle-disk.md §7.4 -**Surfaced by**: e2e test -**Reproduction steps**: - -1. Create a 2-replica Resource on nodes A and B. -2. Issue `linstor resource-definition migrate-disk `. - -**Expected behaviour**: REST handler accepts the request, schedules a synchronous disk migration (toggle-disk on, then off on source) without losing redundancy. - -**Actual behaviour** (pre-fix): Endpoint returned 404; the upstream LINSTOR client could not orchestrate disk migration through the apiserver. - -**Recommended fix (landed)**: REST handler `migrate-disk` wired in 05b8709 to a two-step toggle-disk pipeline (add on target → wait UpToDate → remove on source). - -**Option B follow-up (open)**: Wrap the two-step sequence in a single reconciler-driven state machine so an apiserver restart mid-migration does not strand the user with an extra diskful replica. Track at next sprint. - -**Related commits / tests**: 05b8709; scenario 07 §7.4 covers the happy path. - -## Bug 35: placer FreeCapacity gate missing - -**Status**: open -**Severity**: P0 -**Scenario reference**: tests/scenarios/08-placer.md §8.2 -**Surfaced by**: scenario -**Reproduction steps**: - -1. Create two storage pools with vastly different free capacity (e.g. 100 GiB and 1 GiB free). -2. Create a Resource of size 10 GiB without an explicit node selector. - -**Expected behaviour**: Placer rejects the small pool because the requested size exceeds its FreeCapacity and chooses the larger pool (or fails fast with a clear error). - -**Actual behaviour**: Placer ignores FreeCapacity entirely and may select the smaller pool, leading to `ENOSPC` at ZVOL/LV create time and a stuck Resource in `Creating`. - -**Recommended fix**: Add a hard FreeCapacity gate inside the candidate-pool filter before scoring. Treat pools with `free < requested + reservedHeadroom` as ineligible. Add a placer unit test with two pools at different free sizes. - -**Related commits / tests**: none yet; blocker for placer 1.0. - -## Bug 36: handleVDUpdate wholesale-replace - -**Status**: open -**Severity**: P1 -**Scenario reference**: tests/scenarios/09-vd-update.md §9.1 -**Surfaced by**: agent -**Reproduction steps**: - -1. Create a VolumeDefinition with `props: {a: 1, b: 2}`. -2. PATCH the VD with `override_props: {a: 9}` (intent: change only `a`). - -**Expected behaviour**: Final props are `{a: 9, b: 2}` — only the keys named in `override_props` are touched. - -**Actual behaviour**: `handleVDUpdate` rewrites the entire props map from the request body, so `b` is silently dropped. - -**Recommended fix**: Replace the wholesale assignment with a merge: apply `override_props` keys on top of the existing map, then remove keys named in `delete_props`. Mirrors the upstream LINSTOR controller semantics and is the precondition for Bug 37. - -**Related commits / tests**: none; depends on Bug 37. - -## Bug 37: VolumeDefinitionModify override_props/delete_props no-op - -**Status**: open -**Severity**: P1 -**Scenario reference**: tests/scenarios/09-vd-update.md §9.2 -**Surfaced by**: e2e test -**Reproduction steps**: - -1. Apply `VolumeDefinitionModify` with `override_props: {DrbdOptions/Net/protocol: C}`. -2. GET the VD and inspect `props`. - -**Expected behaviour**: `DrbdOptions/Net/protocol` is set to `C`; `delete_props` removes named keys. - -**Actual behaviour**: Both fields are accepted by the REST schema but ignored by the handler — the props map is unchanged. - -**Recommended fix**: Wire `override_props` and `delete_props` through the same merge path introduced by Bug 36's fix. Add e2e assertions that round-trip both add and remove. - -**Related commits / tests**: depends on Bug 36; scenario 09 §9.2 has the assertion stub. - -## Bug 38: VD shrink accepted without STATE_INFO warning - -**Status**: open -**Severity**: P2 (cosmetic) -**Scenario reference**: tests/scenarios/09-vd-update.md §9.3 -**Surfaced by**: scenario -**Reproduction steps**: - -1. Create a VD of size 10 GiB. -2. PATCH the VD to size 5 GiB. - -**Expected behaviour**: Either the request is rejected, or it is accepted with a `STATE_INFO` / warning in the response indicating that shrinking is unsafe and won't reclaim space on the backing pool until the FS is also shrunk. - -**Actual behaviour**: Request is accepted silently; no warning surfaces to the operator. Backing volume keeps its original allocation. - -**Recommended fix**: Append a `STATE_INFO` entry to the API response when `newSize < currentSize`. No semantic change to storage; purely advisory. - -**Related commits / tests**: none. - -## Bug 39: toggle-disk retry counter missing - -**Status**: open -**Severity**: P1 -**Scenario reference**: tests/scenarios/07-toggle-disk.md §7.6 -**Surfaced by**: agent -**Reproduction steps**: - -1. Initiate `toggle-disk` on a Resource where the target node's storage pool is briefly unavailable. -2. Observe reconciler logs. - -**Expected behaviour**: Reconciler retries with bounded backoff and a visible retry counter on the Resource status; after N attempts it surfaces a terminal error condition. - -**Actual behaviour**: Reconciler retries forever with no visibility; if the failure is permanent, the Resource is stuck "in flight" indefinitely. - -**Recommended fix**: Add `status.toggleDisk.attempts` and `status.toggleDisk.lastError`. Cap at e.g. 10 attempts before transitioning to `Failed` with a clear condition. Couple with Bug 40. - -**Related commits / tests**: none; precondition for the migrate-disk Option B state machine. - -## Bug 40: toggle-disk cancel state machine missing - -**Status**: open -**Severity**: P1 -**Scenario reference**: tests/scenarios/07-toggle-disk.md §7.7 -**Surfaced by**: agent -**Reproduction steps**: - -1. Start a long-running `toggle-disk` (e.g. large initial sync). -2. Issue a cancel/abort via the REST API. - -**Expected behaviour**: Reconciler observes the cancel intent, rolls back the partial change (removes the half-added diskful, or re-adds the half-removed one) and returns the Resource to its pre-toggle state. - -**Actual behaviour**: No cancel endpoint exists; the only escape is to delete the Resource entirely. - -**Recommended fix**: Model toggle-disk as an explicit state machine (`Pending → Syncing → Promoting → Done | Cancelling → Rolled-back`). Add a cancel verb in the REST surface and have the reconciler honour it idempotently. - -**Related commits / tests**: none; pairs with Bug 39. - -## Bug 41: RBAC nodes get/list/watch missing in stand yaml - -**Status**: closed (13722e1) -**Severity**: P2 -**Scenario reference**: tests/scenarios/01-bootstrap.md §1.3 -**Surfaced by**: e2e test (stand bring-up) -**Reproduction steps**: - -1. Deploy blockstor on the dev stand with the previous controller-manifests yaml. -2. Watch controller logs at startup. - -**Expected behaviour**: Controller starts and lists Node objects to seed its topology cache. - -**Actual behaviour** (pre-fix): `nodes "" is forbidden: User "system:serviceaccount:..." cannot list resource "nodes" in API group "" at the cluster scope`; controller crashloops. - -**Recommended fix (landed)**: Add `nodes` verbs `get/list/watch` to the ClusterRole shipped with the stand yaml. - -**Related commits / tests**: 13722e1. - -## Bug 42: piraeus pod-CIDR drift on e2e-iptables - -**Status**: open (investigation pending) -**Severity**: P1 -**Scenario reference**: tests/scenarios/10-e2e-iptables.md §10.1 -**Surfaced by**: e2e test -**Reproduction steps**: - -1. Bring up the e2e-iptables Talos+QEMU stand via `make iter`. -2. Deploy piraeus-operator with the default values. -3. Wait for the satellite DaemonSet pods to schedule and the LINSTOR controller to register them. - -**Expected behaviour**: Satellite pod IPs come from the cluster's pod CIDR and are routable to/from the LINSTOR controller; resources reach `UpToDate` without DRBD net errors. - -**Actual behaviour**: Some satellite pods receive addresses outside the expected pod CIDR (suspected CNI bridge / kube-proxy iptables drift on the iptables-mode stand), DRBD peer connections flap, resources oscillate between `Connecting` and `Connected`. - -**Recommended fix**: Investigation pending. First steps: - -- Confirm whether the drift is at the CNI layer (bridge IPAM) or kube-proxy (SNAT/MASQUERADE on the iptables stand). -- Capture `ip a`, `iptables-save`, and `kubectl get pods -o wide` on a failing run. -- Compare against the ipvs-mode stand where the issue does not reproduce. - -**Related commits / tests**: scenario 10 §10.1 reproducer; no fix yet. - -## Bug 49: satellite ignores runtime NetInterface changes (deferred from scenario 3.10) - -**Status**: open (deferred — design gap, not a defect) -**Severity**: P2 -**Scenario reference**: tests/scenarios/03-networking.md §3.10 -**Surfaced by**: scenario audit (UG9 §"Managing network interface cards", lines 2167-2169) -**Reproduction steps**: - -1. `kubectl edit node.blockstor.io/` and re-order the entries under `spec.netInterfaces` (or change the first entry's `address`). -2. Observe the running satellite Pod on that worker — it does not re-dial anything, does not re-render its DRBD `on ` block, and continues talking to the kube-apiserver via the same Service VIP. -3. Restart the satellite Pod (`kubectl delete pod -l app=blockstor-satellite,kubernetes.io/hostname=`) — only on restart does the new `Spec.NetInterfaces[0].Address` flow into `LocalAddress` (via `$POD_IP`) and from there into the next reconcile-rendered .res file. - -**Expected behaviour (upstream LINSTOR contract)**: `linstor node interface modify ... --active` flips `StltConn/0/Active=true` and the satellite re-dials the controller via the freshly-active NIC without a process restart. - -**Actual behaviour (blockstor)**: No re-dial happens, and there is no field to flip. Two architectural reasons: - -1. Phase 10.6 retired the satellite→controller gRPC wire entirely. The satellite now talks to the kube-apiserver via the standard in-cluster config (`ctrl.GetConfig()` → `kubernetes.default.svc` Service VIP). There is no LINSTOR-style "controller endpoint" left to re-target. -2. `api/v1alpha1.NodeNetInterface` has no `Active` field. The `IsActive` flag on the REST wire is synthesised at conversion time (`i == 0` in `pkg/store/k8s/nodes.go`) — pure presentation, no behaviour. The only satellite code that reads peer NetInterfaces at runtime is `pkg/satellite/controllers/snapshot_fetcher.go` (peer-to-peer snapshot transport), and it picks by `Name == "default"` or first non-empty `Address`, not by an Active flag. - -**Recommended fix**: Deferred. Implementing Outcome A (live NIC switch) would require: - -- Adding an `Active` field to `NodeNetInterface` and a controller-runtime watch in the satellite that triggers reconciliation when `Spec.NetInterfaces[].Active` changes. -- A redirect plane that the satellite actually re-routes through — but the satellite's only outbound wire is the kube-apiserver REST/watch, which is owned by client-go and routes via the cluster Service VIP rather than this node's NIC selection. -- For the satellite→satellite snapshot stream the change would be marginal (peer-side already picks Address dynamically per Fetch call). - -Net: implementing the upstream contract requires either resurrecting a custom controller-bind wire (rejected by the Phase 10.6 design) or accepting that "active interface" is now a pure presentation field that operators must not rely on for connection re-targeting. - -The spec is pinned by `TestSatelliteFlagsLackControllerBindAddress` and `TestNodeCRDHasNoActiveField` in `pkg/satellite/stream/redial_spec_test.go`. If a future change implements Outcome A, both tests will fail and force the reviewer to replace them with a positive re-dial assertion. - -**Related commits / tests**: `pkg/satellite/stream/redial_spec_test.go`; tests/scenarios/03-networking.md §3.10. - -## Bug 43: mass-churn ZVOL leak on force-stripped finalizers - -**Status**: closed (by storage-orphan sweeper) -**Severity**: P0 -**Scenario reference**: tests/e2e/lc-rd-delete-churn.sh (scenario 5.4 extended — 10 RD rapid create/delete on one node) -**Surfaced by**: e2e test - -**Reproduction steps**: - -1. Run `lc-rd-delete-churn.sh` with `ITERS=10` against a 2-node stand. -2. Inject a satellite stall (e.g. SIGSTOP on the satellite pod) for the iteration's DRBD-down phase so the REST controller times out the per-iter delete and force-strips the Resource CRD finalizer. -3. After the loop, `zfs list -t volume` on each node. - -**Expected behaviour**: All 10 ZVOLs are destroyed in lock-step with the Resource CRD deletion; the final `zfs list -t volume` shows no `churn-N_NNNNN` datasets. - -**Actual behaviour** (pre-fix): 9 of 10 ZVOLs leak. The REST controller stamps `DeletionTimestamp` on the per-node Resource CRD; the satellite finalizer is mid-`drbdsetup down` so doesn't release the finalizer in time; the controller force-strips the finalizer; the satellite's `handleDelete` (which would have called `provider.DeleteVolume`) never runs; the ZVOL survives indefinitely. Bug 32 documents the short-window variant; Bug 43 is the deterministic mass-churn variant where the satellite genuinely cannot keep up and force-strip becomes the steady-state path. - -**Recommended fix (landed)**: New `StorageOrphanSweeperRunnable` in `pkg/satellite/controllers/storage_sweeper.go` periodically (every 5 minutes, same cadence as the DRBD orphan sweeper for Bug 33) walks every registered `storage.Provider` that implements the new `storage.VolumeLister` capability, diffs the on-disk volume list against the Resource CRDs scheduled to this node, and calls `provider.DeleteVolume` on each volume whose owning CRD has fully vanished (force-strip aftermath). - -Safety belts on the new path: - -- Wildcard-owned protection: a Resource CRD whose RD has been cascade-deleted out of order still counts as "owned" (volume number wildcard) so the sweeper never races `handleDelete`. -- Per-cycle delete bound (`StorageSweeperMaxDeletePerCycle = 3`) so a pathological upstream producer of orphans surfaces in logs without being silently cleaned in one tick. -- Storage-resource prefix allowlist (`pvc-`, `bs-`) so operator-created datasets that happen to share the pool are immune to the sweep. -- Per-Node opt-out annotation `blockstor.io/skip-orphan-storage-sweeper` for forensic-mode operators. -- `ListVolumeNames` skips ZFS deferred-delete markers (`__DELETED__`) and LVM snapshot LVs via the lv_attr column so the sweeper never confuses cleanup artefacts for orphans. - -Idempotency of `provider.DeleteVolume` was already guaranteed by Bug 33's contract; the sweeper relies on that — racing the satellite's own `handleDelete` is harmless because the second call returns nil. - -**Related commits / tests**: `pkg/satellite/controllers/storage_sweeper.go` + `pkg/satellite/controllers/storage_sweeper_test.go` (7 unit tests covering owned-leave-alone, orphan-reap, foreign-prefix immunity, per-node scope, rate-limit, skip-annotation, mid-delete-race protection, missing-RD wildcard protection); `pkg/storage/storage.go` adds the `VolumeLister` optional interface + `VolumeRef`; ZFS / LVM-thin / LVM-thick providers implement it. E2e validation via `lc-rd-delete-churn.sh` (passes 10 iters without ZVOL leak). - -## Bug 50: concurrent rapid late `vd c` wedges or drops the second volume - -**Status**: closed (#164 converge + #168 resize-deadlock guard; v0.1.14) (campaign tracking id: BUG-048) -**Severity**: P1 (availability; NOT data-loss, NOT a node-reboot deadlock; recoverable) -**Scenario reference**: tests/e2e/cli-matrix/multi-volume-late-vd-create.sh -**Surfaced by**: release-gate validation campaign -**Reproduction steps**: - -1. Create a multi-replica ResourceDefinition and let it converge (at least one VolumeDefinition, all replicas UpToDate). -2. Issue two `linstor volume-definition create ` calls back-to-back in rapid succession (concurrently, or scripted with no wait between them). - -**Expected behaviour**: Both volume-definitions are created; each new volume is laid down on every replica and reaches `UpToDate` (with one replica electing SyncSource), exactly as a sequential add would. - -**Actual behaviour**: Intermittently one of two faults occurs — (a) the second `vd c` returns SUCCESS but the VolumeDefinition is silently dropped (only one VD ends up persisted), or (b) the second volume is created but wedges `Inconsistent` with no SyncSource elected, so it never reaches `UpToDate`. In case (a) both calls report success. - -**Scope / reachability**: NOT reachable through the production CSI path. linstor-csi maps one PVC to one ResourceDefinition with exactly one VolumeDefinition, created in a single request; autoplace and `ControllerPublishVolume` add no VolumeDefinitions, and resize grows the existing volume in place. The fault is reachable only when an operator (or a script) issues concurrent / rapid back-to-back manual `vd c` calls against an existing multi-replica RD. No data is lost or corrupted (md5 held across all validation runs) and no node reboot is required. - -**Recovery**: Delete the wedged or missing volume-definition (`linstor vd d `) and re-add it sequentially — issue one `vd c`, wait for it to reach `UpToDate`, then issue the next. Always add late volume-definitions one at a time. - -**Recommended fix**: Serialise concurrent VolumeDefinition creation on a given ResourceDefinition (optimistic-concurrency precondition on the RD volume-minor allocation plus the auto-numbered VD create) so the second add observes the first, and seed the new volume's GI / current-UUID before peers attach so a deterministic SyncSource is elected. A first attempt (PR #164) reduced the silent-drop and fixed a day0 split-brain leg but regressed the CSI-reachable single-late-VD path, so it was not merged; a corrected fix that does not touch the single-VD path is tracked. - -**Related commits / tests**: cli-matrix cell `tests/e2e/cli-matrix/multi-volume-late-vd-create.sh` honestly catches this (as of #162); PR #164 (deferred). No closing commit yet. - -## Recommended next-fix order - -1. **Bug 35** (P0, placer FreeCapacity) — only P0; blocks placer 1.0 and risks ENOSPC at create time. -2. **Bug 42** (P1, piraeus pod-CIDR drift) — blocks the iptables-mode e2e lane. -3. **Bug 36 + 37** (P1, VD props merge) — fix together; 37 depends on 36's merge plumbing. -4. **Bug 39 + 40** (P1, toggle-disk retry/cancel) — fix together; together they unlock Bug 34's Option B state machine. -5. **Bug 50** (FIXED in v0.1.14, #164+#168) — serialise VolumeDefinition create on the RD and seed GI before peer attach; operator-only path, recoverable, not data-loss. Closed by #164 (converge) + #168 (resize-deadlock guard). -6. **Bug 34 Option B** (P1 follow-up) — wrap migrate-disk in the new state machine once 39/40 land. -7. **Bug 38** (P2, cosmetic STATE_INFO on shrink) — pure UX, do last. -8. **Bug 32** (P2, observation) — document only; no code fix needed. diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 00000000..b5352b50 --- /dev/null +++ b/netlify.toml @@ -0,0 +1,37 @@ +# Netlify builds deploy previews of the blockstor.org site for pull +# requests. Production is GitHub Pages (.github/workflows/pages.yml). +# +# The site is Hugo with the Docsy theme under site/; base moves the build +# there, so every path below is relative to site/. + +[build] + base = "site" + publish = "public" + command = "npm install && hugo --gc --minify" + # Skip the preview when nothing under site/ or this file changed. The + # command runs from the base directory and exit 0 skips the build, so a + # missing or identical cached ref forces one. + ignore = ''' + if [ -z "$CACHED_COMMIT_REF" ] || [ "$CACHED_COMMIT_REF" = "$COMMIT_REF" ]; then exit 1; fi + git diff --quiet "$CACHED_COMMIT_REF" "$COMMIT_REF" -- . ../netlify.toml + ''' + +[build.environment] + HUGO_VERSION = "0.164.0" + HUGO_EXTENDED = "true" + NODE_VERSION = "22" + GO_VERSION = "1.23" + +[context.production.environment] + HUGO_ENV = "production" + +[context.deploy-preview] + command = "npm install && hugo --gc --minify -b $DEPLOY_PRIME_URL" + +[context.branch-deploy] + command = "npm install && hugo --gc --minify -b $DEPLOY_PRIME_URL" + +[[redirects]] + from = "/*" + to = "/404.html" + status = 404 diff --git a/pkg/satellite/stream/redial_spec_test.go b/pkg/satellite/stream/redial_spec_test.go index 5b9d7429..b29b2003 100644 --- a/pkg/satellite/stream/redial_spec_test.go +++ b/pkg/satellite/stream/redial_spec_test.go @@ -69,8 +69,8 @@ package stream_test // Tracking: // // - tests/scenarios/03-networking.md §3.10 -// - docs/known-issues.md → "Bug 49: satellite ignores runtime -// NetInterface changes (deferred from scenario 3.10)" +// - docs/cli-parity-known-deltas.md → row 86, "node interface +// modify --active" (formerly Bug 49, deferred from scenario 3.10) // // Outcome A (active re-dial loop) was considered and rejected: the // blockstor architecture does not have a satellite→controller dial diff --git a/site/README.md b/site/README.md new file mode 100644 index 00000000..0324fc1a --- /dev/null +++ b/site/README.md @@ -0,0 +1,11 @@ +# blockstor.org + +The project website and documentation, built with [Hugo](https://gohugo.io/) and the [Docsy](https://www.docsy.dev/) theme. Production is published to GitHub Pages from `main`; pull requests get a Netlify deploy preview. + +## Run it locally + +You need Hugo extended (0.161 or newer), Go and Node.js 22. From this directory run `npm install` once, then `hugo server` and open http://localhost:1313/. + +## Where things go + +Documentation pages live under `content/en/docs/`, one markdown file per page with `title`, `weight` and `description` in the front matter; the weight sets the order in the sidebar. The landing page is `content/en/_index.md`. Write one continuous line per paragraph. diff --git a/site/assets/icons/logo.svg b/site/assets/icons/logo.svg new file mode 100644 index 00000000..1e4fb76a --- /dev/null +++ b/site/assets/icons/logo.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/site/assets/scss/_styles_project.scss b/site/assets/scss/_styles_project.scss new file mode 100644 index 00000000..239a91c6 --- /dev/null +++ b/site/assets/scss/_styles_project.scss @@ -0,0 +1,113 @@ +// The logo is the full wordmark, so the site name next to it is dropped. +.navbar-brand__name { + display: none; +} + +.navbar-brand__logo svg { + height: 1.75rem; + width: auto; +} + +// The replication figure on the cover. It sits on the primary background, so +// it is drawn in light strokes in both themes. +.replicas { + margin: 1rem auto 2rem; + max-width: 30rem; + + svg { + display: block; + width: 100%; + font-family: $font-family-monospace; + } + + text { + fill: #fff; + font-size: 13px; + } + + .sub { + fill: rgba(255, 255, 255, 0.7); + font-size: 12px; + } + + .crd rect { + fill: rgba(255, 255, 255, 0.08); + stroke: #fff; + stroke-width: 1.5; + } + + .node > rect:first-child { + fill: rgba(255, 255, 255, 0.06); + stroke: rgba(255, 255, 255, 0.35); + stroke-width: 1.5; + } + + .blk { + fill: #93a4ff; + } + + .links path { + fill: none; + stroke: rgba(255, 255, 255, 0.6); + stroke-width: 1.5; + } + + .sync path { + fill: none; + stroke: #93a4ff; + stroke-width: 2; + stroke-dasharray: 4 4; + } + + figcaption { + margin-top: 1rem; + color: rgba(255, 255, 255, 0.75); + font-size: 0.95rem; + } +} + +@media (prefers-reduced-motion: no-preference) { + .replicas .sync path { + animation: replicas-sync 1.6s linear infinite; + } +} + +@keyframes replicas-sync { + to { + stroke-dashoffset: -16; + } +} + +.cncf-block .container { + padding-top: 4rem; + padding-bottom: 3rem; +} + +.cncf-logo { + width: 50%; + max-width: 36rem; + height: auto; +} + +// Docsy sets data-bs-theme on ; the white CNCF logo replaces the colour +// one on the dark background. +.cncf-logo--light { + display: block; +} + +.cncf-logo--dark { + display: none; +} + +[data-bs-theme="dark"] { + .cncf-logo--light { + display: none; + } + + .cncf-logo--dark { + display: block; + } +} + +// Code blocks follow the light and dark theme (needs highlight.noClasses: false). +@import "td/code-dark"; diff --git a/site/assets/scss/_variables_project.scss b/site/assets/scss/_variables_project.scss new file mode 100644 index 00000000..beb25a96 --- /dev/null +++ b/site/assets/scss/_variables_project.scss @@ -0,0 +1,3 @@ +// Brand colour taken from the logo. +$primary: #1b2a4a; +$secondary: #3b5b8c; diff --git a/site/content/en/_index.md b/site/content/en/_index.md new file mode 100644 index 00000000..c9d95297 --- /dev/null +++ b/site/content/en/_index.md @@ -0,0 +1,49 @@ +--- +title: Blockstor +--- + +{{< blocks/cover image_anchor="top" height="auto" color="primary" >}} +
+
+

Blockstor

+

A Kubernetes control plane for LVM and ZFS storage with DRBD replication.

+Get started +GitHub +
+
+{{< replicas >}} +
+
+{{< /blocks/cover >}} + +{{% blocks/lead color="white" %}} +Blockstor speaks a LINSTOR-compatible REST API, so the clients you already use keep working unchanged: `linstor-csi`, `piraeus-operator`, `ha-controller`, `golinstor` and the `linstor` CLI. + +The difference is underneath. Desired state lives in CRDs, and a set of `controller-runtime` reconcilers drive the cluster toward it. There is no external database to back up, no in-memory state to lose on restart, and no controller-to-node polling to fall behind. +{{% /blocks/lead %}} + +{{% blocks/section color="light" type="row" %}} + +{{% blocks/feature icon="fa-solid fa-dharmachakra" title="Kubernetes-native" %}} +The state of truth lives in Kubernetes CRDs. `Resource`, `ResourceDefinition`, `ResourceGroup`, `StoragePool`, `Snapshot`, `Node` and `PhysicalDevice` are meant to be read, and where it makes sense written, by other operators. Each satellite watches the CRDs for its own node and reports what it observes through Status, with no gRPC dispatch from a central controller. +{{% /blocks/feature %}} + +{{% blocks/feature icon="fa-brands fa-golang" title="Ecosystem fit" %}} +Go is what the apiserver, the kubelet, etcd, most CSI drivers and `controller-runtime` are written in, so blockstor shares their tooling, libraries and contributors. +{{% /blocks/feature %}} + +{{% blocks/feature icon="fa-solid fa-feather" title="Small footprint" %}} +The controller, the apiserver and the satellite are statically linked Go binaries, with no JVM, and DRBD is a Linux kernel module rather than a userspace daemon. The images are multi-arch, `linux/amd64` and `linux/arm64`, so blockstor fits on edge and small ARM nodes. +{{% /blocks/feature %}} + +{{% /blocks/section %}} + +{{< cncf >}} + +{{% blocks/section color="light" %}} +## Acknowledgements {.text-center} + +Blockstor was inspired by LINBIT's [LINSTOR](https://linbit.com/linstor/), and it operates [DRBD](https://linbit.com/drbd/) to provide block-level replication. It deliberately speaks LINSTOR's API so that the ecosystem the LINSTOR community has built keeps working unchanged. Heartfelt thanks to LINBIT and to the wider DRBD, LINSTOR and Piraeus community. + +Blockstor is licensed under [Apache 2.0](https://github.com/cozystack/blockstor/blob/main/LICENSE). LINSTOR, LINBIT and DRBD are trademarks or registered trademarks of LINBIT. Blockstor is an independent project and is not affiliated with, endorsed by, or sponsored by LINBIT. +{{% /blocks/section %}} diff --git a/site/content/en/developer-guide/_index.md b/site/content/en/developer-guide/_index.md new file mode 100644 index 00000000..ae25ed3f --- /dev/null +++ b/site/content/en/developer-guide/_index.md @@ -0,0 +1,21 @@ +--- +title: "Developer guide" +linkTitle: "Developer guide" +weight: 30 +menu: + main: + weight: 20 +description: "Build blockstor, run its tests, and find your way around the code." +cascade: + type: docs +--- + +This section is for people who work on blockstor itself. If you want to run blockstor, the [Documentation](/docs/) is the place to start. + +- [Building](building/): binaries, images and the local dev stand. +- [Testing](testing/): the test tiers, from unit tests to the operator CLI matrix. +- [Repository layout](repository/): where each part of the code lives. +- [Internals](internals/): the rules the code relies on, with references to it. +- [Contributing](contributing/): licensing rules, commits and pull requests. + +Design notes and audits that are useful background but not part of this guide live in the repository's [`docs/`](https://github.com/cozystack/blockstor/tree/main/docs) directory. diff --git a/site/content/en/developer-guide/building.md b/site/content/en/developer-guide/building.md new file mode 100644 index 00000000..8e6d4e24 --- /dev/null +++ b/site/content/en/developer-guide/building.md @@ -0,0 +1,67 @@ +--- +title: "Building" +linkTitle: "Building" +weight: 10 +description: "Build the binaries and images, and bring up the dev stand." +--- + +## Binaries + +Blockstor is a Go module, `github.com/cozystack/blockstor`. `make build` regenerates the manifests and deep-copy code, runs `go fmt` and `go vet`, and builds the controller, the satellite, the CLI and the migration tool into `bin/`. The API server builds the usual Go way: + +```sh +make build +go build -o bin/apiserver ./cmd/apiserver +``` + +| Binary | Source | What it is | +|---|---|---| +| `controller` | `cmd/controller` | controller-runtime manager with the controller-side reconcilers. | +| `apiserver` | `cmd/apiserver` | the stateless LINSTOR-compatible REST front end. | +| `satellite` | `cmd/satellite` | the per-node manager that drives DRBD, LUKS and storage. | +| `blockstor` | `cmd/blockstor` | the operator CLI. | +| `linstor-migrate` | `cmd/linstor-migrate` | the LINSTOR database converter. | + +Other generators: `make manifests` regenerates the CRDs and RBAC from the kubebuilder markers in `api/v1alpha1`, and `make generate` regenerates the deep-copy code. + +## Images + +One multi-stage [`Dockerfile`](https://github.com/cozystack/blockstor/blob/main/Dockerfile) builds all three images: the `controller`, `apiserver` and `satellite` targets. The controller and API server images are distroless; the satellite image is Debian-based, because it shells out to `drbdadm`, `lvs`, `zfs` and `cryptsetup`. + +Releases are built by [`release.yml`](https://github.com/cozystack/blockstor/blob/main/.github/workflows/release.yml) on a `v*` tag and pushed to `ghcr.io/cozystack/blockstor-{controller,apiserver,satellite}` for `linux/amd64` and `linux/arm64`. + +## The dev stand + +`stand/` brings up a throwaway Talos cluster in QEMU, with DRBD, ZFS and LVM in the node image, so you can exercise the real data path on your machine. + +Host requirements: + +- Linux x86_64 with KVM (`/dev/kvm` accessible); +- `talosctl`, `kubectl`, `helm`, `qemu-system-x86_64`; +- the DRBD 9 kernel module loaded on the host; +- about 8 GB of free RAM and 20 GB of disk per cluster. + +```sh +make up # create the cluster (default name "blockstor") +make piraeus # linstor-csi in external-controller mode +make blockstor # build and install blockstor +make smoke-blockstor # quick end-to-end check +make down +``` + +Real-disk pools on extra disks: + +```sh +make pools +STORPOOL=zfs-thin make smoke-blockstor +``` + +Several clusters can run side by side, each with its own `10..0.0/24` network. Their configuration lands in `.work//`: + +```sh +make up NAME=alice +eval "$(make use NAME=alice)" +kubectl get nodes +``` + +The stand pins every image by digest, so each rebuild rolls the pods. diff --git a/site/content/en/developer-guide/contributing.md b/site/content/en/developer-guide/contributing.md new file mode 100644 index 00000000..e83baf40 --- /dev/null +++ b/site/content/en/developer-guide/contributing.md @@ -0,0 +1,29 @@ +--- +title: "Contributing" +linkTitle: "Contributing" +weight: 50 +description: "Licensing rules, and what a change needs before it merges." +--- + +## Licensing: Apache 2.0, clean room + +Blockstor is Apache 2.0. Do not copy structure or code from GPL sources, in particular `linstor-server` and `drbd-utils`. You may read them to understand behaviour, never to reproduce how they are written. + +Hard rules: + +- No GPL-licensed spec files in the tree. LINSTOR's `rest_v1_openapi.yaml` declares GPLv3, and code generated from it is a derivative work. +- No code generated from a GPL source by any tool. +- No Go dependency under GPL, AGPL, LGPL or SSPL. CI allows only Apache-2.0, BSD-2-Clause, BSD-3-Clause, MIT, MPL-2.0 and ISC, through [`license-check.yml`](https://github.com/cozystack/blockstor/blob/main/.github/workflows/license-check.yml). +- Every Go source file carries an `SPDX-License-Identifier: Apache-2.0` header. + +Apache-2.0 projects that are safe to study and depend on: [`golinstor`](https://github.com/LINBIT/golinstor), the authoritative description of the REST wire shape; [`piraeus-operator`](https://github.com/piraeusdatastore/piraeus-operator); [`linstor-csi`](https://github.com/piraeusdatastore/linstor-csi); [`drbd-reactor`](https://github.com/LINBIT/drbd-reactor); and the public DRBD 9 documentation and man pages. + +When in doubt, work clean room: one person reads the GPL source and writes a behavioural spec, and another implements it without seeing the original. + +## Before you open a pull request + +- `go test ./...` passes and `make lint` is clean. +- A bug fix comes with a test at the lowest tier that catches it. A bug in the operator CLI also gets a cell in `tests/e2e/cli-matrix/`. +- A new CLI verb or a change to a wire shape is checked with the parity refresh, and an intended difference from upstream gets a row in [`docs/cli-parity-known-deltas.md`](https://github.com/cozystack/blockstor/blob/main/docs/cli-parity-known-deltas.md). +- User-facing behaviour is documented on this site, under `site/content/en/docs/`. +- Commits follow Conventional Commits and are signed off (`git commit --signoff`). diff --git a/site/content/en/developer-guide/internals.md b/site/content/en/developer-guide/internals.md new file mode 100644 index 00000000..dc43212e --- /dev/null +++ b/site/content/en/developer-guide/internals.md @@ -0,0 +1,77 @@ +--- +title: "Internals" +linkTitle: "Internals" +weight: 40 +description: "The rules the code relies on, with pointers into it." +--- + +This page is about the invariants whose violation would quietly corrupt cluster state. For the user-level picture, see [Architecture](/docs/architecture/). For a tour of the code, follow the call graph from `cmd/controller/main.go` and `cmd/satellite/main.go`. + +## Spec and Status + +Every blockstor CRD has two halves: + +- **Spec** is desired state. Users, REST handlers and the controller's placement write it. The satellite never writes Spec. +- **Status** is observed state. The satellite writes what it reads from the kernel (`drbdsetup events2`, `drbdadm status`, `lvs`), and the controller's allocators write values derived from Spec. The REST API never writes Status. + +A whole-object `Update` is unsafe whenever both halves have different writers: a Spec write in flight would overwrite a concurrent Status write, and the other way round. Status goes through the Status subresource. Fields with more than one writer, such as `Resource.Status.DRBDPort` from the controller next to `DiskState` and `CurrentGi` from the satellite, go through Server-Side Apply with separate field managers, `blockstor-controller` and `blockstor-satellite`, so each side owns only its own fields. + +| Field | Half | Why | +|---|---|---| +| `Resource.Spec.NodeName` | Spec | Placement target. | +| `Resource.Spec.Flags` | Spec | `DISKLESS` and `TIE_BREAKER` are requested, not observed. | +| `Resource.Spec.StoragePool` | Spec | Written by the allocator. | +| `Resource.Spec.Volumes[i].SeedFromGi` | Spec | DRBD generation ID to stamp on first activation. | +| `Resource.Status.DrbdState`, `InUse` | Status | Kernel state from `events2`. | +| `Resource.Status.Volumes[i].DiskState`, `CurrentGi` | Status | Per-volume kernel state. | +| `Node.Status.ConnectionStatus` | Status | Whether the satellite is alive. | + +A field that fits neither half, typically a transient hint, goes into an annotation. + +## Writes are patches + +Every mutating path, in the REST handlers and in the CLI, goes through the store's `Patch*` entry points. They fetch the current object, apply the caller's change to it, and retry on conflict. A caller hands over a change, never a whole object, so a key another writer added in between is kept. Checks that depend on current state, such as refusing a shrink, run inside the patch against the state the write lands on. + +Invariants that must hold for every writer live in the CRD, not in a client: size bounds are `Minimum` and `Maximum` on the field, and the controller-allocated identities (`drbdPort`, `drbdNodeID`, `skipInitialSync`) are settable-once CEL rules. A check that lives in one client is not a check the data is subject to. + +## Option inheritance + +DRBD configuration resolves from the broadest scope to the narrowest: + +``` +ControllerConfig -> ResourceGroup -> ResourceDefinition -> Resource +``` + +The typed resolver is `ResolveDRBDOptions` in `pkg/drbd/typed_resolver.go`. Pointer fields carry the nil-versus-set distinction: nil inherits, and any non-nil value, including `false` and zero, overrides. A merge written as `if *src.X { out.X = src.X }` would drop an explicit `false`; `pkg/drbd/typed_resolver_test.go` pins this. + +## Wire shape and CRD shape + +Two shapes coexist: + +1. The wire shape in `pkg/api/v1`, identical to upstream LINSTOR's REST API, with property bags as `map[string]string`. +2. The CRD shape in `api/v1alpha1`, with typed `Spec.DRBDOptions` and `Spec.ExtraProps` for keys not typed yet. + +`pkg/store/k8s/` is the boundary. `drbd_transcode.go` parses wire properties into typed fields on write and emits them back on read, so `golinstor` sees the shape it expects. Unknown `DrbdOptions/*` keys round-trip through `ExtraProps`. + +## Skipping the initial sync + +Adding a third replica to a two-replica volume would otherwise copy the whole device. The skip pipeline: + +1. The satellite's `events2` observer parses `current-uuid` and publishes it as `Resource.Status.Volumes[i].CurrentGi`. +2. When it allocates a new replica, the controller picks the lowest-named `UpToDate` peer's `CurrentGi` and stamps it on the new replica's `Spec.Volumes[i].SeedFromGi`. +3. On first activation the satellite runs `drbdmeta ... set-gi` between `drbdadm create-md` and `drbdadm adjust`, so the first handshake sees the new replica as already in sync. + +`tests/e2e/replica-add-no-resync.sh` gates it end to end. Promoting a tiebreaker to a replica with data deliberately does a full sync instead; see row 84 of the [parity deltas](https://github.com/cozystack/blockstor/blob/main/docs/cli-parity-known-deltas.md). + +## Reconcilers + +Controller side, in `internal/controller/`: + +- `ResourceReconciler` allocates the DRBD node ID, port and minor, picks `SeedFromGi`, and promotes a diskless replica that is in active use. +- `ResourceDefinitionReconciler` adds a diskless tiebreaker when a definition has an even number of replicas with data, and sets the quorum policy. +- `NodeReconciler`, `ResourceGroupReconciler`, `StoragePoolReconciler` and `SnapshotReconciler` own the invariants of their kinds. +- Cross-cutting controllers: the node heartbeat watchdog, node label sync, auto-diskful, auto-evict, auto-snapshot, group rebalancing and replica migration. + +Satellite side, in `pkg/satellite/controllers/`: each satellite runs controller-runtime reconcilers that watch only the objects whose `spec.nodeName` is its own node, drive the DRBD, LUKS and storage layers, and write observed state back through Status. `pkg/dispatcher/` translates the CRDs into the desired state they apply. + +There is no satellite to controller connection. Both sides talk only to the Kubernetes API. diff --git a/site/content/en/developer-guide/repository.md b/site/content/en/developer-guide/repository.md new file mode 100644 index 00000000..ce820156 --- /dev/null +++ b/site/content/en/developer-guide/repository.md @@ -0,0 +1,36 @@ +--- +title: "Repository layout" +linkTitle: "Repository layout" +weight: 30 +description: "Where each part of blockstor lives." +--- + +``` +api/v1alpha1/ CRD types and kubebuilder markers +cmd/ controller, apiserver, satellite, blockstor CLI, linstor-migrate +internal/controller/ controller-side reconcilers +internal/cli/ the blockstor CLI +pkg/ REST, stores, satellite, storage, DRBD, LUKS, placer, dispatcher +config/ generated CRDs, RBAC and kustomize bases +stand/ the Talos and QEMU dev stand and the reference manifests +tests/ contract, integration, e2e, cli-matrix and replay suites +docs/ design notes, parity deltas and audit history +site/ this website +``` + +| Path | What lives there | +|---|---| +| `api/v1alpha1/` | The CRD types. `make manifests` turns their markers into `config/crd/bases/`. | +| `pkg/api/v1/` | The REST wire types, shaped like upstream LINSTOR's, and the layer-stack resolver. | +| `pkg/rest/` | The LINSTOR-compatible REST handlers. | +| `pkg/store/`, `pkg/store/k8s/` | The in-memory and CRD-backed stores behind one `store.Store` interface, and the translation between wire and CRD shapes. | +| `internal/controller/` | Controller-side reconcilers: resources, definitions, groups, pools, snapshots, nodes, plus tiebreakers, auto-diskful, eviction, rebalancing and migration. | +| `pkg/satellite/controllers/` | Satellite-side reconcilers for resources, pools, snapshots and physical devices, and the `events2` observer. | +| `pkg/satellite/` | The DRBD, LUKS and storage layer reconciler and snapshot shipping. | +| `pkg/storage/` | Storage providers: LVM, LVM-thin, ZFS, ZFS-thin, loopfile and file. | +| `pkg/drbd/` | `drbdadm` and `drbdsetup` wrappers, the `.res` renderer, the `events2` parser and the options resolver. | +| `pkg/luks/` | The `cryptsetup` wrapper. | +| `pkg/placer/` | The autoplacer: capacity-weighted, anti-affinity aware. | +| `pkg/dispatcher/` | Translates CRDs into the desired state a satellite applies: layers, options, passphrases. | +| `internal/cli/` | The `blockstor` CLI. Its design is in [`docs/cli-design.md`](https://github.com/cozystack/blockstor/blob/main/docs/cli-design.md). | +| `cmd/linstor-migrate/` | The LINSTOR database converter. | diff --git a/site/content/en/developer-guide/testing.md b/site/content/en/developer-guide/testing.md new file mode 100644 index 00000000..40c02d23 --- /dev/null +++ b/site/content/en/developer-guide/testing.md @@ -0,0 +1,64 @@ +--- +title: "Testing" +linkTitle: "Testing" +weight: 20 +description: "The test tiers and how to run each of them." +--- + +Each tier answers a different question, and a change is expected to come with tests at the lowest tier that can catch the bug. + +| Tier | What it covers | Where | +|---|---|---| +| L1 unit | Logic, run on every commit | `*_test.go` next to the code | +| L2 contract, golden | Recorded `golinstor` responses replayed against the server, byte diff | `tests/contract/` | +| L3 contract, oracle | The same requests to a real LINSTOR and to blockstor, JSON diff | `tests/contract/` | +| Integration | Controllers and REST against a real Kubernetes API server (envtest), satellites faked | `tests/integration/` | +| L4 integration on DRBD | `make smoke` on the stand | `tests/smoke*.sh` | +| L5 e2e | csi-sanity and piraeus-operator e2e on the stand | `tests/e2e/` | +| L6 operator CLI | The real CLI, through REST and the satellite, down to the DRBD kernel state | `tests/e2e/cli-matrix/` | +| L7 replay | Operator workflows replayed, and the CLI parity diff against upstream | `tests/operator-harness/` | + +## Unit tests and lint + +```sh +go test ./... +make lint +``` + +Tests that need envtest skip themselves when `KUBEBUILDER_ASSETS` is not set. `make test` downloads the envtest binaries and runs the suite with them. `make lint` must be clean before a push. + +## Integration tests + +The integration suite boots an in-process Kubernetes API server and etcd with envtest, applies the CRDs, and runs every reconciler plus the REST server against it. Satellites are replaced with fakes that write Status the way a real satellite would. The tests carry the `integration` build tag: + +```sh +make setup-envtest +go test -tags integration ./tests/integration/... +``` + +## e2e on the stand + +```sh +make e2e-list +make e2e NAME=alice SCENARIO=tiebreaker +``` + +Each scenario is a shell script in `tests/e2e/`. New scenarios register their cleanup with `register_strict_cleanup` from `tests/e2e/lib.sh`: it deletes the named definitions, waits for terminating satellite pods, and checks that the cluster is back to its baseline. A scenario that leaves the cluster dirty is turned into a failure, so the next one is not blamed for it. + +## The operator CLI matrix + +`tests/e2e/cli-matrix/` holds one script per operator-reported CLI bug. A cell passes only when the replica's Status, written by the satellite's observer, and a `drbdsetup status` probe on the node both reach the expected state. A `200 OK` from the API is not enough. See the [cell README](https://github.com/cozystack/blockstor/blob/main/tests/e2e/cli-matrix/README.md). + +## Replay and parity + +```sh +# One replay workflow +BS_URL=http://127.0.0.1:3370 \ + tests/operator-harness/replay-runner.sh tests/operator-harness/replay/pvc-lifecycle.yaml + +# CLI parity against an upstream LINSTOR +BS_URL=http://127.0.0.1:3370 UP_URL=http://127.0.0.1:3371 \ + tests/operator-harness/cli-parity-refresh.sh /tmp/cli-parity +``` + +You set up the port-forwards yourself. The parity run fails on any difference from upstream that is not listed in [`docs/cli-parity-known-deltas.md`](https://github.com/cozystack/blockstor/blob/main/docs/cli-parity-known-deltas.md). Add a row there only for an intended difference; a regression gets fixed instead. diff --git a/site/content/en/docs/_index.md b/site/content/en/docs/_index.md new file mode 100644 index 00000000..332d0ef1 --- /dev/null +++ b/site/content/en/docs/_index.md @@ -0,0 +1,20 @@ +--- +title: "Documentation" +linkTitle: "Docs" +weight: 20 +menu: + main: + weight: 10 +description: "Install, operate and understand blockstor." +--- + +Blockstor is a Kubernetes control plane for LVM and ZFS storage with DRBD replication. It speaks a LINSTOR-compatible REST API, so `linstor-csi`, `piraeus-operator`, `ha-controller` and `golinstor` keep working unchanged, and it ships its own `blockstor` command-line tool. + +Where to start: + +- New to blockstor: read the [Overview](overview/), then [Architecture](architecture/). +- Setting up a cluster: [Installation](installation/), then [Provisioning volumes with CSI](csi/). +- Day-to-day work: [Using the CLI](cli/) and [Resources](resources/). +- Coming from LINSTOR: [Migrating from LINSTOR](migration/) and [Compatibility with LINSTOR](compatibility/). + +If you want to work on blockstor itself, see the [Developer guide](/developer-guide/). diff --git a/site/content/en/docs/architecture.md b/site/content/en/docs/architecture.md new file mode 100644 index 00000000..f96b36d4 --- /dev/null +++ b/site/content/en/docs/architecture.md @@ -0,0 +1,144 @@ +--- +title: "Architecture" +linkTitle: "Architecture" +weight: 30 +description: "How blockstor turns a volume request into replicas on disk." +--- + +Blockstor is built like a Kubernetes operator. The desired state of every node, pool and volume is a custom resource, and controllers keep working until the cluster matches it. Nothing important lives only in a process's memory, so any blockstor pod can restart without losing state. + +## Components + +```mermaid +flowchart TB + subgraph clients["Clients"] + csi["linstor-csi"] + other["piraeus-operator, ha-controller, golinstor"] + cli["blockstor CLI"] + end + + subgraph k8s["Kubernetes API"] + crds[("blockstor custom resources")] + end + + subgraph cp["blockstor-system"] + api["blockstor-apiserver
LINSTOR-compatible REST, mTLS"] + ctrl["blockstor-controller
placement and allocation"] + end + + subgraph nodes["Storage nodes"] + sat1["satellite on node 1"] + sat2["satellite on node 2"] + sat3["satellite on node 3"] + end + + csi -->|REST| api + other -->|REST| api + cli -->|kubeconfig| crds + api <--> crds + ctrl <--> crds + crds <--> sat1 + crds <--> sat2 + crds <--> sat3 +``` + +- **blockstor-apiserver** serves the LINSTOR REST API over mutual TLS. It does not keep state of its own: every request becomes a read or a write of custom resources. It runs as several replicas behind one Service. +- **blockstor-controller** watches the custom resources and fills in what users do not choose: which nodes get replicas, the DRBD port, minor numbers and node IDs, and tiebreakers and quorum settings. +- **blockstor-satellite** runs on every storage node. It creates the logical volumes or zvols, sets up LUKS and DRBD, and reports what it sees back into the custom resources. +- **The `blockstor` CLI** reads and writes the custom resources directly with your kubeconfig, without going through the REST API. + +There is no direct connection between the controller and the satellites. Everything goes through the Kubernetes API. + +## Custom resources are the source of truth + +Every object you know from LINSTOR is a cluster-scoped custom resource in the `blockstor.cozystack.io` group: + +```mermaid +flowchart LR + RG["ResourceGroup
policy: place count, pool, layers"] + RD["ResourceDefinition
one volume set, its size and layers"] + R1["Resource
replica on node 1"] + R2["Resource
replica on node 2"] + R3["Resource
tiebreaker on node 3"] + SP["StoragePool
per node"] + N["Node"] + S["Snapshot"] + + RG --> RD + RD --> R1 + RD --> R2 + RD --> R3 + R1 -. lives in .-> SP + SP -. on .-> N + RD --> S +``` + +Each resource has two halves: + +- **spec** is what you, the controller or the API asked for. +- **status** is what the satellites observed: disk states, connection states, free capacity. + +A satellite never changes a spec, and the API never writes a status. Because the state is ordinary Kubernetes objects, you can read it with `kubectl get resources`, back it up with your usual tools, and drive it from GitOps. + +## How a volume is created + +This is the path from a PersistentVolumeClaim to replicas on disk: + +```mermaid +sequenceDiagram + autonumber + participant PVC as PVC + participant CSI as linstor-csi + participant API as blockstor-apiserver + participant K as Kubernetes API + participant C as blockstor-controller + participant S as satellites + + PVC->>CSI: provision a volume + CSI->>API: create the resource definition and place it + API->>K: write ResourceDefinition and Resources + K-->>C: change seen + C->>K: pick nodes, allocate DRBD port, minors and node IDs, add a tiebreaker + K-->>S: each satellite sees the Resources for its node + S->>S: create LV or zvol, set up LUKS and DRBD + S->>K: report disk and connection state + API-->>CSI: volume is ready + CSI-->>PVC: bound, device /dev/drbdN +``` + +## Each satellite watches only its own node + +A satellite reacts only to the resources assigned to its node: replicas and pools whose node name is its own. Adding nodes adds satellites, and none of them has to talk to a central process. + +```mermaid +flowchart LR + subgraph K["Kubernetes API"] + r1["Resource pvc-1 on node-1"] + r2["Resource pvc-1 on node-2"] + r3["Resource pvc-2 on node-1"] + end + s1["satellite node-1"] + s2["satellite node-2"] + r1 --> s1 + r3 --> s1 + r2 --> s2 +``` + +## Layers on a node + +Every replica is a stack of layers. The satellite builds it from the bottom up when it creates a replica and takes it apart from the top down when it removes one. + +```mermaid +flowchart TB + pod["Pod sees /dev/drbdN"] + drbd["DRBD: replication to the other nodes"] + luks["LUKS: encryption at rest, optional"] + storage["STORAGE: LVM LV, LVM-thin LV, zvol or file"] + pod --> drbd --> luks --> storage +``` + +The default stack is DRBD on top of STORAGE. See [Layer stack](../layer-stack/) for the other combinations. + +## Replicas, tiebreakers and quorum + +With an even number of data replicas, the controller adds a diskless tiebreaker on another node, so DRBD always has a majority to decide which side keeps writing after a network split. A replica without a disk can also serve I/O over the network to a pod on a node that holds no copy of the data. diff --git a/site/content/en/docs/cli.md b/site/content/en/docs/cli.md new file mode 100644 index 00000000..8a35a901 --- /dev/null +++ b/site/content/en/docs/cli.md @@ -0,0 +1,154 @@ +--- +title: "Using the CLI" +linkTitle: "Using the CLI" +weight: 60 +description: "Manage nodes, pools, volumes and snapshots with the blockstor command." +--- + +`blockstor` is the command-line tool for blockstor. It reads your kubeconfig and works on the blockstor custom resources directly, so it needs no port-forward and no API certificate: if `kubectl` can reach the cluster, so can `blockstor`. + +The commands follow the `linstor` client's noun-verb grammar, with the same short aliases, so runbooks written for LINSTOR mostly carry over by changing the command name. Tools that speak the LINSTOR REST API, such as `linstor-csi`, keep working against the blockstor API server. The examples on this page use `blockstor`. + +## Install + +```sh +go install github.com/cozystack/blockstor/cmd/blockstor@v +``` + +The tool uses the same kubeconfig as `kubectl`: the `KUBECONFIG` variable, then `~/.kube/config`, or the service account when it runs inside a pod. + +## Getting help + +```sh +blockstor --help # every object and action +blockstor resource create --help # usage of one command +``` + +Every object and action has a short alias: `blockstor sp l` is `blockstor storage-pool list`, `blockstor rd c` is `blockstor resource-definition create`, `blockstor r l` is `blockstor resource list`. + +Output options that work on list commands: + +- `-m` or `--machine-readable` prints JSON. +- `--color=auto|always|never` controls colours. They are on only for a terminal, and `NO_COLOR` turns them off. + +## Nodes and storage pools + +```sh +blockstor node create worker-1 10.0.0.11 +blockstor node list + +blockstor physical-storage list +blockstor physical-storage create-device-pool zfs worker-1 /dev/sdb --pool-name data +blockstor storage-pool create lvmthin worker-2 data my_vg/my_thinpool +blockstor storage-pool list +``` + +Take a node out of service. `evacuate` moves its replicas to other nodes; `restore` brings it back: + +```sh +blockstor node evacuate worker-2 +blockstor node restore worker-2 +``` + +## Resource groups + +A resource group holds the policy, and every volume spawned from it gets the same settings: + +```sh +blockstor resource-group create replicated --place-count 2 --storage-pool data +blockstor volume-group create replicated +blockstor resource-group list +``` + +Change the policy later with `blockstor resource-group modify`, for example `--place-count 3`, then apply it to the volumes that already exist with `blockstor resource-group adjust`. + +## Volumes + +Spawn a volume from a group in one step. This creates the definition, its volume and the replicas: + +```sh +blockstor resource-group spawn replicated my-volume 10G +``` + +Or step by step, without a group policy: + +```sh +blockstor resource-definition create my-volume +blockstor volume-definition create my-volume 10G +blockstor resource create my-volume --auto-place 2 --storage-pool data +``` + +To put replicas on named nodes instead of letting blockstor choose: + +```sh +blockstor resource create worker-1 worker-2 my-volume --storage-pool data +``` + +Inspect the state: + +```sh +blockstor resource-definition list +blockstor resource list # replicas, nodes and DRBD state +blockstor resource list --faulty # only replicas that are not healthy +blockstor volume list # sizes and device paths +``` + +A healthy replicated volume shows `UpToDate` on every replica with data, and a `TieBreaker` when there is an even number of them. + +Grow a volume online. Shrinking is refused unless you add `--force`, because it cuts off whatever data lives at the end of the device: + +```sh +blockstor volume-definition set-size my-volume 0 20G +``` + +Turn a diskless replica into one with data, or the other way round: + +```sh +blockstor resource toggle-disk worker-3 my-volume --storage-pool data +blockstor resource toggle-disk worker-3 my-volume --diskless +``` + +Delete a volume with all its replicas: + +```sh +blockstor resource-definition delete my-volume +``` + +## Snapshots + +```sh +blockstor snapshot create my-volume before-upgrade +blockstor snapshot list +``` + +Restore a snapshot as a new volume. This creates the new definition, fills its volumes from the snapshot and places the replicas. The original volume is not touched: + +```sh +blockstor snapshot resource restore \ + --from-resource my-volume --from-snapshot before-upgrade --to-resource my-volume-restored +``` + +`blockstor snapshot resource-definition restore` is another spelling of the same command. To copy a live volume without taking a snapshot yourself, use `blockstor resource-definition clone my-volume my-volume-copy`. + +In-place rollback is not available on purpose: it destroys every snapshot newer than the one you roll back to. Restore into a new volume instead. + +```sh +blockstor snapshot delete my-volume before-upgrade +``` + +## Properties and DRBD options + +Every object has `set-property`, `list-properties` and `delete-property`, and groups, definitions and the controller have `drbd-options`. Settings resolve from the controller to the group to the definition to the replica, and the narrowest one wins. See [Resources](../resources/#how-settings-are-inherited). + +```sh +blockstor resource-group set-property replicated DrbdOptions/Resource/on-no-quorum suspend-io +blockstor resource-definition list-properties my-volume +``` + +## Encryption + +```sh +blockstor encryption create-passphrase '' +``` + +See [Layer stack](../layer-stack/#encryption) before you use it. diff --git a/site/content/en/docs/compatibility.md b/site/content/en/docs/compatibility.md new file mode 100644 index 00000000..c9f2de70 --- /dev/null +++ b/site/content/en/docs/compatibility.md @@ -0,0 +1,47 @@ +--- +title: "Compatibility with LINSTOR" +linkTitle: "Compatibility with LINSTOR" +weight: 80 +description: "Where blockstor behaves differently from LINSTOR, and what is not there." +--- + +Blockstor implements the LINSTOR REST API, so `linstor-csi`, `piraeus-operator`, `ha-controller`, `golinstor` and the `linstor` client work against it. This page lists the differences an operator can notice. The full list, kept for the project's parity tests, is in [`docs/cli-parity-known-deltas.md`](https://github.com/cozystack/blockstor/blob/main/docs/cli-parity-known-deltas.md). + +## Not implemented + +| Feature | What happens | +|---|---| +| Backups, backup shipping, remotes such as S3 | The API answers `501 Not Implemented`. Use snapshots and restore inside the cluster. | +| Schedules | The API answers `501 Not Implemented`. | +| `advise` | Not available. | +| In-place snapshot rollback | Refused on purpose, because it destroys every newer snapshot. Restore the snapshot into a new volume instead. | +| Per-connection DRBD peer options (`resource-connection drbd-peer-options`) | Not available. | +| External DRBD metadata (`StorPoolNameDrbdMeta`) | Not supported yet. | +| Other providers: SPDK, NVMe-oF, OpenFlex, Exos | Not supported. | + +## Behaves differently + +| Area | LINSTOR | Blockstor | +|---|---|---| +| Active network interface | `linstor node interface modify --active` switches the satellite's connection to another interface. | Has no effect. The first interface of a node is always the active one; to make another interface active, put it first. A change of address reaches DRBD after the node's satellite pod restarts. | +| Default action on lost quorum | `on-no-quorum io-error` | `on-no-quorum suspend-io`: I/O blocks until quorum returns and then resumes, so a replica that was cut off does not get stuck after the network heals. | +| Shrinking a volume | Allowed for stacks that support it. | Refused unless you pass `--force`. | +| DRBD ports and minors | Ports from 7000, minors from 1000. | Ports from 20000 to 20999, minors from 20000. Blockstor can run on the same nodes as LINSTOR without collisions. Change the port range per node with the `DrbdOptions/TcpPortRange` property. | +| Spawning more replicas than there are nodes | Fails and places nothing. | Places as many replicas as it can, reports the shortfall, and adds the rest when more nodes join. | +| Turning a tiebreaker into a replica with data | Can skip the initial sync. | Runs a full sync, to avoid a DRBD split where both sides refuse to connect. | +| Removing a replica below quorum | Removes the quorum options from the definition and prints a note. | Sets `quorum off` explicitly, with the same effect on DRBD. | +| DRBD option values | Validated against the DRBD schema. | Not validated: a wrong value is stored and fails later on the node. Check values before you set them. | +| Autoplacer weights | `MaxFreeSpace` is 1, the other weights 0. | All four weights default to 1. | +| `controller version` | Reports a git commit hash. | Reports `git=blockstor`. Do not parse a commit hash from it. | +| `node info` | Shows OS, CPU and memory. | These fields are empty. | + +## Using the `linstor` client + +The `linstor` client works against blockstor's API server. Port-forward the plain HTTP port, which is reachable only from inside the pod, and point the client at it: + +```sh +kubectl -n blockstor-system port-forward deploy/blockstor-apiserver 3370:3370 +linstor --controllers http://localhost:3370 node list +``` + +For everyday work the [`blockstor` CLI](../cli/) is simpler: it needs no port-forward and no certificates, and it accepts the same commands. diff --git a/site/content/en/docs/csi.md b/site/content/en/docs/csi.md new file mode 100644 index 00000000..393339c9 --- /dev/null +++ b/site/content/en/docs/csi.md @@ -0,0 +1,101 @@ +--- +title: "Provisioning volumes with CSI" +linkTitle: "CSI and StorageClasses" +weight: 25 +description: "Connect linstor-csi to blockstor and provision PersistentVolumes." +--- + +Kubernetes provisions blockstor volumes through the LINSTOR CSI driver, `linstor-csi`. The driver speaks the LINSTOR REST API, and blockstor serves that API, so you point the driver at the blockstor API server instead of a LINSTOR controller. + +## What to deploy + +Piraeus normally ships a whole LINSTOR distribution. With blockstor you take only the client side: + +| Piraeus component | With blockstor | +|---|---| +| `linstor-controller` | Skip. The blockstor controller and API server replace it. | +| `linstor-satellite` | Skip. Blockstor runs its own satellites. | +| `linstor-csi` (controller and node plugin) | Deploy, pointed at the blockstor API server. | +| HA controller, `drbd-reactor` | Optional. Point them at the API server the same way as CSI. | + +## Point the driver at blockstor + +The API server listens with mutual TLS on port 3371 of the `blockstor-apiserver` Service in `blockstor-system`. The installation created a client certificate for in-cluster clients: the Secret `blockstor-apiserver-client-tls` carries `tls.crt`, `tls.key` and `ca.crt`. Mount it into both the CSI controller and the CSI node plugin, and set the endpoint and certificate paths: + +```yaml +env: + - name: LS_CONTROLLERS + value: "https://blockstor-apiserver.blockstor-system.svc:3371" + - name: LS_USER_CERTIFICATE + value: /etc/linstor/client/tls.crt + - name: LS_USER_KEY + value: /etc/linstor/client/tls.key + - name: LS_ROOT_CA + value: /etc/linstor/client/ca.crt +volumeMounts: + - name: client-tls + mountPath: /etc/linstor/client + readOnly: true +volumes: + - name: client-tls + secret: + secretName: blockstor-apiserver-client-tls +``` + +The node plugin's `--node` argument must match the name of a blockstor `Node`, which is the Kubernetes node name. + +If you install `linstor-csi` with the piraeus operator, set `LinstorCluster.spec.externalController.url` to `https://blockstor-apiserver.blockstor-system.svc:3371`. That turns off the operator's own LINSTOR controller and points the CSI driver at blockstor. Point `spec.apiTLS.certManager` at the `blockstor-api-ca` Issuer, so the operator issues the client certificate for the CSI pods. + +The Secret lives in `blockstor-system`. If the CSI driver runs in another namespace, issue a second client certificate into that namespace from the same `blockstor-api-ca` Issuer. + +cert-manager renews the certificates in place, and the API server picks up a new certificate without a restart. + +## StorageClass + +Use the `linstor.csi.linbit.com` provisioner: + +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: blockstor-replicated +provisioner: linstor.csi.linbit.com +parameters: + linstor.csi.linbit.com/storagePool: "data" + linstor.csi.linbit.com/placementCount: "2" + # Optional: take the policy from a resource group you created. + # linstor.csi.linbit.com/resourceGroup: "replicated" +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +``` + +- `storagePool` is the pool that holds the replicas with data. +- `placementCount` is the number of replicas with data. Blockstor adds a tiebreaker when that number is even. +- `resourceGroup` uses a [resource group](../resources/#resourcegroup) as the policy. Without it, volumes go into the default group, `DfltRscGrp`. + +`storagePool`, `placementCount` and `resourceGroup` are covered by blockstor's own tests. Other `linstor.csi.linbit.com/*` parameters are passed through to the API unchanged; test the ones you rely on in your cluster first. + +## A first PVC + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: blockstor-test +spec: + accessModes: [ReadWriteOnce] + storageClassName: blockstor-replicated + resources: + requests: + storage: 1Gi +``` + +With `WaitForFirstConsumer`, the PVC binds once a pod that uses it is scheduled. The PersistentVolume's name is also the name of the blockstor resource definition: + +```sh +kubectl get pvc blockstor-test +kubectl get pv +blockstor resource list +``` + +Delete volumes through Kubernetes: deleting the PVC makes the CSI driver delete the definition and its replicas. diff --git a/site/content/en/docs/installation.md b/site/content/en/docs/installation.md new file mode 100644 index 00000000..8b4fafad --- /dev/null +++ b/site/content/en/docs/installation.md @@ -0,0 +1,141 @@ +--- +title: "Installation" +linkTitle: "Installation" +weight: 20 +description: "Prepare the nodes, deploy blockstor, and check that it is up." +--- + +Blockstor installs onto an existing Kubernetes cluster. You need the storage nodes prepared, cert-manager in the cluster, and the blockstor manifests from the release you want to run. + +## Prerequisites + +On every storage node that will run the satellite: + +- The DRBD 9 kernel module, loaded (`modprobe drbd`). +- For ZFS pools, the ZFS kernel module. On Talos Linux both modules come from the `siderolabs/drbd` and `siderolabs/zfs` system extensions. +- At least one spare block device, or an existing LVM volume group or ZFS pool to back the storage pools. + +The satellite image already carries the userspace tools (`drbd-utils`, `lvm2`, `cryptsetup`, `zfsutils`); only the kernel modules come from the host. + +In the cluster: + +- [cert-manager](https://cert-manager.io/), which issues the certificates for the API's mutual TLS. +- Permission to run privileged pods in the `blockstor-system` namespace. The satellite needs host networking and access to the host's block devices. + +## Get the manifests + +The CRDs and workload manifests are in the blockstor repository. Check out the tag of the release you are installing: + +```sh +git clone https://github.com/cozystack/blockstor.git +cd blockstor +git checkout v +``` + +The workload manifests carry an `__REGISTRY__/:dev` placeholder used by the development stand. Point them at the published images for your version: + +```sh +TAG= +mkdir -p rendered +for f in blockstor-deploy.yaml blockstor-apiserver-tls.yaml blockstor-apiserver-deploy.yaml blockstor-satellite-daemonset.yaml; do + sed -e "s#__REGISTRY__/\(blockstor-[a-z]*\):dev#ghcr.io/cozystack/\1:${TAG}#" "stand/$f" > "rendered/$f" +done +``` + +Each release publishes the images as `MAJOR.MINOR.PATCH` and `MAJOR.MINOR`, and stable releases also as `latest`. + +## Deploy + +Apply the pieces in this order. + +1. The custom resource definitions: + + ```sh + kubectl apply -f config/crd/bases/ + ``` + +2. The namespace, the controller and its RBAC: + + ```sh + kubectl apply -f rendered/blockstor-deploy.yaml + ``` + +3. The certificate chain for the API. It creates a CA, the API server's serving certificate, and a client certificate for in-cluster clients such as CSI: + + ```sh + kubectl apply -f rendered/blockstor-apiserver-tls.yaml + ``` + +4. The API server and the satellites: + + ```sh + kubectl apply -f rendered/blockstor-apiserver-deploy.yaml + kubectl apply -f rendered/blockstor-satellite-daemonset.yaml + ``` + +The satellite DaemonSet runs on every node that is not a control-plane node. Wait for everything to come up: + +```sh +kubectl -n blockstor-system rollout status deploy/blockstor-controller +kubectl -n blockstor-system rollout status deploy/blockstor-apiserver +kubectl -n blockstor-system rollout status daemonset/blockstor-satellite +``` + +## Register nodes and storage pools + +Each storage node is a cluster-scoped `Node` resource named after the Kubernetes node, and each pool on a node is a `StoragePool`. You can create them as YAML (see [Resources](../resources/)) or with the CLI. + +Install the CLI from the release tag. It needs Go and reads your kubeconfig, so it talks to the cluster the same way `kubectl` does: + +```sh +go install github.com/cozystack/blockstor/cmd/blockstor@v +``` + +Register the nodes. The address is the one DRBD uses for replication traffic between nodes: + +```sh +blockstor node create worker-1 10.0.0.11 +blockstor node create worker-2 10.0.0.12 +blockstor node create worker-3 10.0.0.13 +``` + +Create a pool from a raw disk in one step. This prepares the device and registers the pool: + +```sh +blockstor physical-storage create-device-pool zfs worker-1 /dev/sdb --pool-name data +``` + +Or register a volume group or ZFS pool that already exists: + +```sh +blockstor storage-pool create lvmthin worker-1 data my_vg/my_thinpool +``` + +The providers are `lvm`, `lvmthin`, `zfs`, `zfsthin`, `file`, `filethin` and `diskless`. Every node also gets a diskless pool, `DfltDisklessStorPool`, so it can host diskless replicas and tiebreakers. + +## Check the installation + +```sh +blockstor controller version +blockstor node list +blockstor storage-pool list +``` + +Every node should show as online, and every pool should report its free and total capacity once its satellite has seen it. + +Then create a first replicated volume to see the whole path work: + +```sh +blockstor resource-group create mygroup --place-count 2 --storage-pool data +blockstor volume-group create mygroup +blockstor resource-group spawn mygroup test-volume 1G +blockstor resource list +``` + +A healthy result is two `UpToDate` replicas and a `TieBreaker` on a third node. Remove it when you are done: + +```sh +blockstor resource-definition delete test-volume +``` + +Next, connect Kubernetes to it: [Provisioning volumes with CSI](../csi/). diff --git a/site/content/en/docs/layer-stack.md b/site/content/en/docs/layer-stack.md new file mode 100644 index 00000000..5d0658d9 --- /dev/null +++ b/site/content/en/docs/layer-stack.md @@ -0,0 +1,78 @@ +--- +title: "Layer stack" +linkTitle: "Layer stack" +weight: 50 +description: "Combine replication, encryption and plain local storage." +--- + +Each volume is built from layers. The bottom layer allocates space on the node, and each layer above adds something on top of the one below. The topmost layer is the device the pod uses. + +| Layer | What it does | +|---|---| +| `STORAGE` | Allocates the block device: an LVM or LVM-thin logical volume, a ZFS zvol, or a file. Every stack ends with it. | +| `LUKS` | Encrypts the device below it with `cryptsetup`. | +| `DRBD` | Replicates the device to the other nodes over the network. | + +## Supported stacks + +| Stack | Use it for | +|---|---| +| `DRBD`, `STORAGE` | Replicated volumes. This is the default. | +| `DRBD`, `LUKS`, `STORAGE` | Replicated and encrypted at rest. Each replica encrypts its own copy. | +| `LUKS`, `STORAGE` | Encrypted volume with a single replica and no DRBD. | +| `STORAGE` | Plain local volume with a single replica, for caches and scratch space. | + +These four are the only orders blockstor accepts. Other orders, such as LUKS on top of DRBD, are refused when the definition is created. + +## Choosing the stack + +A definition takes its stack from its own `layerStack`, then from its resource group's `selectFilter.layerStack`. If neither is set, it gets `DRBD`, `STORAGE`. + +Set it on a group, so every volume spawned from it gets the same stack: + +```sh +blockstor resource-group create local --place-count 1 --storage-pool data --layer-list storage +blockstor volume-group create local +blockstor resource-group spawn local scratch-1 20G +``` + +Or on one definition, in YAML: + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: ResourceDefinition +metadata: + name: scratch-1 +spec: + layerStack: ["STORAGE"] + volumeDefinitions: + - volumeNumber: 0 + sizeKib: 20971520 +``` + +Treat the stack as fixed once a volume has replicas. Changing `layerStack` later does not encrypt or decrypt data that is already there. + +## Encryption + +LUKS volumes are opened with the cluster passphrase, which lives in a Secret in `blockstor-system`. Create it once, before the first encrypted volume: + +```sh +blockstor encryption create-passphrase '' +``` + +Keep a copy of the passphrase somewhere safe. Without it, encrypted volumes cannot be opened. Running the command again with the same passphrase is a no-op, and blockstor refuses to replace it with a different one, because that would lock every existing encrypted volume. A definition with `LUKS` in its stack and no passphrase available is refused rather than created unencrypted. + +Then create volumes with LUKS in the stack: + +```sh +blockstor resource-group create secure --place-count 2 --storage-pool data --layer-list drbd,luks,storage +blockstor volume-group create secure +blockstor resource-group spawn secure secret-volume 10G +``` + +Growing an encrypted volume resizes every layer in order: the storage below, then LUKS, then DRBD. + +Limits today: + +- All volumes of a definition share one key. +- Changing the passphrase of existing volumes is not supported. To re-key a volume, create a new one and move the data. diff --git a/site/content/en/docs/migration.md b/site/content/en/docs/migration.md new file mode 100644 index 00000000..1d74a08c --- /dev/null +++ b/site/content/en/docs/migration.md @@ -0,0 +1,103 @@ +--- +title: "Migrating from LINSTOR" +linkTitle: "Migrating from LINSTOR" +weight: 70 +description: "Take over an existing LINSTOR cluster in place, without copying data." +--- + +Blockstor can take over a LINSTOR cluster that runs on Kubernetes. The `linstor-migrate` tool converts LINSTOR's database into blockstor resources, and blockstor then adopts the volumes and DRBD metadata that are already on disk. No data is copied or resynced, and replica placement does not change. + +Do this in a maintenance window, after a full rehearsal on a staging cluster. + +## What moves and what does not + +Converted: + +- nodes, storage pools, resource groups, resource definitions with their volumes, replicas, and successful snapshots; +- controller-wide DRBD options; +- DRBD minor numbers, node IDs and the shared secret, unchanged; +- DRBD ports, if you capture them from the running nodes (step 3 below). + +Not converted, and listed in the tool's report: + +- **LUKS passphrases.** Encrypted volumes convert with their layers intact, but blockstor cannot open them until you set up the passphrase by hand. +- Backup shipping remotes, and replicas or snapshots that LINSTOR marked as deleted or failed. +- Snapshots of definitions with more than one volume are adopted with their first volume only. + +## Before you start + +1. **Check that the cluster is healthy.** `linstor resource list --faulty` is empty, every replica with data is `UpToDate`, and no node is offline. A degraded replica is adopted as it is and stays degraded. + +2. **Dump the LINSTOR database.** LINSTOR on Kubernetes keeps it in custom resources: + + ```sh + mkdir dump && cd dump + kubectl get crds | grep -o ".*.internal.linstor.linbit.com" \ + | xargs -I{} sh -c 'kubectl get {} -ojson > {}.json' + cd .. + ``` + +3. **Capture the DRBD ports.** LINSTOR 1.33 does not store them in its database. Without them, blockstor would assign new ports, and every replica would reconnect under load. Collect them from the satellites: + + ```sh + for pod in $(kubectl -n cozy-linstor get pods -l app.kubernetes.io/component=linstor-satellite \ + -o jsonpath='{.items[*].metadata.name}'); do + kubectl -n cozy-linstor exec "$pod" -- sh -c ' + for f in /var/lib/linstor.d/*.res; do + rd=$(basename "$f" .res) + port=$(grep -oE "address[^;]*:[0-9]+" "$f" | grep -oE "[0-9]+$" | head -1) + [ -n "$port" ] && echo "$rd $port" + done' + done | sort -u > drbd-ports.txt + ``` + + Change the namespace and label to match your installation. Every replica of a volume shares one port. If one volume shows two different ports, stop and find out why. + +4. **Take pool snapshots.** A recursive `zfs snapshot` of every pool gives you a rollback point that does not depend on either control plane. + +5. **Check the volume names on disk.** Blockstor finds existing volumes by name: `/_`, for example `data/pvc-1234_00000`. If the name does not match exactly, blockstor creates a new empty volume next to the real one. Replicated volumes would heal from their peers, but a volume with a single replica and no DRBD would come up empty. On every node, compare the list of volumes with the names the converted manifests expect: + + ```sh + zfs list -H -o name -t volume | sort + zfs list -H -o name -t snapshot | sort + ``` + + Snapshots are found as `/_00000@`. Do not continue while any single-replica volume is missing under its expected name. + +6. **Rehearse on staging.** Run the whole procedure on a copy of the pools. Check that every pool registers, every replica reaches `UpToDate` without a resync, and no new empty volumes appear. + +## Convert + +Build the tool from the blockstor release you are installing and run it on the dump: + +```sh +go install github.com/cozystack/blockstor/cmd/linstor-migrate@v +linstor-migrate -in ./dump -drbd-ports ./drbd-ports.txt -out ./blockstor-manifests.yaml +``` + +Read the whole report it prints. Each `warning:` line is something the tool refused to guess. Resolve every `LUKS passphrase NOT migrated` and every `no DRBD port` line before you go on. + +## Cut over + +1. **Stop new volumes.** Scale the CSI provisioner and attacher to zero. +2. **Stop the LINSTOR controller.** Scale `deploy/linstor-controller` to zero. Leave the LINSTOR satellites and the DRBD devices running: they keep serving I/O. +3. **Install blockstor** as described in [Installation](../installation/), with satellites on the same nodes and pools. Check the image versions before you start them. +4. **Apply the converted manifests.** They are already in dependency order: + + ```sh + kubectl apply -f ./blockstor-manifests.yaml + ``` + +5. **Wait for adoption.** Every replica should report `UpToDate` without starting a resync, `blockstor resource list --faulty` should be empty, and every pool should report its real free space. +6. **Point CSI at blockstor**, as in [Provisioning volumes with CSI](../csi/), and scale the provisioner and attacher back up. Create a test PVC, and check that an existing PVC still mounts read-write. + +## Check before you call it done + +- The number of nodes, pools, groups, definitions and replicas matches the counts in the converter's report. +- For at least five volumes, the size, `/dev/drbd`, node IDs and port match the values from before the migration. A changed minor or node ID is a reason to stop and roll back. +- No replica is unexpectedly syncing. +- Encrypted volumes stay closed until you set up their passphrase. Keep their consumers stopped until then. + +## Roll back + +Until CSI has been pointed at blockstor and checked, rolling back means: scale blockstor to zero, scale `linstor-controller` back up, and point CSI at LINSTOR again. Blockstor did not touch the data on disk or the running DRBD devices, so this undoes only the control plane. The pool snapshots from step 4 are the last-resort rollback for the data itself. diff --git a/site/content/en/docs/overview.md b/site/content/en/docs/overview.md new file mode 100644 index 00000000..d8c9b43a --- /dev/null +++ b/site/content/en/docs/overview.md @@ -0,0 +1,57 @@ +--- +title: "Overview" +linkTitle: "Overview" +weight: 10 +description: "What blockstor is, what it does today, and when to use it." +--- + +Blockstor turns the local disks of your Kubernetes nodes into replicated block storage. Each volume is a logical volume (LVM) or a zvol (ZFS) on a node, and DRBD keeps copies of it on other nodes in sync, so a pod can lose its node and come back elsewhere with its data. + +Blockstor speaks the LINSTOR REST API. The clients built for LINSTOR, `linstor-csi`, `piraeus-operator`, `ha-controller` and `golinstor`, work against it unchanged. What is different is how the cluster keeps its state: everything lives in Kubernetes custom resources, and controllers converge the cluster toward it. There is no separate database to back up and no in-memory state to lose when a pod restarts. + +## When to use it + +Blockstor fits when you want replicated block volumes for Kubernetes on your own hardware, with LVM or ZFS underneath and DRBD for replication, and you want that storage to be managed the same way as the rest of the cluster: with custom resources, `kubectl` and GitOps. + +It is also a fit if you already run LINSTOR on Kubernetes. Blockstor can take over an existing LINSTOR cluster in place, without copying data. See [Migrating from LINSTOR](../migration/). + +## What works today + +- Replicated volumes over DRBD on LVM, LVM-thin, ZFS, ZFS-thin or file backends. +- Volumes without DRBD: plain local storage, single replica. +- LUKS encryption at rest. +- Automatic placement with constraints: zones, node properties, replicas on different nodes. +- Tiebreakers and quorum policies. +- Snapshots: create, restore as a new volume, and clone. +- Snapshot shipping between nodes of the same cluster, for clones and new replicas. +- Online volume resize. +- Creating storage pools from raw disks. +- The LINSTOR-compatible REST API over mutual TLS. + +## What is not implemented + +The API answers these with `501 Not Implemented`: + +- Snapshot shipping to another cluster (disaster recovery). +- Backups: create, restore, ship, abort, and the backup queue. +- Schedules (cron-driven backups). +- Remote backends such as S3 and LINSTOR remotes. +- Other storage providers: SPDK, NVMe-oF, OpenFlex, Exos. + +## On the roadmap + +- Bring-your-own-key encryption, with Secret references in the spec. +- Shared-LUN provisioning: thick LVM plus thin qcow2 on LVM. +- A VDUSE backend through `qemu-storage-daemon`, for shared-SAN Kubernetes. + +## Components + +Blockstor runs as three images, published to GHCR on every release: + +| Image | What it runs | +|---|---| +| `ghcr.io/cozystack/blockstor-controller` | The controllers that place replicas and allocate DRBD ports, minors and node IDs. | +| `ghcr.io/cozystack/blockstor-apiserver` | The LINSTOR-compatible REST API that CSI and other clients talk to. | +| `ghcr.io/cozystack/blockstor-satellite` | The per-node agent that creates volumes and runs DRBD. | + +[Architecture](../architecture/) shows how they fit together. diff --git a/site/content/en/docs/resources.md b/site/content/en/docs/resources.md new file mode 100644 index 00000000..60eb53ad --- /dev/null +++ b/site/content/en/docs/resources.md @@ -0,0 +1,201 @@ +--- +title: "Resources" +linkTitle: "Resources" +weight: 40 +description: "The blockstor custom resources, with examples." +--- + +Everything blockstor manages is a cluster-scoped custom resource in the `blockstor.cozystack.io/v1alpha1` API group. You can create and change them with `kubectl` or GitOps, with the [`blockstor` CLI](../cli/), or through the REST API. All three edit the same objects. + +| Kind | What it describes | Name | +|---|---|---| +| `Node` | A storage node and its replication address | the Kubernetes node name | +| `StoragePool` | A pool of space on one node | `.` | +| `PhysicalDevice` | A raw disk the satellite found on a node | set by the satellite | +| `ResourceGroup` | A placement and replication policy | any | +| `ResourceDefinition` | One volume set: sizes, layers, policy | any | +| `Resource` | One replica of a definition on one node | `.` | +| `Snapshot` | A snapshot of a definition | `.` | +| `ControllerConfig` | Cluster-wide settings | always `default` | + +The naming rules are enforced by the API server, so a `Resource` or `StoragePool` with a name that does not match its spec is refused. Each kind has printer columns, so `kubectl get resources` or `kubectl get storagepools` gives a useful table. + +Most objects have a `spec` you write and a `status` that blockstor writes. The examples below show only `spec`. + +## Node + +A node that runs the satellite. The address of the first network interface is the one DRBD uses for replication. + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: Node +metadata: + name: worker-1 +spec: + type: SATELLITE + netInterfaces: + - name: default + address: 10.0.0.11 +``` + +## StoragePool + +Space for replicas on one node. The provider decides how volumes are allocated, and the properties name the backing volume group, thin pool, ZFS pool or directory. + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: StoragePool +metadata: + name: data.worker-1 +spec: + nodeName: worker-1 + poolName: data + providerKind: LVM_THIN + props: + StorDriver/LvmVg: my_vg + StorDriver/ThinPool: my_thinpool +``` + +For a ZFS thin pool: + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: StoragePool +metadata: + name: zfs-data.worker-1 +spec: + nodeName: worker-1 + poolName: zfs-data + providerKind: ZFS_THIN + props: + StorDriver/ZPoolThin: tank +``` + +The providers are `LVM`, `LVM_THIN`, `ZFS`, `ZFS_THIN`, `FILE`, `FILE_THIN` and `DISKLESS`. Pools with the same name on different nodes form one pool from the placement point of view: a policy that says "pool `data`" can place replicas on every node that has a `data` pool. + +## PhysicalDevice + +The satellites report the raw disks they find as `PhysicalDevice` objects, with their size, model and path in `status`. To build a pool from one, set `attachTo`: + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: PhysicalDevice +metadata: + name: +spec: + attachTo: + storagePoolName: data + providerKind: ZFS_THIN + zPoolName: data +``` + +The CLI does the same thing with `blockstor physical-storage create-device-pool`. + +## ResourceGroup + +A policy that volumes inherit: how many replicas, in which pool, with which layers, and with which DRBD options. + +```yaml +apiVersion: blockstor.cozystack.io/v1alpha1 +kind: ResourceGroup +metadata: + name: replicated +spec: + description: Two data replicas plus a tiebreaker + selectFilter: + placeCount: 2 + storagePool: data + layerStack: ["DRBD", "STORAGE"] + replicasOnDifferent: ["Aux/topology.kubernetes.io/zone"] + drbdOptions: + resource: + quorum: majority + onNoQuorum: suspend-io + volumeGroups: + - volumeNumber: 0 +``` + +Placement constraints match node properties. Blockstor copies the Kubernetes node labels under `topology.kubernetes.io/` and `node-role.kubernetes.io/`, and `kubernetes.io/hostname`, onto each `Node` as `Aux/