diff --git a/.agents/skills/console-ui-e2e/SKILL.md b/.agents/skills/console-ui-e2e/SKILL.md index bbf30137c..c04404cab 100644 --- a/.agents/skills/console-ui-e2e/SKILL.md +++ b/.agents/skills/console-ui-e2e/SKILL.md @@ -54,6 +54,10 @@ does not need that reference. ## Run +Follow [test and CI progression](../debug-test/SKILL.md#test-and-ci-progression) +when sequencing suites or CI jobs. Fix a failure with its affected spec/group, +then resume the remaining selection; use this skill for browser diagnosis. + ```sh pixi run clean-env pixi run test-container-e2e --scope console.operations --case --concurrency 1 --node-cpus 0.25 --cpus 8 --memory-mib 16384 diff --git a/.agents/skills/debug-test/SKILL.md b/.agents/skills/debug-test/SKILL.md index c171fe9e9..31df09af7 100644 --- a/.agents/skills/debug-test/SKILL.md +++ b/.agents/skills/debug-test/SKILL.md @@ -1,6 +1,6 @@ --- name: debug-test -description: Diagnose a non-browser failing test from first divergence to root cause. +description: Run CROWDB test and CI jobs incrementally, and diagnose non-browser failures from first divergence to root cause. --- @@ -11,6 +11,47 @@ description: Diagnose a non-browser failing test from first divergence to root c Do not increase timeouts, suppress errors, weaken assertions, add caller-side retries, or otherwise make only the symptom pass. +## Test and CI progression + +For expensive suites or multiple jobs, preserve an ordered list of commands, +completed results, logs, relevant source/image inputs, and the next unrun item. +Use existing runner records where available. + +- Run until an error, then fix it before advancing the affected sequence. Do + not first run every job against a known defect. Prioritize bugs that make + subsequent tests slow, hang, or fail through the same dependency. +- For container E2E, use `--fail-fast` to stop queued cases while letting + already owned fixtures finish and clean up. Read failure records throughout + the log, not only its tail or completed-case count. Do not interrupt both a + launcher and its child; forwarded duplicate signals can break cleanup. +- Reproduce and repair the failure with its test or smallest meaningful test + group. Run that group's regression and relevant quality gates after the fix. +- Resume the remaining tests/jobs from the saved position. Retain earlier + passing results; repeat them only when the fix changes their behavior or + inputs. Preserve original ordering and coverage when selecting a remainder. +- Independent work already in flight may finish when it is unaffected. Stop + dispatching work that depends on the defect or would repeat its slowdown. +- Complete shared build prerequisites before parallel test jobs. Never run + concurrent builders against the same output directory; use independent + output directories when parallel builds are necessary. Perform shared + environment cleanup before dispatch; do not repeat `clean-env` while other + local jobs still own running fixtures. Host browser fixtures that require + stable Docker networks also share that constraint across local CI jobs; + keep their browser phases separate from other jobs' bridge creation/removal. +- Before long suites, inspect actual fixture slots and declared CPU/memory + costs. An outer parallel job count does not prove cases run concurrently. + Distinguish build, fixture startup, client execution, and cleanup time. +- Reuse valid builds and images. Rebuild when their relevant inputs change; + avoid unrelated feature/profile changes and repeated rebuilds while debugging. +- After the requested jobs have each been repaired and locally verified, run + the requested or required overall verification once as a closing gate. If + that gate finds another defect, repair its group and resume the affected + remainder rather than restarting valid completed groups from the beginning. +- Report actual coverage, failures, and unrun/blocked work. A focused pass does + not by itself establish that a whole suite or CI job passed. + +## Diagnose a failure + 1. Check relevant environment variables, stale processes, logs, persisted state, and build artifacts. 2. Reproduce the exact test. If it passes alone, compare the affected suite diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8e885bb6..590b3bd44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -305,7 +305,7 @@ jobs: name: verified-oci-image path: target - name: Import shared release image - run: pixi run image-import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev + run: pixi run --manifest-path container/oci-image/pixi.toml import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev - name: Run complete S3 acceptance run: pixi run test-e2e-s3 - name: Capture runner state on failure @@ -374,7 +374,7 @@ jobs: name: verified-oci-image path: target - name: Import shared release image - run: pixi run image-import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev + run: pixi run --manifest-path container/oci-image/pixi.toml import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev - name: Run Iceberg acceptance env: E2E_PARTITION: ${{ matrix.partition }} @@ -597,7 +597,7 @@ jobs: pixi run sudo apt-get update pixi run sudo apt-get install -y protobuf-compiler - name: Import the shared verified image - run: pixi run image-import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev + run: pixi run --manifest-path container/oci-image/pixi.toml import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev - name: Verify centralized KV fixtures and protocol layer run: | pixi run test-container-e2e-fixture @@ -650,7 +650,7 @@ jobs: name: verified-oci-image path: target - name: Import formal release image - run: pixi run image-import target/crowdb.oci.tar --image crowdb-node:dev + run: pixi run --manifest-path container/oci-image/pixi.toml import target/crowdb.oci.tar --image crowdb-node:dev - name: Verify installed system browser run: | for b in /snap/bin/chromium /usr/bin/chromium /usr/bin/chromium-browser /usr/bin/google-chrome /usr/bin/google-chrome-stable /usr/bin/microsoft-edge; do @@ -681,15 +681,17 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - max-parallel: 3 + max-parallel: 4 matrix: include: - group: disk domains: diskdb diskio - group: chunk domains: chunkdb chunk-kv chunk-stream chunk-client - - group: platform - domains: dataset deployment + - group: dataset + domains: dataset + - group: deployment + domains: deployment permissions: contents: read steps: @@ -709,7 +711,17 @@ jobs: pixi run sudo apt-get update pixi run sudo apt-get install -y protobuf-compiler - name: Import shared release image - run: pixi run image-import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev + run: pixi run --manifest-path container/oci-image/pixi.toml import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev + - uses: Swatinem/rust-cache@v2 + with: + shared-key: container-storage-${{ matrix.group }}-${{ hashFiles('pixi.lock') }} + cache-targets: false + cache-directories: | + target/debug + sdk/target + cmd-format: "pixi run {0}" + cache-workspace-crates: true + cache-on-failure: true - name: Verify container domain env: E2E_DOMAINS: ${{ matrix.domains }} diff --git a/.github/workflows/docker-preview.yml b/.github/workflows/docker-preview.yml index ff9d87dbc..7eba2cab7 100644 --- a/.github/workflows/docker-preview.yml +++ b/.github/workflows/docker-preview.yml @@ -7,6 +7,9 @@ concurrency: group: crowdb-container-preview-${{ github.ref_name }} cancel-in-progress: false +env: + CROWDB_E2E_PRINT_SERVICE_ERRORS: "1" + jobs: construct: runs-on: ubuntu-24.04 @@ -48,7 +51,7 @@ jobs: pixi run sudo apt-get clean pixi run df -h / - name: Stage runtime without a container daemon - run: pixi run stage-single-node-container + run: pixi run bash container/oci-image/stage.sh - name: Install locked standalone builder run: pixi install --locked --manifest-path container/oci-image/pixi.toml - name: Build OCI without Docker @@ -87,7 +90,7 @@ jobs: name: verified-oci-image path: target - name: Import exact OCI artifact into Docker - run: pixi run image-import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev + run: pixi run --manifest-path container/oci-image/pixi.toml import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev - name: Verify Docker startup and recovery env: CROWDB_PREVIEW_TEST_ARTIFACTS: ${{ runner.temp }}/crowdb-preview-artifacts @@ -99,19 +102,24 @@ jobs: pixi run clean-env pixi run --skip-deps test-single-node-container pixi run test-container-e2e --profile bridge-admission --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 - pixi run test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 + pixi run sudo --preserve-env=CROWDB_E2E_PRINT_SERVICE_ERRORS "$(command -v pixi)" run --as-is test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 pixi install --locked --manifest-path container/oci-image/pixi.toml -e runtime - pixi run test-container-e2e --profile containerd-cluster --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 + pixi run test-container-e2e --profile containerd-cluster --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 8192 - name: Record verified digest id: artifact run: pixi run python container/oci-image/ci-receipt.py target/crowdb.oci.tar + - name: Export bounded host and container diagnostics + if: always() + run: | + pixi run sudo "$(command -v pixi)" run --as-is python e2e-test/publish.py --source .crowdb-runtime/artifacts --output target/oci-acceptance-public + pixi run sudo "$(command -v pixi)" run --as-is python e2e-test/publish.py --source "${{ runner.temp }}/crowdb-preview-artifacts" --output target/oci-preview-public - uses: actions/upload-artifact@v4 if: failure() with: name: docker-acceptance-diagnostics path: | - ${{ runner.temp }}/crowdb-preview-artifacts - .crowdb-runtime/artifacts + target/oci-acceptance-public + target/oci-preview-public if-no-files-found: ignore containerd: @@ -142,7 +150,7 @@ jobs: - name: Verify containerd host-network startup and recovery run: | set -o pipefail - pixi run test-container-e2e --profile containerd-host --concurrency 1 --cpus 3 --memory-mib 2048 --archive "$PWD/target/crowdb.oci.tar" 2>&1 | pixi run tee target/containerd-acceptance.log + pixi run sudo --preserve-env=CROWDB_E2E_PRINT_SERVICE_ERRORS "$(command -v pixi)" run --as-is test-container-e2e --profile containerd-host --concurrency 1 --cpus 3 --memory-mib 3072 --archive "$PWD/target/crowdb.oci.tar" 2>&1 | pixi run tee target/containerd-acceptance.log - uses: actions/upload-artifact@v4 if: failure() with: @@ -178,7 +186,7 @@ jobs: pixi run sudo apt-get update pixi run sudo apt-get install -y protobuf-compiler - name: Import the shared verified image - run: pixi run image-import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev + run: pixi run --manifest-path container/oci-image/pixi.toml import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev - name: Verify centralized KV fixtures and protocol layer run: | pixi run test-container-e2e-fixture diff --git a/.github/workflows/iceberg-container-ecosystem.yml b/.github/workflows/iceberg-container-ecosystem.yml index e4f3c1de0..e696d7098 100644 --- a/.github/workflows/iceberg-container-ecosystem.yml +++ b/.github/workflows/iceberg-container-ecosystem.yml @@ -38,7 +38,7 @@ jobs: pixi-version: latest - uses: Swatinem/rust-cache@v2 - name: Build formal release image - run: pixi run build-single-node-container + run: pixi run oci-build --load - name: Prepare locked clients run: | pixi install --locked -e iceberg-ecosystem diff --git a/.github/workflows/release-container.yml b/.github/workflows/release-container.yml index 05adef4e4..b2614697d 100644 --- a/.github/workflows/release-container.yml +++ b/.github/workflows/release-container.yml @@ -7,6 +7,9 @@ concurrency: group: crowdb-container-${{ github.ref_name }} cancel-in-progress: false +env: + CROWDB_E2E_PRINT_SERVICE_ERRORS: "1" + jobs: construct: runs-on: ubuntu-24.04 @@ -50,7 +53,7 @@ jobs: pixi run sudo apt-get clean pixi run df -h / - name: Stage runtime without a container daemon - run: pixi run stage-single-node-container + run: pixi run bash container/oci-image/stage.sh - name: Install locked standalone builder run: pixi install --locked --manifest-path container/oci-image/pixi.toml - name: Build OCI without Docker @@ -89,7 +92,7 @@ jobs: name: verified-oci-image path: target - name: Import exact OCI artifact into Docker - run: pixi run image-import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev + run: pixi run --manifest-path container/oci-image/pixi.toml import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev - name: Verify Docker startup and recovery env: CROWDB_PREVIEW_TEST_ARTIFACTS: ${{ runner.temp }}/crowdb-preview-artifacts @@ -101,19 +104,24 @@ jobs: pixi run clean-env pixi run --skip-deps test-single-node-container pixi run test-container-e2e --profile bridge-admission --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 - pixi run test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 + pixi run sudo --preserve-env=CROWDB_E2E_PRINT_SERVICE_ERRORS "$(command -v pixi)" run --as-is test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 pixi install --locked --manifest-path container/oci-image/pixi.toml -e runtime - pixi run test-container-e2e --profile containerd-cluster --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 + pixi run test-container-e2e --profile containerd-cluster --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 8192 - name: Record verified digest id: artifact run: pixi run python container/oci-image/ci-receipt.py target/crowdb.oci.tar + - name: Export bounded host and container diagnostics + if: always() + run: | + pixi run sudo "$(command -v pixi)" run --as-is python e2e-test/publish.py --source .crowdb-runtime/artifacts --output target/oci-acceptance-public + pixi run sudo "$(command -v pixi)" run --as-is python e2e-test/publish.py --source "${{ runner.temp }}/crowdb-preview-artifacts" --output target/oci-preview-public - uses: actions/upload-artifact@v4 if: failure() with: name: docker-acceptance-diagnostics path: | - ${{ runner.temp }}/crowdb-preview-artifacts - .crowdb-runtime/artifacts + target/oci-acceptance-public + target/oci-preview-public if-no-files-found: ignore containerd: @@ -144,7 +152,7 @@ jobs: - name: Verify containerd host-network startup and recovery run: | set -o pipefail - pixi run test-container-e2e --profile containerd-host --concurrency 1 --cpus 3 --memory-mib 2048 --archive "$PWD/target/crowdb.oci.tar" 2>&1 | pixi run tee target/containerd-acceptance.log + pixi run sudo --preserve-env=CROWDB_E2E_PRINT_SERVICE_ERRORS "$(command -v pixi)" run --as-is test-container-e2e --profile containerd-host --concurrency 1 --cpus 3 --memory-mib 3072 --archive "$PWD/target/crowdb.oci.tar" 2>&1 | pixi run tee target/containerd-acceptance.log - uses: actions/upload-artifact@v4 if: failure() with: @@ -181,7 +189,7 @@ jobs: pixi run sudo apt-get update pixi run sudo apt-get install -y protobuf-compiler - name: Import the shared verified image - run: pixi run image-import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev + run: pixi run --manifest-path container/oci-image/pixi.toml import "$PWD/target/crowdb.oci.tar" --image crowdb-node:dev - name: Verify centralized KV fixtures and protocol layer run: | pixi run test-container-e2e-fixture diff --git a/AGENTS.md b/AGENTS.md index b37a01b21..c606ee69b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,7 +52,8 @@ and transport exposed through FFI. aliases or compatibility symlinks such as `.devin`. - Code changes: `/coding`; add `/console-ui-e2e` only for visible UI or Playwright work. -- Test diagnosis: `/debug-test`, or `/console-ui-e2e` for browser/UI failures. +- Test execution and CI progression: `/debug-test`; use `/console-ui-e2e` for + browser/UI diagnosis. - Docs: one matched `/doc-*` guide; `/doc` only for general or unclear targets. - Requirements: `/implement-requirement`; open the detail directly and consult the backlog index only for selection, ordering, or status. diff --git a/Cargo.lock b/Cargo.lock index 2e54151f6..eb2b412a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,6 +782,7 @@ dependencies = [ "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", + "flatbuffers", "serde", "serde_json", "thiserror 2.0.18", @@ -845,6 +846,7 @@ dependencies = [ "crowdb-kv-client", "crowdb-protocol", "crowdb-rpc-ffi", + "crowdb-test-harness", "crowdb-tree-ffi", "serde", "serde_json", @@ -861,15 +863,17 @@ version = "0.3.0" dependencies = [ "arc-swap", "async-trait", - "bincode", "bytes", "crc32fast", "crossbeam-skiplist", "crowdb-chunk-client", "crowdb-chunk-stream", + "crowdb-common", + "crowdb-kv", "crowdb-kv-client", "crowdb-protocol", "crowdb-test-harness", + "futures", "serde", "serde_json", "thiserror 2.0.18", @@ -882,14 +886,17 @@ name = "crowdb-chunkdb" version = "0.3.0" dependencies = [ "arc-swap", + "async-trait", "axum", "bincode", "bytes", "clap", "crossbeam-skiplist", + "crowdb-chunk-client", "crowdb-chunkdb", "crowdb-chunkdb-client", "crowdb-common", + "crowdb-console-shared", "crowdb-diskdb", "crowdb-diskdb-client", "crowdb-diskio-client", @@ -948,6 +955,7 @@ dependencies = [ "crowdb-test-harness", "crowdb-web", "flatbuffers", + "futures", "hex", "libc", "quick-xml", @@ -958,12 +966,14 @@ dependencies = [ "thiserror 1.0.69", "tokio", "tracing", + "uuid", ] [[package]] name = "crowdb-common" version = "0.3.0" dependencies = [ + "crowdb-common", "crowdb-test-harness", "flate2", "libc", @@ -985,10 +995,14 @@ version = "0.3.0" dependencies = [ "async-trait", "axum", + "bincode", + "chrono", + "clap", "crowdb-chunkdb-client", "crowdb-console-shared", "crowdb-kv", "crowdb-kv-client", + "crowdb-kv-server", "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", @@ -996,6 +1010,8 @@ dependencies = [ "form_urlencoded", "futures", "hex", + "hmac", + "percent-encoding", "reqwest", "russh", "serde", @@ -1163,6 +1179,7 @@ dependencies = [ "tokio", "tracing", "utoipa", + "uuid", ] [[package]] @@ -1171,6 +1188,7 @@ version = "0.3.0" dependencies = [ "axum", "clap", + "crowdb-chunk-kv-client", "crowdb-console-shared", "crowdb-diskio-client", "crowdb-kv", @@ -1193,6 +1211,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "toml", + "tracing", "uuid", ] @@ -1212,6 +1231,7 @@ dependencies = [ "serde_json", "sha2", "thiserror 2.0.18", + "url", "utoipa", "xxhash-rust", ] @@ -1231,6 +1251,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "crowdb-rpc-test" +version = "0.3.0" +dependencies = [ + "clap", + "crowdb-chunk-client", + "crowdb-common", + "crowdb-protocol", + "crowdb-rpc-ffi", + "crowdb-test-harness", + "flatbuffers", + "serde", + "serde_json", + "tokio", + "tracing", +] + [[package]] name = "crowdb-test-harness" version = "0.3.0" @@ -1273,12 +1310,14 @@ dependencies = [ "crowdb-protocol", "crowdb-rpc-ffi", "crowdb-test-harness", + "crowdb-web", "dirs", "flatbuffers", "futures", "hex", "hmac", "percent-encoding", + "quick-xml", "reqwest", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 91f0a3143..7d8855b18 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "app/crowdb-access-server", "container/crowdb-monitor", "e2e-test", + "tools/rpc-test", "lib/crowdb-access-s3", "lib/crowdb-access-iceberg", "lib/crowdb-access-dataset", diff --git a/app/crowdb-access-server/src/config.rs b/app/crowdb-access-server/src/config.rs index 8cea2e84f..f17c5f381 100644 --- a/app/crowdb-access-server/src/config.rs +++ b/app/crowdb-access-server/src/config.rs @@ -33,7 +33,7 @@ pub struct HealthConfig { pub enum DeploymentMode { #[default] Production, - TestSingleNode, + Local, } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -265,6 +265,17 @@ impl Default for DatasetConfig { } impl AccessConfig { + #[must_use] + pub fn dataset_small_write(&self) -> SmallWritePolicy { + let accepted = self.small_write.policy(); + let mut policy = SmallWritePolicy::new(crowdb_protocol::chunkdb::rpc::ChunkType::Dataset); + policy.mirror_copies = accepted.mirror_copies; + policy.conversion_enabled = accepted.conversion_enabled; + policy.conversion_data_num = accepted.conversion_data_num; + policy.conversion_code_num = accepted.conversion_code_num; + policy + } + #[must_use] pub fn s3_small_write(&self) -> &SmallWriteConfig { self.s3.small_write.as_ref().unwrap_or(&self.small_write) @@ -361,20 +372,24 @@ impl BaseConfig for AccessConfig { return Err("production access writes require protected strips".into()); } } - DeploymentMode::TestSingleNode => { + DeploymentMode::Local => { if self.deployment.max_node_failures != 0 { - return Err("test_single_node requires max_node_failures = 0".into()); + return Err("local requires max_node_failures = 0".into()); } for config in [self.s3_small_write(), self.iceberg_small_write()] { if config.conversion_enabled - || config.policy().mirror_copies != 1 + || !matches!(config.policy().mirror_copies, 1 | 2) || config.disk_block_bytes != 1024 * 1024 { - return Err("test_single_node requires one-copy 1 MiB mirror writes".into()); + return Err( + "local requires one or two-copy 1 MiB mirror writes without conversion".into(), + ); } } - if self.s3.large_mirror_copies != Some(1) || self.iceberg.large_mirror_copies != Some(1) { - return Err("test_single_node requires one-copy large mirror strips".into()); + if !matches!(self.s3.large_mirror_copies, Some(1 | 2)) + || !matches!(self.iceberg.large_mirror_copies, Some(1 | 2)) + { + return Err("local requires one or two-copy large mirror strips".into()); } } } diff --git a/app/crowdb-access-server/src/dataset.rs b/app/crowdb-access-server/src/dataset.rs index a12897862..a2523639f 100644 --- a/app/crowdb-access-server/src/dataset.rs +++ b/app/crowdb-access-server/src/dataset.rs @@ -166,7 +166,13 @@ pub async fn run_with_shutdown( let client = Arc::new(ChunkKvClient::new(chunk_config, source, transport)?); client.refresh_catalog().await?; let store = Arc::new(ChunkKvDatasetStore::new(client)); - let chunks = connect_payloads(seeds, control, access.dataset.max_in_flight).await?; + let chunks = connect_payloads( + seeds, + control, + access.dataset.max_in_flight, + access.dataset_small_write(), + ) + .await?; let uploads = Arc::new(DatasetUploads::new( store.clone(), Arc::new(ChunkPayloadStorage::new(chunks.clone())), @@ -224,9 +230,9 @@ async fn connect_payloads( seeds: Vec, control: Arc, capacity: usize, + mut small_write: crowdb_chunk_client::SmallWritePolicy, ) -> Result { - use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, SmallWritePolicy}; - let mut small_write = SmallWritePolicy::new(crowdb_protocol::chunkdb::rpc::ChunkType::Dataset); + use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig}; small_write.object_limit = 4 * 1024 * 1024; small_write.max_batch_bytes = 4 * 1024 * 1024; small_write.max_pipelines = capacity.clamp(1, 8); diff --git a/app/crowdb-access-server/src/iceberg/file_http/stream.rs b/app/crowdb-access-server/src/iceberg/file_http/stream.rs index b9a2e34c8..854368f5b 100644 --- a/app/crowdb-access-server/src/iceberg/file_http/stream.rs +++ b/app/crowdb-access-server/src/iceberg/file_http/stream.rs @@ -13,7 +13,7 @@ use crowdb_access_iceberg::file::{ }; use crowdb_access_s3::native_buffer::NativeBodyReceiver; use crowdb_chunk_client::{FramedWriteBuffer, LargeWritePolicy}; -use crowdb_protocol::frame::{FrameMagic, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_protocol::frame::{FrameMagic, MAX_FRAME_V2_PAYLOAD_BYTES}; use http_body_util::BodyExt; use hyper::body::{Bytes, Incoming}; use tokio::sync::mpsc; @@ -28,7 +28,7 @@ use crate::upload_flow::{drive_transfer, write_buffers, OfferStatus, UploadBuffe // Sixteen complete frames occupy one 1-MiB physical mirror strip. A full // logical MiB would spill frame overhead into the next strip and serialize // otherwise independent writes in the non-native receive path. -const TARGET_BUFFER_BYTES: usize = 16 * MAX_FRAME_PAYLOAD_BYTES; +const TARGET_BUFFER_BYTES: usize = 16 * MAX_FRAME_V2_PAYLOAD_BYTES; struct WriteObject<'a> { body: UploadBody, @@ -95,9 +95,9 @@ pub(super) async fn upload( if let Some(receiver) = receiver { receiver .enable_owner_handoff_with_magic(if small.is_some() { - FrameMagic::RepoSmallV1 + FrameMagic::RepoSmallV2 } else { - FrameMagic::RepoLargeV1 + FrameMagic::RepoLargeV2 }) .map_err(|_| FileS3ErrorCode::InternalError)?; } @@ -107,9 +107,9 @@ pub(super) async fn upload( object: object::ObjectWriter::new(writer, small.is_some(), expected_sha256.is_some()) .map_err(|()| FileS3ErrorCode::InternalError)?, frame_magic: if small.is_some() { - FrameMagic::RepoSmallV1 + FrameMagic::RepoSmallV2 } else { - FrameMagic::RepoLargeV1 + FrameMagic::RepoLargeV2 }, held_buffers: large_write.client.large_held_buffers, admission, @@ -200,7 +200,7 @@ impl WriteObject<'_> { } async fn transfer(&mut self) -> Result<(u64, crate::upload_flow::WriteStats), FileS3ErrorCode> { - if self.frame_magic == FrameMagic::RepoSmallV1 { + if self.frame_magic == FrameMagic::RepoSmallV2 { if let Some(receiver) = self.receiver { let (writer, digest) = self.object.parts_mut(); return small_receive::receive( diff --git a/app/crowdb-access-server/src/iceberg/file_http/stream/small_receive.rs b/app/crowdb-access-server/src/iceberg/file_http/stream/small_receive.rs index 7116d485f..aaf7df03a 100644 --- a/app/crowdb-access-server/src/iceberg/file_http/stream/small_receive.rs +++ b/app/crowdb-access-server/src/iceberg/file_http/stream/small_receive.rs @@ -89,7 +89,7 @@ pub(super) async fn receive( }; let started = Instant::now(); owner - .prepare_frames(FrameMagic::RepoSmallV1, super::super::now_ms()?) + .prepare_frames(FrameMagic::RepoSmallV2, super::super::now_ms()?) .map_err(|_| FileS3ErrorCode::InternalError)?; observation.frame_prepare(owner.frame_count(), started.elapsed()); let started = Instant::now(); diff --git a/app/crowdb-access-server/src/main.rs b/app/crowdb-access-server/src/main.rs index 9a298d9cf..523b43849 100644 --- a/app/crowdb-access-server/src/main.rs +++ b/app/crowdb-access-server/src/main.rs @@ -208,11 +208,8 @@ async fn run_s3( )? .unwrap_or(10_000); let health = Arc::new(S3Health::starting(cleanup_backlog_limit)); - health.set_metadata(DependencyHealth::Ready); - health.set_chunks(DependencyHealth::Ready); - health.set_authentication(DependencyHealth::Ready); let (operations, object_inspector) = - configured_s3_operations(storage, service_config, Arc::clone(&metrics), Arc::clone(&health))?; + configured_s3_operations(storage, service_config, Arc::clone(&metrics))?; let expiry_task = start_multipart_expiry(Arc::clone(&operations)); let native_budget = configured_usize( access_config.s3.native_budget_bytes, @@ -237,6 +234,7 @@ async fn run_s3( let health_listener = bind_health_listener(access_config).await?; let health_handler = Arc::new(AccessHealthHandler(Arc::clone(&handler))); health.set_listener(DependencyHealth::Ready); + let health_observer = handler.start_health_observer()?; let serve_result = crowdb_access_server::listeners::serve_s3( serve(listener, handler, wait_for_shutdown(shutdown.clone())), serve(health_listener, health_handler, wait_for_shutdown(shutdown)), @@ -244,6 +242,7 @@ async fn run_s3( ) .await; health.stop(); + drop(health_observer); expiry_task.abort(); let shutdown_result = chunks.shutdown_small_writes().await; if let Some(task) = credential_refresh { @@ -501,7 +500,6 @@ fn configured_s3_operations( storage: S3StorageClients, config: S3ServiceConfig, metrics: Arc, - health: Arc, ) -> Result< ( Arc, @@ -517,7 +515,6 @@ fn configured_s3_operations( .map_err(|error| format!("S3 object inspection configuration failed: {error}"))?; let operations = ProductionS3Operations::new(storage, config) .map_err(|_| "invalid S3 service configuration")? - .with_metrics(metrics) - .with_health(health); + .with_metrics(metrics); Ok((Arc::new(operations), inspector)) } diff --git a/app/crowdb-access-server/src/s3.rs b/app/crowdb-access-server/src/s3.rs index ff93f1fd4..816fcd151 100644 --- a/app/crowdb-access-server/src/s3.rs +++ b/app/crowdb-access-server/src/s3.rs @@ -20,12 +20,14 @@ use hyper_util::rt::TokioIo; use tokio::net::TcpListener; pub mod copy_body; +mod dependencies; mod dispatcher; +pub use dependencies::{DependencyProbeFuture, StorageReadiness}; mod inspection; pub use inspection::{ObjectInspector, OBJECT_LOCATIONS_PATH}; mod health; mod operations; -pub use health::AccessHealthHandler; +pub use health::{AccessHealthHandler, HealthObserver}; pub use crate::http_receive::install_body_receive_provider; pub use crate::multipart_complete::{CompletePart, CompleteRequestError, CompleteSelection}; diff --git a/app/crowdb-access-server/src/s3/dependencies.rs b/app/crowdb-access-server/src/s3/dependencies.rs new file mode 100644 index 000000000..201cbf976 --- /dev/null +++ b/app/crowdb-access-server/src/s3/dependencies.rs @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Fresh read-only dependency evidence, independent of application traffic. + +use std::{future::Future, pin::Pin, time::Duration}; + +use crowdb_access_s3::{metadata::MetadataKey, metrics::DependencyHealth}; + +use super::operations::ProductionS3Operations; + +pub type DependencyProbeFuture = Pin> + Send + 'static>>; + +pub struct StorageReadiness { + pub metadata: Result, + pub chunks: Result, +} + +impl ProductionS3Operations { + pub(super) async fn observe_storage(&self) -> StorageReadiness { + let (metadata, chunks) = tokio::join!( + bounded("metadata", async { + self.storage + .metadata + .get( + MetadataKey::bucket_name(&self.config.tenant, b"crowdb-health-observation") + .map_err(|error| error.to_string())?, + ) + .await + .map_err(|error| error.to_string())?; + Ok("Routed Chunk-KV bucket namespace read succeeded".into()) + }), + async { + Box::pin(self.storage.chunks.observe_readiness(Duration::from_millis(750))) + .await + .map_err(|error| error.to_string()) + }, + ); + StorageReadiness { metadata, chunks } + } +} + +pub(super) async fn bounded( + dependency: &str, + probe: impl Future>, +) -> Result { + tokio::time::timeout(Duration::from_millis(750), probe) + .await + .unwrap_or_else(|_| Err(format!("{dependency} read exceeded its 750 ms deadline"))) +} + +pub(super) fn health(result: &Result) -> DependencyHealth { + if result.is_ok() { + DependencyHealth::Ready + } else { + DependencyHealth::Unavailable + } +} diff --git a/app/crowdb-access-server/src/s3/dispatcher.rs b/app/crowdb-access-server/src/s3/dispatcher.rs index 94843836d..9c16556ca 100644 --- a/app/crowdb-access-server/src/s3/dispatcher.rs +++ b/app/crowdb-access-server/src/s3/dispatcher.rs @@ -7,7 +7,7 @@ use std::time::Instant; use crowdb_access_s3::auth::{AuthError, RawAuthRequest, RequestAuthenticator}; use crowdb_access_s3::error::{S3Error, S3ErrorCode}; -use crowdb_access_s3::metrics::{DependencyHealth, OutcomeClass, RequestMeasurement, S3Health, S3Metrics}; +use crowdb_access_s3::metrics::{OutcomeClass, RequestMeasurement, S3Health, S3Metrics}; use crowdb_access_s3::native_buffer::NativeBodyAllocator; use crowdb_access_s3::route::{classify_request, RouteError, S3Operation}; use crowdb_chunk_client::ChunkIoClient; @@ -21,15 +21,16 @@ use super::{ use crate::http_receive::DeferredBodyReceiveProvider; pub struct S3Dispatcher { - authenticator: Arc, - operations: Arc, - metrics: Arc, + pub(super) authenticator: Arc, + pub(super) operations: Arc, + pub(super) metrics: Arc, host_id: String, trusted_network: bool, body_receive_provider_factory: Option DeferredBodyReceiveProvider + Send + Sync>>, - body_allocator: Option>, + pub(super) body_allocator: Option>, chunk_metrics: Option>, - health: Arc, + pub(super) health: Arc, + pub(super) observations: Arc, next_request_id: AtomicU64, object_inspector: Option>, } @@ -53,6 +54,7 @@ impl S3Dispatcher { body_allocator: None, chunk_metrics: None, health: Arc::new(S3Health::ready(u64::MAX)), + observations: Arc::default(), next_request_id: AtomicU64::new(1), object_inspector: None, } @@ -103,53 +105,15 @@ impl S3Dispatcher { if request.method() != Method::GET && request.method() != Method::HEAD { return None; } - let snapshot = self - .health - .snapshot(&self.metrics, self.body_allocator.as_deref()); let (status, content_type, body) = match request.uri().path() { "/_crowdb/health/live" => { - let status = if snapshot.live { + let live = self.health.is_live(); + let status = if live { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE }; - ( - status, - "application/json", - format!(r#"{{"live":{}}}"#, snapshot.live), - ) - } - "/_crowdb/health/ready" => { - let ready = snapshot.live && snapshot.readiness.is_ready(); - let status = if ready { - StatusCode::OK - } else { - StatusCode::SERVICE_UNAVAILABLE - }; - let readiness = snapshot.readiness; - ( - status, - "application/json", - format!( - concat!( - r#"{{"ready":{},"listener":"{}","metadata":"{}","chunks":"{}",#, - r#""native_pool":"{}","cleanup":"{}","authentication":"{}",#, - r#""native_retained_bytes":{},"native_budget_bytes":{},#, - r#""cleanup_backlog":{},"cleanup_backlog_limit":{}}}"# - ), - ready, - readiness.listener.as_str(), - readiness.metadata.as_str(), - readiness.chunks.as_str(), - readiness.native_pool.as_str(), - readiness.cleanup.as_str(), - readiness.authentication.as_str(), - snapshot.native_retained_bytes, - snapshot.native_budget_bytes, - snapshot.cleanup_backlog, - snapshot.cleanup_backlog_limit, - ), - ) + (status, "application/json", format!(r#"{{"live":{live}}}"#)) } "/_crowdb/metrics" => ( StatusCode::OK, @@ -198,7 +162,6 @@ impl S3Dispatcher { let request_id = self.request_id(); let trusted_network = self.trusted_network; let body_receive_provider_factory = self.body_receive_provider_factory.clone(); - let health = Arc::clone(&self.health); Box::pin(async move { let started = Instant::now(); let _in_flight = metrics.begin_request(); @@ -207,9 +170,6 @@ impl S3Dispatcher { let head_only = request.method() == Method::HEAD; let authentication_started = Instant::now(); if let Err(error) = authenticate_request(authenticator.as_ref(), &mut request).await { - if matches!(error, AuthError::Unavailable) { - health.set_authentication(DependencyHealth::Unavailable); - } let (code, outcome) = auth_error_outcome(error); metrics.finish_predispatch(outcome, elapsed_ns(started)); return Ok(error_response( @@ -217,7 +177,6 @@ impl S3Dispatcher { head_only, )); } - health.set_authentication(DependencyHealth::Ready); let authentication_latency_ns = elapsed_ns(authentication_started); if trusted_network { metrics.record_trusted_auth_bypass(); diff --git a/app/crowdb-access-server/src/s3/health.rs b/app/crowdb-access-server/src/s3/health.rs index 0e7251743..549a28767 100644 --- a/app/crowdb-access-server/src/s3/health.rs +++ b/app/crowdb-access-server/src/s3/health.rs @@ -1,22 +1,35 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{full_body, HandlerFuture, S3Dispatcher, S3HttpHandler}; -use hyper::{body::Incoming, Request, Response, StatusCode}; +//! Health queries only read the backend sampler's latest immutable evidence. + +mod observation; +pub(in crate::s3) use observation::HealthObservations; +pub use observation::HealthObserver; + +use super::{full_body, HandlerFuture, ResponseBody, S3Dispatcher, S3HttpHandler}; +use crowdb_access_s3::metrics::S3Readiness; +use hyper::{body::Incoming, Method, Request, Response, StatusCode}; +use observation::{HealthObservation, MAX_AGE}; +use serde_json::json; use std::sync::Arc; -/// Dedicated health surface sharing the data listener's dependency state. +/// Dedicated health surface sharing the data listener's backend observations. pub struct AccessHealthHandler(pub Arc); impl S3HttpHandler for AccessHealthHandler { fn handle(&self, request: Request) -> HandlerFuture { - let response = if matches!( - request.uri().path(), - "/_crowdb/health/live" | "/_crowdb/health/ready" - ) { - self.0.operational_response(&request) - } else { - None + let response = match request.uri().path() { + "/_crowdb/health/ready" if matches!(*request.method(), Method::GET | Method::HEAD) => { + let observed = self.0.observations.latest.load_full(); + Some(readiness_response( + observed.as_deref(), + self.0.health.is_live(), + request.method() == Method::HEAD, + )) + } + "/_crowdb/health/live" => self.0.operational_response(&request), + _ => None, } .unwrap_or_else(|| { Response::builder() @@ -27,3 +40,83 @@ impl S3HttpHandler for AccessHealthHandler { Box::pin(async move { Ok(response) }) } } + +fn readiness_response( + observed: Option<&HealthObservation>, + live: bool, + head_only: bool, +) -> Response { + let age = observed.map(|value| value.started.elapsed()); + let current = age.is_some_and(|age| age <= MAX_AGE); + let state = if current { + "current" + } else if observed.is_some() { + "expired" + } else { + "pending" + }; + let snapshot = observed.map(|value| value.health); + let readiness = snapshot + .filter(|_| current) + .map_or_else(S3Readiness::unknown, |health| health.readiness); + let ready = live && current && readiness.is_ready(); + let reasons = failure_reasons(readiness, live, current, observed.is_some()); + let body = json!({ + "ready": ready, "live": live, + "listener": readiness.listener.as_str(), "metadata": readiness.metadata.as_str(), + "chunks": readiness.chunks.as_str(), "native_pool": readiness.native_pool.as_str(), + "cleanup": readiness.cleanup.as_str(), "authentication": readiness.authentication.as_str(), + "native_retained_bytes": snapshot.map(|value| value.native_retained_bytes), + "native_budget_bytes": snapshot.map(|value| value.native_budget_bytes), + "cleanup_backlog": snapshot.map(|value| value.cleanup_backlog), + "cleanup_backlog_limit": snapshot.map(|value| value.cleanup_backlog_limit), + "reasons": reasons, "details": observed.map(|value| &value.details), + "observation": { + "state": state, + "checked_at_unix_ms": observed.map(|value| value.checked_at_unix_ms), + "age_ms": age.map(|value| u64::try_from(value.as_millis()).unwrap_or(u64::MAX)), + "max_age_ms": u64::try_from(MAX_AGE.as_millis()).unwrap_or(u64::MAX), + "generation": observed.map_or(0, |value| value.generation), + }, + }); + Response::builder() + .status(if ready { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + }) + .header(hyper::header::CONTENT_TYPE, "application/json") + .body(full_body(if head_only { + hyper::body::Bytes::new() + } else { + body.to_string().into() + })) + .expect("static health response") +} + +fn failure_reasons(readiness: S3Readiness, live: bool, current: bool, observed: bool) -> Vec<&'static str> { + let mut reasons = Vec::new(); + if !live { + reasons.push("live"); + } + if !current { + reasons.push(if observed { + "observation-expired" + } else { + "observation-pending" + }); + } + for (name, dependency) in [ + ("listener", readiness.listener), + ("metadata", readiness.metadata), + ("chunks", readiness.chunks), + ("native_pool", readiness.native_pool), + ("cleanup", readiness.cleanup), + ("authentication", readiness.authentication), + ] { + if !dependency.can_admit() { + reasons.push(name); + } + } + reasons +} diff --git a/app/crowdb-access-server/src/s3/health/observation.rs b/app/crowdb-access-server/src/s3/health/observation.rs new file mode 100644 index 000000000..61fb9b834 --- /dev/null +++ b/app/crowdb-access-server/src/s3/health/observation.rs @@ -0,0 +1,141 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One backend-owned sampler publishes immutable, expiring health evidence. + +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +use arc_swap::ArcSwapOption; +use crowdb_access_s3::metrics::{DependencyHealth, S3HealthSnapshot}; +use tokio::time::{Instant, MissedTickBehavior}; + +use super::super::{dependencies, S3Dispatcher, StorageReadiness}; + +pub(super) const MAX_AGE: Duration = Duration::from_secs(3); + +#[derive(Default)] +pub(in crate::s3) struct HealthObservations { + pub(super) latest: ArcSwapOption, + running: AtomicBool, +} + +pub(super) struct HealthObservation { + pub health: S3HealthSnapshot, + pub details: serde_json::Map, + pub started: Instant, + pub checked_at_unix_ms: u64, + pub generation: u64, +} + +/// Owns the background sampler; dropping it cancels in-flight observation work. +pub struct HealthObserver(tokio::task::JoinHandle<()>); + +impl Drop for HealthObserver { + fn drop(&mut self) { + self.0.abort(); + } +} + +struct SamplingLease(Arc); + +impl Drop for SamplingLease { + fn drop(&mut self) { + self.0.running.store(false, Ordering::Release); + } +} + +impl S3Dispatcher { + /// Starts one sampler independently of health and business requests. + /// + /// # Errors + /// Rejects another sampler while an existing worker or its cancellation is active. + pub fn start_health_observer(self: &Arc) -> Result { + self.observations + .running + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| "Access health sampler is already running")?; + let dispatcher = Arc::clone(self); + let lease = SamplingLease(Arc::clone(&self.observations)); + Ok(HealthObserver(tokio::spawn(async move { + let _lease = lease; + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + interval.tick().await; + let observation = dispatcher.sample_health().await; + dispatcher.observations.latest.store(Some(Arc::new(observation))); + } + }))) + } + + async fn sample_health(&self) -> HealthObservation { + let started = Instant::now(); + let checked_at_unix_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) + }); + let observed = tokio::time::timeout( + Duration::from_millis(800), + Arc::clone(&self.operations).observe_dependencies(), + ) + .await + .unwrap_or_else(|_| { + Some(StorageReadiness { + metadata: Err("Dependency observation exceeded its 800 ms deadline".into()), + chunks: Err("Dependency observation exceeded its 800 ms deadline".into()), + }) + }); + let mut health = self + .health + .snapshot(&self.metrics, self.body_allocator.as_deref()); + let mut details = serde_json::Map::new(); + if let Some(observed) = observed { + health.readiness.metadata = dependencies::health(&observed.metadata); + health.readiness.chunks = dependencies::health(&observed.chunks); + for (name, result) in [("metadata", observed.metadata), ("chunks", observed.chunks)] { + let detail = match result { + Ok(detail) | Err(detail) => detail, + }; + details.insert(name.into(), serde_json::Value::String(detail)); + } + } + if let Some(ready) = self.authenticator.is_ready() { + health.readiness.authentication = if ready { + DependencyHealth::Ready + } else { + DependencyHealth::Unavailable + }; + } + let generation = self + .observations + .latest + .load() + .as_ref() + .map_or(1, |last| last.generation + 1); + HealthObservation { + health, + details, + started, + checked_at_unix_ms, + generation, + } + } + + /// Latest completed backend sampling generation, without causing any work. + #[cfg(feature = "test-util")] + #[must_use] + pub fn health_generation_for_tests(&self) -> u64 { + self.observations + .latest + .load() + .as_ref() + .map_or(0, |last| last.generation) + } +} diff --git a/app/crowdb-access-server/src/s3/operations.rs b/app/crowdb-access-server/src/s3/operations.rs index 648f99ab5..846559786 100644 --- a/app/crowdb-access-server/src/s3/operations.rs +++ b/app/crowdb-access-server/src/s3/operations.rs @@ -12,7 +12,6 @@ use crowdb_access_s3::condition::ObjectConditions; use crowdb_access_s3::continuation::ContinuationTokenSigner; use crowdb_access_s3::error::{S3Error, S3ErrorCode}; use crowdb_access_s3::metadata::{ObjectRecord, TenantId}; -use crowdb_access_s3::metrics::{DependencyHealth, S3Health}; use crowdb_access_s3::object::{self, ListObjectsV2Request, ObjectMetadataError}; use crowdb_access_s3::publication::PublicationRequest; use crowdb_access_s3::retrieval::{self, ObjectHeaders, RetrievalError}; @@ -52,6 +51,10 @@ const DEFAULT_LIST_SCAN_BYTES: usize = 4 * 1024 * 1024; pub type S3OperationsFuture = Pin> + Send + 'static>>; pub trait S3Operations: Send + Sync + 'static { + fn observe_dependencies(self: Arc) -> super::DependencyProbeFuture { + Box::pin(async { None }) + } + fn execute( self: Arc, route: S3Route, @@ -93,12 +96,11 @@ impl S3ServiceConfig { } pub struct ProductionS3Operations { - storage: S3StorageClients, - config: S3ServiceConfig, + pub(super) storage: S3StorageClients, + pub(super) config: S3ServiceConfig, signer: ContinuationTokenSigner, bucket_ids: RandomBucketIdGenerator, metrics: Option>, - health: Option>, } impl ProductionS3Operations { @@ -116,7 +118,6 @@ impl ProductionS3Operations { signer, bucket_ids: RandomBucketIdGenerator, metrics: None, - health: None, }) } @@ -126,12 +127,6 @@ impl ProductionS3Operations { self } - #[must_use] - pub fn with_health(mut self, health: Arc) -> Self { - self.health = Some(health); - self - } - async fn dispatch( self: &Arc, route: S3Route, @@ -162,46 +157,12 @@ impl ProductionS3Operations { S3Operation::AbortMultipartUpload => self.abort_multipart_upload(route).await, S3Operation::ListMultipartUploads => self.list_multipart_uploads(route, &request).await, }; - self.record_dependency_outcome(operation, &result); result.unwrap_or_else(|code| { tracing::debug!(%request_id, ?code, "S3 request failed"); error_response(&S3Error::new(code, resource, request_id, host_id), head_only) }) } - fn record_dependency_outcome( - &self, - operation: S3Operation, - result: &Result, S3ErrorCode>, - ) { - let Some(health) = &self.health else { - return; - }; - let uses_chunks = matches!( - operation, - S3Operation::PutObject - | S3Operation::GetObject - | S3Operation::UploadPart - | S3Operation::CopyObject - | S3Operation::UploadPartCopy - ); - match result { - Ok(_) => { - health.set_metadata(DependencyHealth::Ready); - if uses_chunks { - health.set_chunks(DependencyHealth::Ready); - } - } - Err(S3ErrorCode::ServiceUnavailable) => { - health.set_metadata(DependencyHealth::Unavailable); - if uses_chunks { - health.set_chunks(DependencyHealth::Unavailable); - } - } - Err(_) => {} - } - } - async fn create_bucket(&self, route: S3Route) -> Result, S3ErrorCode> { let name = required_bucket(&route)?; bucket::create_bucket( @@ -491,7 +452,7 @@ impl ProductionS3Operations { .and_then(|length| usize::try_from(length).ok()) .filter(|length| *length <= self.config.small_object_limit) { - if length > crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES { + if length > crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES { return self .storage .chunks @@ -520,6 +481,10 @@ impl ProductionS3Operations { } impl S3Operations for ProductionS3Operations { + fn observe_dependencies(self: Arc) -> super::DependencyProbeFuture { + Box::pin(async move { Some(Box::pin(self.observe_storage()).await) }) + } + fn execute( self: Arc, route: S3Route, diff --git a/app/crowdb-access-server/src/s3/operations/upload.rs b/app/crowdb-access-server/src/s3/operations/upload.rs index 6d09c53e9..1980416b8 100644 --- a/app/crowdb-access-server/src/s3/operations/upload.rs +++ b/app/crowdb-access-server/src/s3/operations/upload.rs @@ -176,7 +176,7 @@ async fn handoff_owner( u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) }); owner - .prepare_frames(FrameMagic::RepoLargeV1, now_ms) + .prepare_frames(FrameMagic::RepoLargeV2, now_ms) .map_err(|error| failed(PutErrorCode::ChunkWrite, &error))?; handoff(flow, digest, UploadBuffer::Framed(Box::new(owner)), payload).await } diff --git a/app/crowdb-access-server/tests/access_readiness_test.rs b/app/crowdb-access-server/tests/access_readiness_test.rs new file mode 100644 index 000000000..4e54c6e14 --- /dev/null +++ b/app/crowdb-access-server/tests/access_readiness_test.rs @@ -0,0 +1,150 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#![cfg(feature = "s3")] + +#[path = "common/access_health.rs"] +mod fixture; +use crowdb_access_s3::auth::{ + AuthError, CredentialCache, RawAuthRequest, RequestAuthenticator, SigV4Verifier, + TrustedNetworkAuthenticator, +}; +use fixture::TestHealth; +use serde_json::Value; +use std::{ + sync::{atomic::Ordering, Arc}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +#[tokio::test] +async fn backend_recovers_without_queries_and_preserves_stop() { + let fixture = TestHealth::new(Arc::new(TrustedNetworkAuthenticator::new()), 1, true).await; + let failed = fixture.read(503).await; + assert_eq!(failed["metadata"], "unavailable"); + assert_eq!(failed["chunks"], "ready"); + assert_eq!(failed["details"]["metadata"], "metadata owner unavailable"); + let generation = fixture.generation(); + fixture.operations.phase.store(2, Ordering::Relaxed); + fixture.after(generation).await; + let failed = fixture.read(503).await; + assert_eq!(failed["metadata"], "ready"); + assert_eq!(failed["chunks"], "unavailable"); + let generation = fixture.generation(); + fixture.operations.phase.store(0, Ordering::Relaxed); + fixture.after(generation).await; + assert_eq!(fixture.read(200).await["ready"], true); + fixture.health.stop(); + let stopped = fixture.read(503).await; + assert!(stopped["reasons"] + .as_array() + .unwrap() + .contains(&Value::from("live"))); +} + +#[tokio::test] +async fn pending_and_expired_evidence_remain_unknown_and_queries_never_probe() { + let mut fixture = TestHealth::new(Arc::new(TrustedNetworkAuthenticator::new()), 0, false).await; + let pending = fixture.read(503).await; + assert_eq!(pending["metadata"], "unknown"); + assert_eq!(pending["observation"]["state"], "pending"); + assert_eq!(fixture.operations.calls.load(Ordering::Relaxed), 0); + fixture.observer = Some(fixture.dispatcher.start_health_observer().unwrap()); + fixture.after(0).await; + let before = fixture.read(200).await; + fixture.observer.take(); + tokio::task::yield_now().await; + let calls = fixture.operations.calls.load(Ordering::Relaxed); + let reads = futures::future::join_all((0..32).map(|_| fixture.read(200))).await; + for read in reads { + assert_eq!( + read["observation"]["generation"], + before["observation"]["generation"] + ); + } + assert_eq!(fixture.operations.calls.load(Ordering::Relaxed), calls); + tokio::time::sleep(Duration::from_millis(3050)).await; + let expired = fixture.read(503).await; + assert_eq!(expired["metadata"], "unknown"); + assert_eq!(expired["chunks"], "unknown"); + assert_eq!(expired["observation"]["state"], "expired"); + assert_eq!( + expired["observation"]["checked_at_unix_ms"], + before["observation"]["checked_at_unix_ms"] + ); + assert_eq!(fixture.operations.calls.load(Ordering::Relaxed), calls); +} + +#[tokio::test] +async fn stalled_sampler_is_bounded_single_flight_and_queries_stay_fast() { + let mut fixture = TestHealth::new(Arc::new(TrustedNetworkAuthenticator::new()), 3, true).await; + assert!(fixture.dispatcher.start_health_observer().is_err()); + let started = Instant::now(); + let failed = fixture.read(503).await; + assert!(started.elapsed() < Duration::from_millis(500)); + assert!(failed["details"]["metadata"] + .as_str() + .unwrap() + .contains("deadline")); + let generation = fixture.generation(); + fixture.after(generation).await; + assert_eq!(fixture.operations.peak.load(Ordering::Relaxed), 1); + fixture.observer.take(); + tokio::task::yield_now().await; + assert_eq!(fixture.operations.in_flight.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn credential_refresh_and_cleanup_update_in_background() { + let cache = Arc::new(CredentialCache::new(90)); + let fixture = TestHealth::new( + Arc::new(SigV4Verifier::new(cache.clone(), "us-east-1".into(), 900)), + 0, + true, + ) + .await; + assert_eq!(fixture.read(503).await["authentication"], "unavailable"); + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_secs(); + cache.install([], now - 91).unwrap(); + let generation = fixture.generation(); + fixture.after(generation).await; + assert_eq!(fixture.read(503).await["authentication"], "unavailable"); + let generation = fixture.generation(); + cache.install([], now).unwrap(); + fixture.after(generation).await; + assert_eq!(fixture.read(200).await["authentication"], "ready"); + let generation = fixture.generation(); + fixture.metrics.enqueue_cleanup(2); + fixture.after(generation).await; + assert_eq!(fixture.read(503).await["cleanup"], "unavailable"); + let generation = fixture.generation(); + fixture.metrics.complete_cleanup(2); + fixture.after(generation).await; + assert_eq!(fixture.read(200).await["cleanup"], "ready"); +} + +struct TestUnavailableAuthenticator; +#[async_trait::async_trait] +impl RequestAuthenticator for TestUnavailableAuthenticator { + fn is_ready(&self) -> Option { + Some(true) + } + async fn authenticate(&self, _: RawAuthRequest<'_>) -> Result<(), AuthError> { + Err(AuthError::Unavailable) + } +} +#[tokio::test] +async fn business_request_failure_cannot_change_cached_health() { + let mut fixture = TestHealth::new(Arc::new(TestUnavailableAuthenticator), 0, true).await; + fixture.observer.take(); + let before = fixture.read(200).await; + fixture.business_failure().await; + let after = fixture.read(200).await; + assert_eq!(before["authentication"], after["authentication"]); + assert_eq!( + before["observation"]["generation"], + after["observation"]["generation"] + ); + assert_eq!( + before["observation"]["checked_at_unix_ms"], + after["observation"]["checked_at_unix_ms"] + ); +} diff --git a/app/crowdb-access-server/tests/common/access_health.rs b/app/crowdb-access-server/tests/common/access_health.rs new file mode 100644 index 000000000..a1a4a1bf0 --- /dev/null +++ b/app/crowdb-access-server/tests/common/access_health.rs @@ -0,0 +1,170 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_access_s3::{ + auth::RequestAuthenticator, + metrics::{DependencyHealth, S3Health, S3Metrics}, + route::S3Route, +}; +use crowdb_access_server::s3::{ + serve, AccessHealthHandler, DependencyProbeFuture, HealthObserver, S3Dispatcher, S3Operations, + S3OperationsFuture, StorageReadiness, +}; +use hyper::{body::Incoming, Request}; +use serde_json::Value; +use std::{ + sync::{ + atomic::{AtomicU8, AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; + +pub struct TestDependencies { + pub phase: AtomicU8, + pub calls: AtomicUsize, + pub in_flight: AtomicUsize, + pub peak: AtomicUsize, +} + +struct TestProbe<'a>(&'a AtomicUsize); +impl Drop for TestProbe<'_> { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::Relaxed); + } +} + +impl S3Operations for TestDependencies { + fn observe_dependencies(self: Arc) -> DependencyProbeFuture { + Box::pin(async move { + self.calls.fetch_add(1, Ordering::Relaxed); + let count = self.in_flight.fetch_add(1, Ordering::Relaxed) + 1; + self.peak.fetch_max(count, Ordering::Relaxed); + let _probe = TestProbe(&self.in_flight); + let phase = self.phase.load(Ordering::Relaxed); + if phase == 3 { + std::future::pending::<()>().await; + } + Some(StorageReadiness { + metadata: if phase == 1 { + Err("metadata owner unavailable".into()) + } else { + Ok("current routed read".into()) + }, + chunks: if phase == 2 { + Err("physical disk read failed".into()) + } else { + Ok("current disk reads".into()) + }, + }) + }) + } + fn execute( + self: Arc, + _: S3Route, + _: Request, + _: String, + _: String, + ) -> S3OperationsFuture { + panic!("readiness must not dispatch application operations"); + } +} + +pub struct TestHealth { + pub url: String, + pub data_url: String, + tasks: Vec>>, + pub dispatcher: Arc, + pub operations: Arc, + pub health: Arc, + pub metrics: Arc, + pub observer: Option, + client: reqwest::Client, +} + +impl TestHealth { + pub async fn new(authenticator: Arc, phase: u8, start: bool) -> Self { + let operations = Arc::new(TestDependencies { + phase: AtomicU8::new(phase), + calls: AtomicUsize::new(0), + in_flight: AtomicUsize::new(0), + peak: AtomicUsize::new(0), + }); + let health = Arc::new(S3Health::starting(1)); + health.set_listener(DependencyHealth::Ready); + let metrics = Arc::new(S3Metrics::default()); + let dispatcher = Arc::new( + S3Dispatcher::new( + authenticator, + operations.clone(), + metrics.clone(), + "test".into(), + false, + ) + .with_health(health.clone()), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/_crowdb/health/ready", listener.local_addr().unwrap()); + let task = tokio::spawn(serve( + listener, + Arc::new(AccessHealthHandler(dispatcher.clone())), + std::future::pending(), + )); + let data = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let data_url = format!("http://{}/", data.local_addr().unwrap()); + let data_task = tokio::spawn(serve(data, dispatcher.clone(), std::future::pending())); + let observer = start.then(|| dispatcher.start_health_observer().unwrap()); + let fixture = Self { + url, + data_url, + tasks: vec![task, data_task], + dispatcher, + operations, + health, + metrics, + observer, + client: reqwest::Client::builder().no_proxy().build().unwrap(), + }; + if start { + fixture.after(0).await; + } + fixture + } + pub fn generation(&self) -> u64 { + self.dispatcher.health_generation_for_tests() + } + pub async fn after(&self, generation: u64) { + tokio::time::timeout(Duration::from_secs(3), async { + while self.generation() <= generation { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("backend sampler did not publish the next generation"); + } + pub async fn read(&self, status: u16) -> Value { + let response = self.client.get(&self.url).send().await.unwrap(); + assert_eq!(response.status().as_u16(), status); + serde_json::from_slice(&response.bytes().await.unwrap()).unwrap() + } + pub async fn business_failure(&self) { + assert_eq!( + self.client + .get(&self.data_url) + .send() + .await + .unwrap() + .status() + .as_u16(), + 503 + ); + } +} +impl Drop for TestHealth { + fn drop(&mut self) { + self.observer.take(); + for task in &self.tasks { + task.abort(); + } + } +} diff --git a/app/crowdb-access-server/tests/common/iceberg_file_recovery/allocator.rs b/app/crowdb-access-server/tests/common/iceberg_file_recovery/allocator.rs index 631636cc4..988b6a678 100644 --- a/app/crowdb-access-server/tests/common/iceberg_file_recovery/allocator.rs +++ b/app/crowdb-access-server/tests/common/iceberg_file_recovery/allocator.rs @@ -229,7 +229,8 @@ impl ChunkAllocator for TestAllocator { self.update(|state| { let chunk = state.chunks.get_mut(&(id.high, id.low)).unwrap(); chunk.state = ChunkState::Sealed as i32; - chunk.sealed_length = req.seal_length; + chunk.sealed_length = u32::try_from(req.seal_bytes.div_ceil(1024)).unwrap(); + chunk.acknowledged_cursor = req.seal_bytes; Ok(SealChunkResponse { chunk: Some(chunk.clone()), }) @@ -247,10 +248,6 @@ impl ChunkAllocator for TestAllocator { }) } - async fn update_chunk_strip(&self, _req: UpdateChunkStripRequest) -> Result { - unreachable!() - } - async fn query_chunk(&self, req: QueryChunkRequest) -> Result { let id = req.chunk_id.unwrap(); let state = self.state.load(); diff --git a/app/crowdb-access-server/tests/common/iceberg_single_node.toml b/app/crowdb-access-server/tests/common/iceberg_single_node.toml index 6321d79d8..2fb03b548 100644 --- a/app/crowdb-access-server/tests/common/iceberg_single_node.toml +++ b/app/crowdb-access-server/tests/common/iceberg_single_node.toml @@ -1,5 +1,5 @@ [deployment] -mode = "test_single_node" +mode = "local" max_node_failures = 0 [small_write] diff --git a/app/crowdb-access-server/tests/config_test.rs b/app/crowdb-access-server/tests/config_test.rs index 92f1f9f6d..07c72b2db 100644 --- a/app/crowdb-access-server/tests/config_test.rs +++ b/app/crowdb-access-server/tests/config_test.rs @@ -162,3 +162,35 @@ fn independent_health_listener_rejects_data_listener_ports() { assert!(config.validate().is_err(), "{value}"); } } + +#[test] +fn local_one_and_three_node_mirrors_are_explicit_and_never_relax_production() { + use crowdb_access_server::config::DeploymentMode; + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../container/single-node-container/templates/access.toml"); + for copies in [1, 2] { + let mut config: AccessConfig = load_from_file(&path).unwrap(); + config.small_write.mirror_copies = Some(copies); + config.s3.small_write = None; + config.iceberg.small_write = None; + config.s3.large_mirror_copies = Some(copies); + config.iceberg.large_mirror_copies = Some(copies); + config.validate().unwrap(); + assert_eq!(config.deployment.mode, DeploymentMode::Local); + let dataset = config.dataset_small_write(); + assert_eq!(dataset.mirror_copies, copies); + assert_eq!( + dataset.chunk_type, + crowdb_protocol::chunkdb::rpc::ChunkType::Dataset + ); + assert!(!dataset.conversion_enabled); + config.small_write.conversion_enabled = true; + assert!(config.validate().is_err()); + config.small_write.conversion_enabled = false; + config.deployment.mode = DeploymentMode::Production; + assert!( + config.validate().is_err(), + "local loss budget must never become production protection" + ); + } +} diff --git a/app/crowdb-access-server/tests/dataset_sdk_python_split_test.rs b/app/crowdb-access-server/tests/dataset_sdk_python_split_test.rs index 8c1d4597b..6d01c70db 100644 --- a/app/crowdb-access-server/tests/dataset_sdk_python_split_test.rs +++ b/app/crowdb-access-server/tests/dataset_sdk_python_split_test.rs @@ -6,7 +6,7 @@ use dataset::{TestDatasetHttp, TestStore}; use std::{path::Path, process::Command, sync::Arc}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires CROWDB_DATASET_PYTHON pointing to an installed wheel consumer"] +#[ignore = "requires an installed SDK wheel; run pixi run test-dataset-sdk-python"] async fn installed_python_split_sdk_uses_the_real_http_listener() { let python = std::env::var("CROWDB_DATASET_PYTHON").expect("installed wheel Python path"); let test = TestDatasetHttp::start_payload(Arc::new(TestStore::default())).await; diff --git a/app/crowdb-access-server/tests/service_test.rs b/app/crowdb-access-server/tests/service_test.rs index 8a248595f..eb3905396 100644 --- a/app/crowdb-access-server/tests/service_test.rs +++ b/app/crowdb-access-server/tests/service_test.rs @@ -108,8 +108,9 @@ mod s3_dispatcher { let address = listener.local_addr().unwrap(); let (health_address, health_server) = start_health_server(&dispatcher).await; let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let data_dispatcher = Arc::clone(&dispatcher); let server = tokio::spawn(async move { - serve(listener, dispatcher, async move { + serve(listener, data_dispatcher, async move { let _ = shutdown_rx.await; }) .await @@ -186,7 +187,7 @@ mod s3_dispatcher { assert_eq!(body_allocator.retained_bytes(), 0); assert_terminal_outcomes(address, &operations, &metrics).await; - assert_health_isolation(address, health_address, &metrics).await; + assert_health_isolation(address, health_address, &metrics, &dispatcher).await; health_server.abort(); let _ = shutdown_tx.send(()); server.await.unwrap(); @@ -200,7 +201,12 @@ mod s3_dispatcher { let health_handler = Arc::new(crowdb_access_server::s3::AccessHealthHandler(Arc::clone( dispatcher, ))); - let health_server = tokio::spawn(serve(health_listener, health_handler, std::future::pending())); + let observer = dispatcher.start_health_observer().unwrap(); + wait_health_sample(dispatcher, 0).await; + let health_server = tokio::spawn(async move { + let _observer = observer; + serve(health_listener, health_handler, std::future::pending()).await + }); (health_address, health_server) } @@ -208,11 +214,18 @@ mod s3_dispatcher { address: std::net::SocketAddr, health_address: std::net::SocketAddr, metrics: &S3Metrics, + dispatcher: &Arc, ) { + let generation = dispatcher.health_generation_for_tests(); metrics.enqueue_cleanup(2); + wait_health_sample(dispatcher, generation).await; let not_ready = request(health_address, "GET /_crowdb/health/ready HTTP/1.1").await; assert!(not_ready.starts_with("HTTP/1.1 503")); assert!(not_ready.contains(r#""cleanup":"unavailable""#)); + let body: serde_json::Value = + serde_json::from_str(not_ready.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body["ready"], false); + assert_eq!(body["cleanup"], "unavailable"); let data_on_health = request(health_address, "GET /bucket/key HTTP/1.1").await; assert!(data_on_health.starts_with("HTTP/1.1 404")); @@ -220,6 +233,16 @@ mod s3_dispatcher { assert!(!health_on_data.contains(r#""ready":true"#)); } + async fn wait_health_sample(dispatcher: &S3Dispatcher, generation: u64) { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while dispatcher.health_generation_for_tests() <= generation { + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } + }) + .await + .expect("backend health sample did not advance"); + } + async fn assert_operational_endpoints( address: std::net::SocketAddr, health_address: std::net::SocketAddr, @@ -231,6 +254,9 @@ mod s3_dispatcher { let ready = request(health_address, "GET /_crowdb/health/ready HTTP/1.1").await; assert!(ready.starts_with("HTTP/1.1 200")); assert!(ready.contains(r#""ready":true"#)); + let body: serde_json::Value = serde_json::from_str(ready.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!(body["ready"], true); + assert_eq!(body["authentication"], "ready"); let exported = request(address, "GET /_crowdb/metrics HTTP/1.1").await; assert!(exported.starts_with("HTTP/1.1 200")); assert!(exported.contains(r#"crowdb_s3_requests_total{operation="put_object",outcome="success"} 0"#)); diff --git a/app/crowdb-chunk-kv-server/Cargo.toml b/app/crowdb-chunk-kv-server/Cargo.toml index 666ebb60b..d441698d8 100644 --- a/app/crowdb-chunk-kv-server/Cargo.toml +++ b/app/crowdb-chunk-kv-server/Cargo.toml @@ -35,6 +35,7 @@ toml = "0.8" tracing = { workspace = true } [dev-dependencies] +crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } crowdb-kv = { path = "../../lib/crowdb-kv" } crowdb-chunk-kv-server = { path = ".", features = ["test-util"] } crowdb-chunk-kv = { path = "../../lib/crowdb-chunk-kv", features = ["test-util"] } diff --git a/app/crowdb-chunk-kv-server/src/catalog/transition.rs b/app/crowdb-chunk-kv-server/src/catalog/transition.rs index 8e0a4560a..d45c1cab6 100644 --- a/app/crowdb-chunk-kv-server/src/catalog/transition.rs +++ b/app/crowdb-chunk-kv-server/src/catalog/transition.rs @@ -49,7 +49,7 @@ impl ChunkKvRangeCatalogCutover { .await? .ok_or(ChunkKvRangeCatalogError::TransitionConflict)?; let desired = transfer_entry(transition); - if exact_entries(&pages, |entry| entry == &desired) == 1 { + if exact_entries(&pages, |entry| same_assignment(entry, &desired)) == 1 { reject_reused_transition_id(&pages, transition.transition_id, 1)?; return Ok(head.generation); } @@ -59,12 +59,12 @@ impl ChunkKvRangeCatalogCutover { ); reject_reused_transition_id(&pages, transition.transition_id, usize::from(advancing))?; let source = |entry: &ChunkKvRangeCatalogEntry| { - let source_serving = entry.owner == transition.source + let source_serving = entry.owner.instance_id == transition.source.instance_id && entry.owner_epoch == transition.source_epoch && entry.state == ChunkKvRangeCatalogPartitionState::Serving && entry.artifact == transition.artifact && entry.transition_id.is_none(); - let target_catching_up = entry.owner == transition.target + let target_catching_up = entry.owner.instance_id == transition.target.instance_id && entry.owner_epoch == transition.target_epoch && entry.state == ChunkKvRangeCatalogPartitionState::TargetCatchingUp && entry.artifact == transition.target_artifact @@ -106,7 +106,7 @@ impl ChunkKvRangeCatalogCutover { let desired = split_entries(transition); if desired .iter() - .all(|wanted| exact_entries(&pages, |entry| entry == wanted) == 1) + .all(|wanted| exact_entries(&pages, |entry| same_assignment(entry, wanted)) == 1) { reject_reused_transition_id(&pages, transition.transition_id, 2)?; return Ok(head.generation); @@ -115,7 +115,7 @@ impl ChunkKvRangeCatalogCutover { let parent = |entry: &ChunkKvRangeCatalogEntry| { entry.partition_id == transition.parent_id && entry.range == transition.parent_range - && entry.owner == transition.parent_owner + && entry.owner.instance_id == transition.parent_owner.instance_id && entry.owner_epoch == transition.parent_epoch && entry.state == ChunkKvRangeCatalogPartitionState::Serving && entry.artifact == transition.parent_artifact @@ -128,6 +128,15 @@ impl ChunkKvRangeCatalogCutover { } } +fn same_assignment(current: &ChunkKvRangeCatalogEntry, expected: &ChunkKvRangeCatalogEntry) -> bool { + let mut current = current.clone(); + current + .owner + .rpc_endpoint + .clone_from(&expected.owner.rpc_endpoint); + current == *expected +} + fn transfer_entry(transition: &TransferTransition) -> ChunkKvRangeCatalogEntry { ChunkKvRangeCatalogEntry { partition_id: transition.partition_id, diff --git a/app/crowdb-chunk-kv-server/src/config.rs b/app/crowdb-chunk-kv-server/src/config.rs index 57390909f..ae19dd380 100644 --- a/app/crowdb-chunk-kv-server/src/config.rs +++ b/app/crowdb-chunk-kv-server/src/config.rs @@ -187,9 +187,8 @@ pub struct StorageConfig { pub stream_writer_lease_ms: u64, pub stream_mirror_copies: u32, pub tree_chunk_capacity_bytes: u64, + pub tree_strip_capacity_bytes: u32, pub stream_chunk_capacity_bytes: u64, - /// Maximum sealed extents per immutable stream directory page. - pub stream_extent_page_entries: usize, pub diskio_connections_per_endpoint: usize, pub diskio_rpc_workers: u32, } @@ -201,8 +200,8 @@ impl Default for StorageConfig { stream_writer_lease_ms: 30_000, stream_mirror_copies: 2, tree_chunk_capacity_bytes: 256 * 1024 * 1024, + tree_strip_capacity_bytes: 0, stream_chunk_capacity_bytes: 256 * 1024 * 1024, - stream_extent_page_entries: 1_024, diskio_connections_per_endpoint: 1, diskio_rpc_workers: 2, } @@ -215,13 +214,15 @@ impl StorageConfig { || self.stream_mirror_copies == 0 || self.stream_mirror_copies > 5 || !(1024 * 1024..=256 * 1024 * 1024).contains(&self.tree_chunk_capacity_bytes) + || self.tree_strip_capacity_bytes != 0 + && (!(1024 * 1024..=256 * 1024 * 1024).contains(&self.tree_strip_capacity_bytes) + || self.tree_strip_capacity_bytes % (1024 * 1024) != 0) || !(1024 * 1024..=256 * 1024 * 1024).contains(&self.stream_chunk_capacity_bytes) - || self.stream_extent_page_entries == 0 || self.diskio_connections_per_endpoint == 0 || self.diskio_rpc_workers == 0 { return Err(ConfigError::Invalid( - "storage lease, mirror copies, connections, workers, tree/stream chunk capacities, and extent page entries must be valid" + "storage lease, mirror copies, connections, workers, and tree/stream chunk capacities must be valid" .into(), )); } diff --git a/app/crowdb-chunk-kv-server/src/lib.rs b/app/crowdb-chunk-kv-server/src/lib.rs index ba7aad1b2..aaf83a08d 100644 --- a/app/crowdb-chunk-kv-server/src/lib.rs +++ b/app/crowdb-chunk-kv-server/src/lib.rs @@ -24,13 +24,14 @@ pub use management::{management_router, ManagementState}; pub use metrics::{ServerMetrics, ServerMetricsSnapshot}; pub use rpc::ChunkKvRpcService; pub use server::{ - ChunkKvRangeCatalogReconcileError, ChunkKvService, HostedPartitionHealth, ServerHealth, ServerLifecycle, + ChunkKvRangeCatalogReconcileError, ChunkKvService, HostedPartitionHealth, RangeReadiness, ServerHealth, + ServerLifecycle, }; pub use serving::{ choose_split, choose_transfer, classify_instance, desired_partition_count, replacement_may_activate, AuthorityError, BalanceConfig, DomainMonitorDriver, DomainMonitorRegistry, MonitorDescriptorStore, MonitorError, MonitorTick, OwnerLoad, PartitionLoad, PreparedLocalSplit, PreparedMonitor, - ServingAuthority, SplitAction, SplitProposal, SplitStateMachine, TransferAction, TransferProposal, - TransferStateMachine, TransitionExecutor, TransitionProcessor, TransitionStorage, + ServingAuthority, ServingContext, SplitAction, SplitProposal, SplitStateMachine, TransferAction, + TransferProposal, TransferStateMachine, TransitionExecutor, TransitionProcessor, TransitionStorage, }; pub use storage::{ChunkKvStorage, StorageRuntimeError}; diff --git a/app/crowdb-chunk-kv-server/src/main.rs b/app/crowdb-chunk-kv-server/src/main.rs index 9a619c63c..bfed44c12 100644 --- a/app/crowdb-chunk-kv-server/src/main.rs +++ b/app/crowdb-chunk-kv-server/src/main.rs @@ -14,7 +14,7 @@ use crowdb_chunk_kv_server::{ ChunkKvStorage, DomainMonitorRegistry, Group0ControlStore, ManagementState, StorageRuntimeError, TransitionExecutor, TransitionProcessor, }; -use crowdb_kv_client::{ServiceRegistryClient, WatchNotifyClient, WatchSubscription}; +use crowdb_kv_client::{WatchNotifyClient, WatchSubscription}; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, ChunkKvRangeCatalogPartitionState, EnsureDomainMonitorOutcome, EnsureDomainMonitorRequest, KeyRange, @@ -51,6 +51,10 @@ struct Cli { #[arg(short = 'l', long)] log: bool, + /// Perf log interval in seconds; zero disables periodic metrics. + #[arg(long, default_value_t = 1)] + metrics_interval: u64, + /// Maximum log file size in MiB before rotation. #[arg(long, default_value_t = crowdb_common::logging::DEFAULT_LOG_MAX_FILE_MB)] log_max_file_mb: usize, @@ -142,6 +146,14 @@ async fn main() -> std::process::ExitCode { "crowdb-chunk-kv-server starting" ); + let mut metrics_runner = crowdb_chunk_kv_server::metrics::start_stream_perf_log( + &log_dir, + args.metrics_interval, + args.log_max_file_mb, + args.log_max_files, + ) + .expect("failed to open stream perf log"); + let storage = match connect_storage_with_transient_topology_retry(&config).await { Ok(storage) => Arc::new(storage), Err(error) => { @@ -149,13 +161,6 @@ async fn main() -> std::process::ExitCode { return std::process::ExitCode::FAILURE; } }; - let service = match ChunkKvService::new(config.instance_id, config.max_hosted_partitions) { - Ok(service) => Arc::new(service), - Err(error) => { - error!(%error, "failed to initialize chunk KV service"); - return std::process::ExitCode::FAILURE; - } - }; let control_store = Arc::new(Group0ControlStore::from_client(Arc::clone(storage.kv()))); let monitor_registry = DomainMonitorRegistry::new( control_store.clone(), @@ -180,6 +185,35 @@ async fn main() -> std::process::ExitCode { } } + let process_lease = match crowdb_chunk_kv_server::serving::process::ProcessLeaseSession::start( + Arc::clone(storage.kv()), + config.instance_id, + &config.monitor, + ) + .await + { + Ok(lease) => Arc::new(lease), + Err(error) => { + error!(%error, "failed to claim chunk KV process lease"); + return std::process::ExitCode::FAILURE; + } + }; + if let Err(error) = storage + .streams() + .registry() + .bind_owner_process(process_lease.authority()) + { + error!(%error, "failed to bind WAL stream process authority"); + return std::process::ExitCode::FAILURE; + } + let service = match ChunkKvService::new(process_lease.authority(), config.max_hosted_partitions) { + Ok(service) => Arc::new(service), + Err(error) => { + error!(%error, "failed to initialize chunk KV service"); + return std::process::ExitCode::FAILURE; + } + }; + let catalog = Arc::new(ChunkKvRangeCatalogPublisher::new(control_store.clone())); let recovery_storage = &storage; let recovery_service = &service; @@ -322,7 +356,7 @@ async fn main() -> std::process::ExitCode { continue; } }; - match refresh_service.install_catalog_and_reconcile(&head, &pages, &recovered) { + match refresh_service.reconcile_catalog(&head, &pages, &recovered).await { Ok(()) => info!( generation = head.generation, recovered = recovered.len(), @@ -396,7 +430,6 @@ async fn main() -> std::process::ExitCode { } }); - let service_registry = Arc::new(ServiceRegistryClient::from_shared(Arc::clone(storage.kv()))); let capacity_bytes = u64::try_from(config.max_hosted_partitions) .unwrap_or(u64::MAX) .saturating_mul(config.balance.target_partition_bytes); @@ -406,12 +439,8 @@ async fn main() -> std::process::ExitCode { .http_advertise_addr .as_ref() .map(|address| format!("http://{address}")); - if let Err(error) = service_registry - .register_chunk_kv( - config.instance_id, - &rpc_advertise_addr.to_string(), - &initial_observation, - ) + if let Err(error) = process_lease + .publish_observation(&rpc_advertise_addr.to_string(), &initial_observation) .await { error!(%error, "failed to register chunk KV instance"); @@ -433,7 +462,7 @@ async fn main() -> std::process::ExitCode { } }); let heartbeat_service = Arc::clone(&service); - let heartbeat_registry = Arc::clone(&service_registry); + let heartbeat_lease = Arc::clone(&process_lease); let heartbeat_store = Arc::clone(&control_store); let heartbeat_config = config.clone(); let heartbeat_endpoint = rpc_advertise_addr.to_string(); @@ -467,8 +496,8 @@ async fn main() -> std::process::ExitCode { .http_advertise_addr .as_ref() .map(|address| format!("http://{address}")); - if let Err(error) = heartbeat_registry - .heartbeat_chunk_kv(heartbeat_config.instance_id, &heartbeat_endpoint, &observation) + if let Err(error) = heartbeat_lease + .publish_observation(&heartbeat_endpoint, &observation) .await { warn!(%error, "chunk KV heartbeat failed"); @@ -505,10 +534,20 @@ async fn main() -> std::process::ExitCode { let shutdown_service = Arc::clone(&service); let shutdown_rpc = Arc::clone(&rpc_server); + let shutdown_lease = Arc::clone(&process_lease); let app = management_router(ManagementState::new(Arc::clone(&service))); let serving = axum::serve(listener, app) .with_graceful_shutdown(async move { - shutdown_signal().await; + tokio::select! { + () = shutdown_signal() => {}, + () = shutdown_lease.wait_for_fence() => { + error!("process lease lost; stopping for supervisor restart"); + } + partitions = shutdown_service.wait_for_recovery() => { + error!(?partitions, + "writer outcome requires durable replay; stopping for supervisor restart without reopening uncertain handles"); + } + } shutdown_service.begin_drain(); shutdown_rpc.stop(); info!("chunk KV service admission drained"); @@ -521,9 +560,21 @@ async fn main() -> std::process::ExitCode { materialization_task.abort(); transition_task.abort(); refresh_task.abort(); - if let Err(error) = service_registry.unregister("chunk-kv", config.instance_id).await { + let _ = tokio::join!( + heartbeat_task, + materialization_task, + transition_task, + refresh_task + ); + if let Err(error) = service.drain_writers().await { + warn!(%error, "WAL writer drain incomplete; restart must retain lease exclusion"); + } + if let Err(error) = process_lease.withdraw_observation().await { warn!(%error, "failed to unregister chunk KV instance"); } + if let Some(runner) = &mut metrics_runner { + runner.stop().await; + } if serving.is_ok() { std::process::ExitCode::SUCCESS } else { diff --git a/app/crowdb-chunk-kv-server/src/management.rs b/app/crowdb-chunk-kv-server/src/management.rs index 88c23c3f1..977285fae 100644 --- a/app/crowdb-chunk-kv-server/src/management.rs +++ b/app/crowdb-chunk-kv-server/src/management.rs @@ -28,11 +28,14 @@ impl ManagementState { #[derive(Debug, Serialize)] struct HealthResponse { instance_id: u64, + process_lease: crowdb_kv_client::StreamProcessLeaseStatus, lifecycle: &'static str, catalog_generation: u64, hosted_partitions: usize, serving_partitions: usize, ready: bool, + detail: String, + ranges: Vec, } pub fn management_router(state: ManagementState) -> Router { @@ -148,11 +151,18 @@ fn health_response(state: &ManagementState) -> HealthResponse { let health = state.service.health(state.service.monotonic_ms()); HealthResponse { instance_id: health.instance_id, + process_lease: state + .service + .authority() + .process() + .status(std::time::Instant::now()), lifecycle: lifecycle_name(health.lifecycle), catalog_generation: health.catalog_generation, hosted_partitions: health.partitions.len(), serving_partitions: health.serving_partitions, - ready: health.lifecycle == ServerLifecycle::Serving, + ready: health.ready, + detail: health.detail, + ranges: health.range_readiness, } } diff --git a/app/crowdb-chunk-kv-server/src/metrics.rs b/app/crowdb-chunk-kv-server/src/metrics.rs index b70637384..0d92e219e 100644 --- a/app/crowdb-chunk-kv-server/src/metrics.rs +++ b/app/crowdb-chunk-kv-server/src/metrics.rs @@ -150,3 +150,31 @@ impl ServerMetrics { fn increment(counter: &AtomicU64) { counter.fetch_add(1, Ordering::Relaxed); } + +/// Starts periodic stream phase output alongside the service logs. +/// Zero interval disables output. Call `stop().await` to flush the last window. +/// +/// # Errors +/// Returns an IO error when the metrics log cannot be opened. +/// +/// # Panics +/// Panics if the metrics registry is poisoned or a histogram name conflicts. +pub fn start_stream_perf_log( + log_dir: &str, + interval_secs: u64, + max_file_mb: usize, + max_files: usize, +) -> std::io::Result> { + if interval_secs == 0 { + return Ok(None); + } + let file = + crowdb_common::logging::open_metrics_log(log_dir, "crowdb-chunk-kv-server", max_file_mb, max_files) + .map_err(std::io::Error::other)?; + let mut runner = crowdb_common::metrics::MetricsRunner::new(file, interval_secs); + crowdb_chunk_stream::register_stream_perf_metrics( + &mut runner.registry().lock().expect("metrics registry poisoned"), + ); + runner.start(); + Ok(Some(runner)) +} diff --git a/app/crowdb-chunk-kv-server/src/server.rs b/app/crowdb-chunk-kv-server/src/server.rs index 9cb396f8d..b96acd103 100644 --- a/app/crowdb-chunk-kv-server/src/server.rs +++ b/app/crowdb-chunk-kv-server/src/server.rs @@ -11,6 +11,7 @@ use crowdb_chunk_kv::{ ChunkKvError, CompareCondition, JournalPosition, MutationOperation, MutationResult, Partition, PreparedSplitWriterArtifact, RequestId, SplitArtifact, ValueRevision, }; +use crowdb_kv_client::StreamProcessAuthority; use crowdb_protocol::chunk_kv::{ BatchMutationRequest, BatchMutationResponse, BatchMutationResult, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPartitionState, ChunkKvResponse, @@ -29,10 +30,16 @@ use crate::{ mod activation; mod assignments; +mod authority; +mod drain; +mod group; use assignments::{local_partition_for_entry, partition_matches_entry}; mod load_sampling; mod observation; mod prepared_transfer; +mod readiness; +mod recovery; +pub use readiness::RangeReadiness; mod reconcile; mod routing; mod scan; @@ -71,6 +78,9 @@ pub struct ServerHealth { pub catalog_generation: u64, pub partitions: Vec, pub serving_partitions: usize, + pub ready: bool, + pub range_readiness: Vec, + pub detail: String, } #[derive(Debug, Error)] @@ -113,7 +123,6 @@ impl CatalogSnapshot { /// changes replace a snapshot; admitted partition operations retain their own /// handles until completion. pub struct ChunkKvService { - started: Instant, instance_id: u64, authority: Arc, catalog: ArcSwap, @@ -157,9 +166,9 @@ impl ChunkKvService { /// # Errors /// /// Returns an invalid-request error for zero identity or capacity. - pub fn new(instance_id: u64, max_partitions: usize) -> Result { + pub fn new(process: Arc, max_partitions: usize) -> Result { Self::new_with_limits( - instance_id, + process, max_partitions, DEFAULT_SCAN_RESPONSE_BYTES, DEFAULT_INITIALIZATION_WAITERS, @@ -173,13 +182,19 @@ impl ChunkKvService { /// /// Returns an invalid-request error when any identity or bound is zero. pub fn new_with_limits( - instance_id: u64, + process: Arc, max_partitions: usize, max_scan_response_bytes: usize, max_initialization_waiters: usize, initialization_wait_ms: u64, ) -> Result { - if instance_id == 0 + let identity = process + .capture(Instant::now()) + .map_err(|error| ChunkKvError::InvalidRequest(error.to_string()))? + .identity(); + let instance_id = identity.instance_id; + if identity.purpose != crowdb_protocol::chunk_stream::StreamPurpose::Wal + || instance_id == 0 || max_partitions == 0 || max_scan_response_bytes == 0 || max_initialization_waiters == 0 @@ -190,9 +205,8 @@ impl ChunkKvService { )); } Ok(Self { - started: Instant::now(), instance_id, - authority: Arc::new(ServingAuthority::new(instance_id)), + authority: Arc::new(ServingAuthority::new(process)), catalog: ArcSwap::from_pointee(CatalogSnapshot::default()), partitions: ArcSwap::from_pointee(HashMap::new()), prepared_transfer_targets: ArcSwap::from_pointee(HashMap::new()), @@ -281,11 +295,11 @@ impl ChunkKvService { /// Milliseconds elapsed on the process-local monotonic clock. #[must_use] pub fn monotonic_ms(&self) -> u64 { - u64::try_from(self.started.elapsed().as_millis()).unwrap_or(u64::MAX) + self.authority.monotonic_ms() } - /// Stops new admission and clears serving authority. Operations already - /// admitted to an R142 sequencer retain their partition handle and finish. + /// Stops new admission and revokes successful serving replies. Admitted + /// operations retain their partition handle and can finish durable work. pub fn begin_drain(&self) { self.admitting.store(false, Ordering::Release); self.authority.clear(); @@ -316,11 +330,14 @@ impl ChunkKvService { partitions.sort_unstable_by_key(|partition| partition.partition_id); let lifecycle = if !self.admitting.load(Ordering::Acquire) { ServerLifecycle::Draining - } else if catalog_generation != 0 - && (partitions.is_empty() - || self - .authority - .has_live_grant(catalog_generation, now_monotonic_ms)) + } else if self.authority.process().capture(Instant::now()).is_ok() + && catalog_generation != 0 + && (!catalog.entries.iter().any(|entry| { + entry.owner.instance_id == self.instance_id + && entry.state != ChunkKvRangeCatalogPartitionState::Retired + }) || self + .authority + .has_live_grant(catalog_generation, now_monotonic_ms)) { ServerLifecycle::Serving } else { @@ -339,12 +356,27 @@ impl ChunkKvService { }) }) .count(); + let range_readiness = self.assignment_readiness(&catalog, now_monotonic_ms); + let recovery_required = self.recovery_required(); + let ready = lifecycle == ServerLifecycle::Serving + && range_readiness.iter().all(|range| range.ready) + && recovery_required.is_empty(); + let detail = if !recovery_required.is_empty() { + format!("Writer outcome requires durable replay before serving: {recovery_required:?}") + } else if range_readiness.is_empty() && ready { + format!("Idle stateless instance: catalog generation {catalog_generation} loaded, no tree-range ownership assigned") + } else { + format!("Instance lifecycle {lifecycle:?}; catalog generation {catalog_generation}; {} owned tree ranges, {} ready", range_readiness.len(), range_readiness.iter().filter(|range| range.ready).count()) + }; ServerHealth { instance_id: self.instance_id, lifecycle, catalog_generation, partitions, serving_partitions, + ready, + range_readiness, + detail, } } @@ -378,6 +410,13 @@ impl ChunkKvService { .collect(); hosted.sort_unstable_by_key(|partition| partition.partition_id); ChunkKvExtra { + process_epoch: self + .authority + .process() + .status(Instant::now()) + .lease + .identity + .process_epoch, capacity_bytes, durable_bytes, request_rate, @@ -652,13 +691,12 @@ impl ChunkKvService { retained_applied_seq = artifact.retained_parent.applied_seq, retained_tree_manifest = artifact.retained_parent.tree_manifest, retained_root_manifest_generation = artifact.retained_parent.root_manifest_generation, - retained_parent_stream_manifest_generation = - artifact.retained_parent.parent_stream_manifest_generation, + retained_parent_stream_generation = artifact.retained_parent.parent_stream_generation, child_base_seq = artifact.child.base_applied_seq, child_applied_seq = artifact.child.applied_seq, child_tree_manifest = artifact.child.tree_manifest, child_root_manifest_generation = artifact.child.root_manifest_generation, - child_parent_stream_manifest_generation = artifact.child.parent_stream_manifest_generation, + child_parent_stream_generation = artifact.child.parent_stream_generation, "local split writers installed" ); Ok(()) @@ -776,8 +814,8 @@ impl ChunkKvService { /// Handles a point request directly; it never proxies to another owner. /// /// The wall-clock deadline is checked before sequencer admission. Once a - /// mutation is admitted, this method awaits its single durable result even - /// if the caller drops the surrounding transport future. + /// mutation is admitted, this method awaits its durable result and checks + /// serving authority again before returning success. pub async fn handle_point( &self, request: PointRequest, @@ -872,12 +910,13 @@ impl ChunkKvService { } }; - self.execute_point_operation( + self.execute_authorized_point( &partition, entry, catalog.generation, request, transfer_append_artifact, + now_monotonic_ms, ) .await } @@ -995,125 +1034,6 @@ impl ChunkKvService { } } - /// Executes a fully range-validated partition-local multi-get. - pub async fn handle_multi_get( - &self, - request: MultiGetRequest, - now_wall_ms: u64, - now_monotonic_ms: u64, - ) -> MultiGetResponse { - let catalog = self.catalog.load_full(); - let Some(range) = self.logical_range_for_request(request.routing.partition_id, &catalog) else { - return MultiGetResponse { - map_revision: catalog.generation, - result: Err(not_my_range_failure(catalog.generation, None)), - }; - }; - if request.validate_for_range(&range).is_err() { - return MultiGetResponse { - map_revision: catalog.generation, - result: Err(rpc_failure( - ChunkKvRpcErrorCode::InvalidRequest, - "multi-get group contains an out-of-range key", - )), - }; - } - let mut values = Vec::with_capacity(request.keys.len()); - for key in request.keys { - let response = self - .handle_point( - PointRequest { - routing: request.routing.clone(), - operation: PointOperation::Get { key }, - }, - now_wall_ms, - now_monotonic_ms, - ) - .await; - match response.result { - Ok(OperationResult::Value(value)) => values.push(value), - Ok(_) => { - return MultiGetResponse { - map_revision: response.map_revision, - result: Err(rpc_failure( - ChunkKvRpcErrorCode::Internal, - "multi-get produced a non-value result", - )), - }; - } - Err(error) => { - return MultiGetResponse { - map_revision: response.map_revision, - result: Err(error), - }; - } - } - } - MultiGetResponse { - map_revision: catalog.generation, - result: Ok(values), - } - } - - /// Executes a fully range-validated mutation group in input order. - pub async fn handle_batch_mutation( - &self, - request: BatchMutationRequest, - now_wall_ms: u64, - now_monotonic_ms: u64, - ) -> BatchMutationResponse { - let catalog = self.catalog.load_full(); - let Some(range) = self.logical_range_for_request(request.routing.partition_id, &catalog) else { - return BatchMutationResponse { - map_revision: catalog.generation, - result: Err(not_my_range_failure(catalog.generation, None)), - }; - }; - if request.validate_for_range(&range).is_err() { - return BatchMutationResponse { - map_revision: catalog.generation, - result: Err(rpc_failure( - ChunkKvRpcErrorCode::InvalidRequest, - "batch group contains an invalid or out-of-range mutation", - )), - }; - } - let first_id = request.operations[0].request_id; - let routing = RequestRouting { - request_id: first_id, - map_revision: request.routing.map_revision, - partition_id: request.routing.partition_id, - owner_epoch: request.routing.owner_epoch, - min_journal_position: None, - deadline_ms: request.routing.deadline_ms, - }; - let mut results = Vec::with_capacity(request.operations.len()); - for item in request.operations { - let response = self - .handle_point( - PointRequest { - routing: RequestRouting { - request_id: item.request_id, - ..routing.clone() - }, - operation: item.operation, - }, - now_wall_ms, - now_monotonic_ms, - ) - .await; - results.push(BatchMutationResult { - request_id: item.request_id, - journal_position: response.journal_position, - result: response.result, - }); - } - BatchMutationResponse { - map_revision: catalog.generation, - result: Ok(results), - } - } - /// Handles one ordered seek directly against a partition view. pub async fn handle_seek( &self, @@ -1277,9 +1197,19 @@ impl ChunkKvService { let Some(entry) = catalog.entry_for_partition(request.routing.partition_id) else { return not_my_range(catalog.generation, None); }; - let logical_range = self - .logical_range_for_request(request.routing.partition_id, &catalog) - .unwrap_or_else(|| entry.range.clone()); + // Select the routed assignment before clipping. A historical parent + // scan can start wholly inside the child and still needs both writers. + let partition = self.partition_for_request(&request.routing, &entry.range.start, entry); + let logical_range = partition.as_ref().map_or_else( + || entry.range.clone(), + |partition| { + let range = partition.snapshot().range; + KeyRange { + start: range.start.unwrap_or_default(), + end: range.end, + } + }, + ); let clipped = match validate_and_clip_scan(&request, &logical_range) { Ok(clipped) => clipped, Err(ScanValidationError::RefreshRequired) => { @@ -1308,7 +1238,7 @@ impl ChunkKvService { ) { return authority_failure(catalog.generation, &error, Some(entry)); } - let Some(partition) = self.partition_for_request(&request.routing, &clipped.start, entry) else { + let Some(partition) = partition else { return failure( catalog.generation, ChunkKvRpcErrorCode::Recovering, @@ -1538,7 +1468,7 @@ fn final_transfer_artifact( && artifact.parent_id.low == overlay.source_partition_id.low && artifact.parent_epoch == overlay.source_epoch && artifact.parent_stream_name == overlay.source_stream_name - && artifact.parent_stream_manifest_generation == overlay.source_stream_manifest_generation + && artifact.parent_stream_generation == overlay.source_stream_generation && artifact.parent_replay_offset == overlay.replay_offset && overlay.target_stream_start_seq == overlay.cutover_seq.checked_add(1)?; if !exact_base { diff --git a/app/crowdb-chunk-kv-server/src/server/activation.rs b/app/crowdb-chunk-kv-server/src/server/activation.rs index 386c543bd..45a59cfaa 100644 --- a/app/crowdb-chunk-kv-server/src/server/activation.rs +++ b/app/crowdb-chunk-kv-server/src/server/activation.rs @@ -73,7 +73,7 @@ impl ChunkKvService { if transition.phase != SplitPhase::CatalogCommitted || entry.transition_id != Some(transition.transition_id) || entry.range != range - || &entry.owner != owner + || entry.owner.instance_id != owner.instance_id || owner.instance_id != self.instance_id || epoch != owner_epoch || entry.owner_epoch != epoch @@ -166,7 +166,7 @@ impl ChunkKvService { && entry.transition_id == Some(transition.transition_id) && transition.partition_id == partition_id && transition.range == entry.range - && transition.target == entry.owner + && transition.target.instance_id == entry.owner.instance_id && transition.target.instance_id == self.instance_id && transition.target_epoch == owner_epoch && transition.target_epoch == entry.owner_epoch diff --git a/app/crowdb-chunk-kv-server/src/server/authority.rs b/app/crowdb-chunk-kv-server/src/server/authority.rs new file mode 100644 index 000000000..3ef651c42 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/authority.rs @@ -0,0 +1,68 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Captured serving authority around asynchronous point execution. + +use super::{ + authority_failure, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, ChunkKvResponse, + ChunkKvService, Instant, Partition, PointRequest, PreparedSplitWriterArtifact, +}; + +impl ChunkKvService { + pub(super) async fn execute_authorized_point( + &self, + partition: &Partition, + entry: &ChunkKvRangeCatalogEntry, + generation: u64, + request: PointRequest, + transfer: Option, + received_monotonic_ms: u64, + ) -> ChunkKvResponse { + let started = Instant::now(); + let Ok(process) = self.authority.process().capture(Instant::now()) else { + return authority_failure(generation, &super::AuthorityError::LeaseExpired, Some(entry)); + }; + // A catching-up target uses the already validated transfer artifact. + // Serving assignments use their independently renewable grant context. + let context = if entry.state == ChunkKvRangeCatalogPartitionState::TargetCatchingUp { + None + } else { + match self.authority.capture( + generation, + entry.partition_id, + entry.owner_epoch, + self.request_monotonic_ms(received_monotonic_ms, started), + ) { + Ok(context) => Some(context), + Err(error) => return authority_failure(generation, &error, Some(entry)), + } + }; + let response = self + .execute_point_operation(partition, entry, generation, request, transfer) + .await; + if response.result.is_ok() { + if self + .authority + .process() + .validate_context(&process, Instant::now()) + .is_err() + { + return authority_failure(generation, &super::AuthorityError::LeaseExpired, Some(entry)); + } + if let Some(context) = context { + if let Err(error) = self.authority.validate_context( + &context, + self.request_monotonic_ms(received_monotonic_ms, started), + ) { + return authority_failure(generation, &error, Some(entry)); + } + } + } + response + } + + pub(super) fn request_monotonic_ms(&self, received: u64, started: Instant) -> u64 { + let elapsed = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX); + self.monotonic_ms().max(received.saturating_add(elapsed)) + } +} diff --git a/app/crowdb-chunk-kv-server/src/server/drain.rs b/app/crowdb-chunk-kv-server/src/server/drain.rs new file mode 100644 index 000000000..c489a6d3f --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/drain.rs @@ -0,0 +1,25 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ChunkKvError, ChunkKvService}; + +impl ChunkKvService { + /// Drain admitted mutations and release each hosted WAL writer for a new boot. + /// Background assignment changes must be stopped before calling this method. + /// # Errors + /// Leaves uncertain or fenced writers subject to their normal lease exclusion. + pub async fn drain_writers(&self) -> Result<(), ChunkKvError> { + self.begin_drain(); + let partitions = self.partitions.load_full(); + let mut first_error = None; + for partition in partitions.values() { + if let Err(error) = partition + .suspend_for_shutdown(partition.snapshot().ownership_epoch) + .await + { + first_error.get_or_insert(error); + } + } + first_error.map_or(Ok(()), Err) + } +} diff --git a/app/crowdb-chunk-kv-server/src/server/group.rs b/app/crowdb-chunk-kv-server/src/server/group.rs new file mode 100644 index 000000000..39527a072 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/group.rs @@ -0,0 +1,159 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Partition-local grouped point execution. + +use super::{ + not_my_range_failure, rpc_failure, BatchMutationRequest, BatchMutationResponse, BatchMutationResult, + ChunkKvRpcErrorCode, ChunkKvService, MultiGetRequest, MultiGetResponse, OperationResult, PointOperation, + PointRequest, RequestRouting, +}; + +mod authority; + +impl ChunkKvService { + /// Executes a fully range-validated partition-local multi-get. + pub async fn handle_multi_get( + &self, + request: MultiGetRequest, + now_wall_ms: u64, + now_monotonic_ms: u64, + ) -> MultiGetResponse { + let catalog = self.catalog.load_full(); + let Some(range) = self.logical_range_for_request(request.routing.partition_id, &catalog) else { + return MultiGetResponse { + map_revision: catalog.generation, + result: Err(not_my_range_failure(catalog.generation, None)), + }; + }; + if request.validate_for_range(&range).is_err() { + return MultiGetResponse { + map_revision: catalog.generation, + result: Err(rpc_failure( + ChunkKvRpcErrorCode::InvalidRequest, + "multi-get group contains an out-of-range key", + )), + }; + } + let group = match self.capture_point_group( + &catalog, + request.keys.iter().map(Vec::as_slice), + now_wall_ms, + now_monotonic_ms, + ) { + Ok(group) => group, + Err(error) => { + return MultiGetResponse { + map_revision: catalog.generation, + result: Err(error), + } + } + }; + let mut values = Vec::with_capacity(request.keys.len()); + for key in request.keys { + let response = self + .execute_group_point( + &group, + PointRequest { + routing: request.routing.clone(), + operation: PointOperation::Get { key }, + }, + ) + .await; + match response.result { + Ok(OperationResult::Value(value)) => values.push(value), + Ok(_) => { + return MultiGetResponse { + map_revision: response.map_revision, + result: Err(rpc_failure( + ChunkKvRpcErrorCode::Internal, + "multi-get produced a non-value result", + )), + }; + } + Err(error) => { + return MultiGetResponse { + map_revision: response.map_revision, + result: Err(error), + }; + } + } + } + MultiGetResponse { + map_revision: catalog.generation, + result: Ok(values), + } + } + + /// Executes a fully range-validated mutation group in input order. + pub async fn handle_batch_mutation( + &self, + request: BatchMutationRequest, + now_wall_ms: u64, + now_monotonic_ms: u64, + ) -> BatchMutationResponse { + let catalog = self.catalog.load_full(); + let Some(range) = self.logical_range_for_request(request.routing.partition_id, &catalog) else { + return BatchMutationResponse { + map_revision: catalog.generation, + result: Err(not_my_range_failure(catalog.generation, None)), + }; + }; + if request.validate_for_range(&range).is_err() { + return BatchMutationResponse { + map_revision: catalog.generation, + result: Err(rpc_failure( + ChunkKvRpcErrorCode::InvalidRequest, + "batch group contains an invalid or out-of-range mutation", + )), + }; + } + let group = match self.capture_point_group( + &catalog, + request.operations.iter().map(|item| item.operation.key()), + now_wall_ms, + now_monotonic_ms, + ) { + Ok(group) => group, + Err(error) => { + return BatchMutationResponse { + map_revision: catalog.generation, + result: Err(error), + } + } + }; + let first_id = request.operations[0].request_id; + let routing = RequestRouting { + request_id: first_id, + map_revision: request.routing.map_revision, + partition_id: request.routing.partition_id, + owner_epoch: request.routing.owner_epoch, + min_journal_position: None, + deadline_ms: request.routing.deadline_ms, + }; + let mut results = Vec::with_capacity(request.operations.len()); + for item in request.operations { + let response = self + .execute_group_point( + &group, + PointRequest { + routing: RequestRouting { + request_id: item.request_id, + ..routing.clone() + }, + operation: item.operation, + }, + ) + .await; + results.push(BatchMutationResult { + request_id: item.request_id, + journal_position: response.journal_position, + result: response.result, + }); + } + BatchMutationResponse { + map_revision: catalog.generation, + result: Ok(results), + } + } +} diff --git a/app/crowdb-chunk-kv-server/src/server/group/authority.rs b/app/crowdb-chunk-kv-server/src/server/group/authority.rs new file mode 100644 index 000000000..907b4daff --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/group/authority.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One immutable authority context for a complete grouped RPC. + +use std::{collections::HashSet, time::Instant}; + +use super::super::{not_my_range_failure, rpc_failure, AuthorityError, CatalogSnapshot}; +use crate::{ChunkKvService, ServingContext}; +use crowdb_protocol::chunk_kv::{ + ChunkKvRangeCatalogPartitionState, ChunkKvResponse, ChunkKvRpcErrorCode, PointRequest, RpcFailure, +}; + +pub(super) struct PointGroup { + contexts: Vec, + generation: u64, + wall_ms: u64, + monotonic_ms: u64, + started: Instant, +} + +impl ChunkKvService { + pub(super) fn capture_point_group<'a>( + &self, + catalog: &CatalogSnapshot, + keys: impl Iterator, + wall_ms: u64, + monotonic_ms: u64, + ) -> Result { + let started = Instant::now(); + let mut seen = HashSet::new(); + let mut contexts = Vec::new(); + for key in keys { + let entry = catalog + .entry_for_key(key) + .ok_or_else(|| not_my_range_failure(catalog.generation, None))?; + if !seen.insert(entry.partition_id) + || entry.state == ChunkKvRangeCatalogPartitionState::TargetCatchingUp + { + continue; + } + contexts.push( + self.authority + .capture( + catalog.generation, + entry.partition_id, + entry.owner_epoch, + self.request_monotonic_ms(monotonic_ms, started), + ) + .map_err(|error| match error { + AuthorityError::LeaseExpired => { + rpc_failure(ChunkKvRpcErrorCode::LeaseExpired, &error.to_string()) + } + _ => not_my_range_failure(catalog.generation, Some(entry)), + })?, + ); + } + Ok(PointGroup { + contexts, + generation: catalog.generation, + wall_ms, + monotonic_ms, + started, + }) + } + + pub(super) async fn execute_group_point( + &self, + group: &PointGroup, + request: PointRequest, + ) -> ChunkKvResponse { + if let Err(error) = self.validate_point_group(group) { + return group_failure(group.generation, error); + } + let elapsed = u64::try_from(group.started.elapsed().as_millis()).unwrap_or(u64::MAX); + let response = self + .handle_point( + request, + group.wall_ms.saturating_add(elapsed), + self.request_monotonic_ms(group.monotonic_ms, group.started), + ) + .await; + if response.result.is_ok() { + if let Err(error) = self.validate_point_group(group) { + return group_failure(group.generation, error); + } + } + response + } + + fn validate_point_group(&self, group: &PointGroup) -> Result<(), RpcFailure> { + self.authority + .validate_contexts( + &group.contexts, + self.request_monotonic_ms(group.monotonic_ms, group.started), + ) + .map_err(|error| rpc_failure(ChunkKvRpcErrorCode::LeaseExpired, &error.to_string())) + } +} + +fn group_failure(generation: u64, error: RpcFailure) -> ChunkKvResponse { + ChunkKvResponse { + map_revision: generation, + journal_position: None, + result: Err(error), + } +} diff --git a/app/crowdb-chunk-kv-server/src/server/observation/storage.rs b/app/crowdb-chunk-kv-server/src/server/observation/storage.rs index e21f9df0c..cf6537f44 100644 --- a/app/crowdb-chunk-kv-server/src/server/observation/storage.rs +++ b/app/crowdb-chunk-kv-server/src/server/observation/storage.rs @@ -43,30 +43,17 @@ pub(super) fn tree(observation: Result) -> Value { } pub(super) fn journal(value: &StreamMetadataObservation) -> Value { - let active = value.active.as_ref().map(|active| { - json!({ - "chunk_id":format!("{:016x}{:016x}",active.chunk_id.high,active.chunk_id.low), - "physical_start":active.physical_start.to_string(), - "logical_start":active.logical_start.to_string(), - "acknowledged_cursor":active.acknowledged_cursor.to_string(), - "capacity":active.capacity.to_string(), - }) + let chunk = &value.chunk; + let chunk = json!({ + "chunk_id":format!("{:016x}{:016x}",chunk.chunk_id.high,chunk.chunk_id.low), + "state":format!("{:?}",chunk.state), + "logical_start":chunk.logical_start.to_string(),"logical_end":chunk.logical_end.to_string(), + "mapping_count":chunk.mapping_count.to_string(),"mapping_bytes":chunk.mapping_bytes.to_string(), }); - let pages: Vec<_> = value - .extent_pages - .iter() - .map(|page| { - json!({ - "page_index":page.page_index.to_string(), - "first_logical":page.first_logical.to_string(), - "end_logical":page.end_logical.to_string(), - }) - }) - .collect(); json!({ "generation":value.generation.to_string(), "writer_epoch":value.writer_epoch.to_string(), "metadata_group_id":value.metadata_group_id.to_string(), "trim_offset":value.trim_offset.to_string(), - "sealed_tail":value.sealed_tail.to_string(), "closed":value.closed, - "active":active, "extent_pages":pages, "offset":value.offset, "next_offset":value.next_offset, + "published_tail":value.published_tail.to_string(), "closed":value.closed, + "chunk_record":chunk, "offset":value.offset, "next_offset":value.next_offset, }) } diff --git a/app/crowdb-chunk-kv-server/src/server/readiness.rs b/app/crowdb-chunk-kv-server/src/server/readiness.rs new file mode 100644 index 000000000..76a3e6fb7 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/readiness.rs @@ -0,0 +1,98 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! A live grant cannot substitute for activated writers of current assignments. + +use super::{CatalogSnapshot, ChunkKvService}; +use crowdb_chunk_kv::PartitionLifecycle; +use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogPartitionState, RequestRouting}; + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub struct RangeReadiness { + pub partition_id: crowdb_protocol::chunk_kv::Id128, + pub owner_epoch: u64, + pub ready: bool, + pub detail: String, +} + +impl ChunkKvService { + pub(super) fn assignment_readiness(&self, catalog: &CatalogSnapshot, now: u64) -> Vec { + catalog + .entries + .iter() + .filter(|entry| { + entry.owner.instance_id == self.instance_id + && entry.state != ChunkKvRangeCatalogPartitionState::Retired + }) + .map(|entry| { + let result = self.range_readiness(entry, catalog.generation, now); + RangeReadiness { + partition_id: entry.partition_id, + owner_epoch: entry.owner_epoch, + ready: result.is_ok(), + detail: match result { + Ok(detail) | Err(detail) => detail, + }, + } + }) + .collect() + } + + fn range_readiness( + &self, + entry: &crowdb_protocol::chunk_kv::ChunkKvRangeCatalogEntry, + catalog_generation: u64, + now: u64, + ) -> Result { + if !matches!( + entry.state, + ChunkKvRangeCatalogPartitionState::Serving + | ChunkKvRangeCatalogPartitionState::SplitPreparing + | ChunkKvRangeCatalogPartitionState::SplitFinalizing + ) { + return Err(format!( + "Owned range catalog state {:?} is not serviceable", + entry.state + )); + } + self.authority + .authorize(catalog_generation, entry.partition_id, entry.owner_epoch, now) + .map_err(|error| { + format!( + "Owned range serving authority is unavailable for catalog generation {catalog_generation}: {error:?}; writer recovery required: {:?}", + self.recovery_required() + ) + })?; + let routing = RequestRouting { + request_id: crowdb_protocol::chunk_kv::ClientRequestId::default(), + map_revision: catalog_generation, + partition_id: entry.partition_id, + owner_epoch: entry.owner_epoch, + min_journal_position: None, + deadline_ms: None, + }; + let partition = self + .partition_for_request(&routing, &entry.range.start, entry) + .ok_or_else(|| "Owned range has no recovered dispatch writer".to_owned())?; + let writer = partition.split_ingress().map_or(partition.clone(), |ingress| { + ingress.writer_for_key(&entry.range.start) + }); + if matches!( + writer.lifecycle(), + PartitionLifecycle::Serving + | PartitionLifecycle::SplitPreparing + | PartitionLifecycle::SplitFinalizing + ) { + Ok(format!( + "Owned range has current authority and a {:?} writer at catalog generation {}", + writer.lifecycle(), + catalog_generation + )) + } else { + Err(format!( + "Owned range writer is {:?}; waiting for recovery and activation", + writer.lifecycle() + )) + } + } +} diff --git a/app/crowdb-chunk-kv-server/src/server/reconcile.rs b/app/crowdb-chunk-kv-server/src/server/reconcile.rs index eb83c75a9..96dab8649 100644 --- a/app/crowdb-chunk-kv-server/src/server/reconcile.rs +++ b/app/crowdb-chunk-kv-server/src/server/reconcile.rs @@ -83,6 +83,41 @@ impl ChunkKvService { Ok(()) } + /// Install a catalog and drain WALs whose ownership moved to another instance. + /// # Errors + /// Returns validation or reconciliation failures before publication. A WAL + /// drain failure after publication fences this process for supervisor recovery. + pub async fn reconcile_catalog( + &self, + head: &ChunkKvRangeCatalogHead, + pages: &[ChunkKvRangeCatalogPage], + recovered: &[Partition], + ) -> Result<(), ChunkKvRangeCatalogReconcileError> { + let previous = self.partitions.load_full(); + self.install_catalog_and_reconcile(head, pages, recovered)?; + let mut first_error = None; + for (id, partition) in previous.iter() { + let epoch = partition.snapshot().ownership_epoch; + let moved = pages.iter().flat_map(|page| &page.entries).any(|entry| { + entry.partition_id == *id + && entry.owner.instance_id != self.instance_id + && entry.owner_epoch > epoch + }); + if moved { + if let Err(error) = partition.suspend_for_shutdown(epoch).await { + first_error.get_or_insert(error); + } + } + } + if let Some(error) = first_error { + // A live process cannot retain an uncertain, unassigned WAL forever. + // Stop renewal and let the supervisor restart under lease exclusion. + self.authority.process().fence(); + return Err(error.into()); + } + Ok(()) + } + fn reconciled_partition_snapshot( &self, pages: &[ChunkKvRangeCatalogPage], diff --git a/app/crowdb-chunk-kv-server/src/server/recovery.rs b/app/crowdb-chunk-kv-server/src/server/recovery.rs new file mode 100644 index 000000000..4f7946419 --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/server/recovery.rs @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Terminal writer uncertainty requires process teardown and durable replay. + +use super::{ChunkKvService, HostedPartitionHealth, Id128}; +use crowdb_chunk_kv::PartitionLifecycle; + +impl ChunkKvService { + /// Reports terminal writer failures without reopening or replacing handles. + #[must_use] + pub fn recovery_required(&self) -> Vec { + let mut writers: Vec<_> = self.observed_partition_snapshot().values().cloned().collect(); + writers.extend( + self.local_split_sessions + .load() + .values() + .map(|session| session.dispatcher.clone()), + ); + let ingress_writers: Vec<_> = writers + .iter() + .filter_map(crowdb_chunk_kv::Partition::split_ingress) + .flat_map(|ingress| [ingress.retained_parent(), ingress.child()]) + .collect(); + writers.extend(ingress_writers); + let mut failed: Vec<_> = writers + .iter() + .filter_map(|partition| { + let snapshot = partition.snapshot(); + (snapshot.lifecycle == PartitionLifecycle::Recovering).then_some(HostedPartitionHealth { + partition_id: Id128 { + high: snapshot.partition_id.high, + low: snapshot.partition_id.low, + }, + owner_epoch: snapshot.ownership_epoch, + lifecycle: snapshot.lifecycle, + durable_seq: snapshot.journal_durable_seq, + applied_seq: snapshot.applied_seq, + }) + }) + .collect(); + failed.sort_unstable_by_key(|partition| (partition.partition_id, partition.owner_epoch)); + failed.dedup_by_key(|partition| (partition.partition_id, partition.owner_epoch)); + failed + } + + /// Waits for uncertainty that cannot be repaired by catalog reconciliation. + pub async fn wait_for_recovery(&self) -> Vec { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + let failed = self.recovery_required(); + if !failed.is_empty() { + return failed; + } + } + } +} diff --git a/app/crowdb-chunk-kv-server/src/serving.rs b/app/crowdb-chunk-kv-server/src/serving.rs index 914f1fd53..157eff7e9 100644 --- a/app/crowdb-chunk-kv-server/src/serving.rs +++ b/app/crowdb-chunk-kv-server/src/serving.rs @@ -7,6 +7,7 @@ pub mod balance; pub mod lease; pub mod monitor; +pub mod process; pub mod split; pub mod transfer; mod transition_runtime; @@ -16,7 +17,9 @@ pub use balance::{ choose_split, choose_transfer, desired_partition_count, BalanceConfig, OwnerLoad, PartitionLoad, SplitProposal, TransferProposal, }; -pub use lease::{classify_instance, replacement_may_activate, AuthorityError, ServingAuthority}; +pub use lease::{ + classify_instance, replacement_may_activate, AuthorityError, ServingAuthority, ServingContext, +}; pub use monitor::{ DomainMonitorDriver, DomainMonitorRegistry, MonitorDescriptorStore, MonitorError, MonitorTick, PreparedMonitor, diff --git a/app/crowdb-chunk-kv-server/src/serving/lease.rs b/app/crowdb-chunk-kv-server/src/serving/lease.rs index b1084b276..a626aba49 100644 --- a/app/crowdb-chunk-kv-server/src/serving/lease.rs +++ b/app/crowdb-chunk-kv-server/src/serving/lease.rs @@ -2,17 +2,27 @@ // Licensed under the Apache License, Version 2.0. use std::sync::Arc; +use std::time::Instant; -use arc_swap::ArcSwapOption; -use crowdb_protocol::chunk_kv::{ - DomainMonitorDescriptor, Id128, InstanceHealth, ServingAssignment, ServingGrant, -}; +use arc_swap::ArcSwap; +use crowdb_kv_client::StreamProcessAuthority; +use crowdb_protocol::chunk_kv::{DomainMonitorDescriptor, Id128, InstanceHealth, ServingGrant}; use thiserror::Error; +mod context; +pub use context::ServingContext; + #[derive(Clone, Debug)] struct AuthoritySnapshot { - grant: ServingGrant, + grant: Arc, local_deadline_ms: u64, + continuity: Vec, +} + +#[derive(Default)] +struct AuthorityState { + highest_sequence: u64, + current: Option, } #[derive(Clone, Debug, Error, PartialEq, Eq)] @@ -23,6 +33,8 @@ pub enum AuthorityError { InvalidGrant, #[error("serving grant belongs to another instance")] WrongInstance, + #[error("serving grant belongs to another process boot")] + WrongProcess, #[error("serving grant has expired or reached its safety deadline")] LeaseExpired, #[error("serving grant is older than the installed authority")] @@ -36,27 +48,31 @@ pub enum AuthorityError { /// Lock-free request-path authority derived from monitor-issued serving grants. pub struct ServingAuthority { instance_id: u64, - snapshot: ArcSwapOption, + process: Arc, + clock_origin: Instant, + state: ArcSwap, } impl ServingAuthority { #[must_use] - pub fn new(instance_id: u64) -> Self { + pub fn new(process: Arc) -> Self { Self { - instance_id, - snapshot: ArcSwapOption::empty(), + instance_id: process.status(Instant::now()).lease.identity.instance_id, + process, + clock_origin: Instant::now(), + state: ArcSwap::from_pointee(AuthorityState::default()), } } - /// Installs a validated grant and derives its conservative local deadline. - /// - /// `received_wall_ms` and `received_monotonic_ms` must be sampled together. - /// Request authorization subsequently relies only on the monotonic clock. - /// + /// Local monotonic clock shared by receipt, installation and request checks. + #[must_use] + pub fn monotonic_ms(&self) -> u64 { + u64::try_from(self.clock_origin.elapsed().as_millis()).unwrap_or(u64::MAX) + } + + /// Install a grant using wall/monotonic clocks sampled together at receipt. /// # Errors - /// - /// Returns a typed error for unsafe policy, invalid identity, expiry, or a - /// grant that loses a concurrent monotonic installation race. + /// Rejects invalid policy, identity, expiry and superseded grant sequences. pub fn install( &self, grant: ServingGrant, @@ -64,60 +80,65 @@ impl ServingAuthority { received_wall_ms: u64, received_monotonic_ms: u64, ) -> Result<(), AuthorityError> { + let started = Instant::now(); policy.validate().map_err(|_| AuthorityError::InvalidPolicy)?; grant.validate().map_err(|_| AuthorityError::InvalidGrant)?; if grant.instance_id != self.instance_id { return Err(AuthorityError::WrongInstance); } - let safety_margin = policy - .max_clock_skew_ms - .checked_add(policy.self_fence_margin_ms) - .ok_or(AuthorityError::InvalidPolicy)?; - let safe_wall_deadline = grant - .expires_at_ms - .checked_sub(safety_margin) - .ok_or(AuthorityError::LeaseExpired)?; - let safe_remaining = safe_wall_deadline - .checked_sub(received_wall_ms) - .ok_or(AuthorityError::LeaseExpired)?; - if safe_remaining == 0 { - return Err(AuthorityError::LeaseExpired); + let process = self + .process + .capture(Instant::now()) + .map_err(|_| AuthorityError::LeaseExpired)?; + if grant.process_epoch != process.identity().process_epoch { + return Err(AuthorityError::WrongProcess); } - let local_deadline_ms = received_monotonic_ms - .checked_add(safe_remaining) - .ok_or(AuthorityError::InvalidGrant)?; - let candidate = Arc::new(AuthoritySnapshot { - grant, - local_deadline_ms, - }); - - self.snapshot.rcu(|current| { - if current - .as_ref() - .is_some_and(|installed| installed.grant.lease_sequence >= candidate.grant.lease_sequence) - { - current.clone() - } else { - Some(Arc::clone(&candidate)) + let local_deadline_ms = grant_deadline(&grant, policy, received_wall_ms, received_monotonic_ms)?; + let grant = Arc::new(grant); + self.state.rcu(|installed| { + if grant.lease_sequence <= installed.highest_sequence { + return Arc::clone(installed); } + let published_at = self.current_time(received_monotonic_ms, started); + let continuity = grant + .assignments + .iter() + .map(|assignment| { + installed + .current + .as_ref() + .and_then(|previous| { + (published_at < previous.local_deadline_ms) + .then(|| previous.grant.assignments.binary_search(assignment).ok()) + .flatten() + .map(|index| previous.continuity[index]) + }) + .unwrap_or(grant.lease_sequence) + }) + .collect(); + Arc::new(AuthorityState { + highest_sequence: grant.lease_sequence, + current: Some(AuthoritySnapshot { + grant: Arc::clone(&grant), + local_deadline_ms, + continuity, + }), + }) }); - if self - .snapshot - .load_full() - .as_ref() - .is_some_and(|installed| installed.grant == candidate.grant) - { - Ok(()) - } else { - Err(AuthorityError::StaleGrant) + let installed = self.state.load(); + let current = installed.current.as_ref().ok_or(AuthorityError::StaleGrant)?; + if current.grant != grant { + return Err(AuthorityError::StaleGrant); + } + if self.current_time(received_monotonic_ms, started) >= current.local_deadline_ms { + return Err(AuthorityError::LeaseExpired); } + Ok(()) } - /// Checks one partition request without taking a lock. - /// + /// Check an assignment before request admission. /// # Errors - /// - /// Returns the precise fence reason before a request reaches its WAL. + /// Returns the precise lease, catalog or assignment fence reason. pub fn authorize( &self, catalog_generation: u64, @@ -125,34 +146,62 @@ impl ServingAuthority { owner_epoch: u64, now_monotonic_ms: u64, ) -> Result<(), AuthorityError> { - let snapshot = self.snapshot.load_full().ok_or(AuthorityError::LeaseExpired)?; - if now_monotonic_ms >= snapshot.local_deadline_ms { - return Err(AuthorityError::LeaseExpired); - } - if catalog_generation != snapshot.grant.catalog_generation { - return Err(AuthorityError::StaleCatalog); - } - let assignment = ServingAssignment { - partition_id, - owner_epoch, - }; - if snapshot.grant.assignments.binary_search(&assignment).is_err() { - return Err(AuthorityError::NotAssigned); - } - Ok(()) + self.capture(catalog_generation, partition_id, owner_epoch, now_monotonic_ms) + .map(|_| ()) } + /// Revoke authority without discarding the accepted sequence fence. pub fn clear(&self) { - self.snapshot.store(None); + self.state.rcu(|state| { + Arc::new(AuthorityState { + highest_sequence: state.highest_sequence, + current: None, + }) + }); } #[must_use] pub fn has_live_grant(&self, catalog_generation: u64, now_monotonic_ms: u64) -> bool { - self.snapshot.load().as_ref().is_some_and(|snapshot| { + if self.process.capture(Instant::now()).is_err() { + return false; + } + let now_monotonic_ms = now_monotonic_ms.max(self.monotonic_ms()); + self.state.load().current.as_ref().is_some_and(|snapshot| { snapshot.grant.catalog_generation == catalog_generation && now_monotonic_ms < snapshot.local_deadline_ms }) } + + fn current_time(&self, received: u64, started: Instant) -> u64 { + let elapsed = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX); + self.monotonic_ms().max(received.saturating_add(elapsed)) + } + + #[must_use] + pub fn process(&self) -> &Arc { + &self.process + } +} + +fn grant_deadline( + grant: &ServingGrant, + policy: &DomainMonitorDescriptor, + wall: u64, + monotonic: u64, +) -> Result { + let margin = policy + .max_clock_skew_ms + .checked_add(policy.self_fence_margin_ms) + .ok_or(AuthorityError::InvalidPolicy)?; + let remaining = grant + .expires_at_ms + .checked_sub(margin) + .and_then(|safe| safe.checked_sub(wall)) + .filter(|remaining| *remaining > 0) + .ok_or(AuthorityError::LeaseExpired)?; + monotonic + .checked_add(remaining) + .ok_or(AuthorityError::InvalidGrant) } #[must_use] diff --git a/app/crowdb-chunk-kv-server/src/serving/lease/context.rs b/app/crowdb-chunk-kv-server/src/serving/lease/context.rs new file mode 100644 index 000000000..8f5f57cad --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/serving/lease/context.rs @@ -0,0 +1,125 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Immutable admission context and shared grant-snapshot validation. + +use super::{AuthorityError, AuthoritySnapshot, AuthorityState, ServingAuthority}; +use crowdb_kv_client::StreamProcessContext; +use crowdb_protocol::chunk_kv::{Id128, ServingAssignment}; +use std::time::Instant; + +/// One admitted assignment; later grants cannot revive it after exclusion. +#[derive(Clone, Copy, Debug)] +pub struct ServingContext { + instance_id: u64, + assignment: ServingAssignment, + continuity: u64, + deadline_ms: u64, + process: StreamProcessContext, +} + +impl ServingContext { + /// Original IO admission deadline; renewal does not extend already issued IO. + #[must_use] + pub const fn deadline_ms(&self) -> u64 { + self.deadline_ms + } +} + +impl ServingAuthority { + /// Capture continuity and the original deadline before asynchronous work. + /// # Errors + /// Rejects expired, unassigned or stale-catalog admission. + pub fn capture( + &self, + catalog_generation: u64, + partition_id: Id128, + owner_epoch: u64, + now_monotonic_ms: u64, + ) -> Result { + let process = self + .process + .capture(Instant::now()) + .map_err(|_| AuthorityError::LeaseExpired)?; + let state = self.state.load(); + let snapshot = live_snapshot(&state, now_monotonic_ms.max(self.monotonic_ms()))?; + if catalog_generation != snapshot.grant.catalog_generation { + return Err(AuthorityError::StaleCatalog); + } + let assignment = ServingAssignment { + partition_id, + owner_epoch, + }; + let index = snapshot + .grant + .assignments + .binary_search(&assignment) + .map_err(|_| AuthorityError::NotAssigned)?; + Ok(ServingContext { + instance_id: self.instance_id, + assignment, + continuity: snapshot.continuity[index], + deadline_ms: snapshot.local_deadline_ms.min( + u64::try_from( + process + .deadline() + .saturating_duration_since(self.clock_origin) + .as_millis(), + ) + .unwrap_or(u64::MAX), + ), + process, + }) + } + + /// Validate the admitted assignment before acknowledging asynchronous work. + /// # Errors + /// Rejects expiry, replacement, removal or a break in lease continuity. + pub fn validate_context( + &self, + context: &ServingContext, + now_monotonic_ms: u64, + ) -> Result<(), AuthorityError> { + self.validate_contexts(std::slice::from_ref(context), now_monotonic_ms) + } + + pub(crate) fn validate_contexts( + &self, + contexts: &[ServingContext], + now_monotonic_ms: u64, + ) -> Result<(), AuthorityError> { + self.process + .capture(Instant::now()) + .map_err(|_| AuthorityError::LeaseExpired)?; + if contexts.is_empty() { + return Ok(()); + } + let state = self.state.load(); + let snapshot = live_snapshot(&state, now_monotonic_ms.max(self.monotonic_ms()))?; + for context in contexts { + self.process + .validate_context(&context.process, Instant::now()) + .map_err(|_| AuthorityError::LeaseExpired)?; + if context.instance_id != self.instance_id { + return Err(AuthorityError::WrongInstance); + } + let index = snapshot + .grant + .assignments + .binary_search(&context.assignment) + .map_err(|_| AuthorityError::LeaseExpired)?; + if snapshot.continuity[index] != context.continuity { + return Err(AuthorityError::LeaseExpired); + } + } + Ok(()) + } +} + +fn live_snapshot(state: &AuthorityState, now: u64) -> Result<&AuthoritySnapshot, AuthorityError> { + state + .current + .as_ref() + .filter(|snapshot| now < snapshot.local_deadline_ms) + .ok_or(AuthorityError::LeaseExpired) +} diff --git a/app/crowdb-chunk-kv-server/src/serving/process.rs b/app/crowdb-chunk-kv-server/src/serving/process.rs new file mode 100644 index 000000000..6938af83c --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/serving/process.rs @@ -0,0 +1,179 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One boot's independent Group-0 process-lease renewal. + +use std::{ + sync::Arc, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use crowdb_kv_client::{ + CrowdbKvClient, StreamProcessAuthority, StreamProcessLeaseClient, StreamProcessLeaseError, + StreamProcessLeaseTiming, +}; +use crowdb_protocol::{ + chunk_kv::DomainMonitorDescriptor, + chunk_stream::{StreamProcessLease, StreamPurpose}, +}; +use tokio::task::JoinHandle; +use tracing::warn; + +pub struct ProcessLeaseSession { + authority: Arc, + client: Arc, + renewal: JoinHandle<()>, +} + +impl ProcessLeaseSession { + /// Claim once from a fixed predecessor and renew independently of recovery/data IO. + /// # Errors + /// Returns invalid policy, a changed predecessor or an unavailable lease. + pub async fn start( + kv: Arc, + instance_id: u64, + policy: &DomainMonitorDescriptor, + ) -> Result { + policy + .validate() + .map_err(|error| StreamProcessLeaseError::Invalid(error.to_string()))?; + let client = Arc::new(StreamProcessLeaseClient::new(kv)); + let nonce = StreamProcessLease::generate_boot_nonce() + .map_err(|error| StreamProcessLeaseError::Invalid(error.to_string()))?; + let predecessor = client + .load(StreamPurpose::Wal, instance_id) + .await? + .map(|record| record.identity); + let record = client + .claim( + StreamPurpose::Wal, + instance_id, + nonce, + predecessor, + wall_ms(), + policy.lease_duration_ms, + ) + .await?; + let authority = Arc::new(StreamProcessAuthority::new( + record, + StreamProcessLeaseTiming { + max_clock_skew_ms: policy.max_clock_skew_ms, + self_fence_margin_ms: policy.self_fence_margin_ms, + }, + wall_ms(), + Instant::now(), + )?); + let renewal = tokio::spawn(renew( + Arc::clone(&client), + Arc::clone(&authority), + policy.lease_duration_ms, + policy.heartbeat_interval_ms, + )); + Ok(Self { + authority, + client, + renewal, + }) + } + + #[must_use] + pub fn authority(&self) -> Arc { + Arc::clone(&self.authority) + } + + /// Publish a heartbeat from this exact live boot. + /// # Errors + /// Propagates process fencing and conditional metadata errors. + pub async fn publish_observation( + &self, + endpoint: &str, + observation: &crowdb_protocol::common::ChunkKvExtra, + ) -> Result<(), StreamProcessLeaseError> { + let context = self.authority.capture(Instant::now())?; + let result = self + .client + .publish_chunk_kv_observation(context.identity(), endpoint, observation, wall_ms()) + .await; + if matches!( + &result, + Err(StreamProcessLeaseError::Replaced | StreamProcessLeaseError::Expired) + ) { + self.authority.fence(); + } + result?; + self.authority.validate_context(&context, Instant::now()) + } + + /// Remove this boot's discovery entry while retaining its exclusion boundary. + /// # Errors + /// Fails when the lease is expired, replaced or the conditional write fails. + pub async fn withdraw_observation(&self) -> Result<(), StreamProcessLeaseError> { + let context = self.authority.capture(Instant::now())?; + self.client + .withdraw_chunk_kv_observation(context.identity(), wall_ms()) + .await + } + + /// Wake process shutdown after permanent expiry or confirmed replacement. + pub async fn wait_for_fence(&self) { + let mut interval = tokio::time::interval(Duration::from_millis(100)); + while self.authority.capture(Instant::now()).is_ok() { + interval.tick().await; + } + } +} + +impl Drop for ProcessLeaseSession { + fn drop(&mut self) { + self.authority.fence(); + self.renewal.abort(); + } +} + +async fn renew( + client: Arc, + authority: Arc, + lease_ms: u64, + interval_ms: u64, +) { + let mut interval = tokio::time::interval(Duration::from_millis(interval_ms)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + while let Ok(context) = authority.capture(Instant::now()) { + let renewal = async { + interval.tick().await; + client.renew(context.identity(), wall_ms(), lease_ms).await + }; + let result = tokio::time::timeout_at(context.deadline().into(), renewal).await; + match result { + Ok(Ok(record)) => { + if let Err(error) = authority.observe(record, wall_ms(), Instant::now()) { + warn!(%error, "process lease renewal lost local authority"); + break; + } + } + Ok(Err( + error @ (StreamProcessLeaseError::Replaced + | StreamProcessLeaseError::Expired + | StreamProcessLeaseError::Invalid(_)), + )) => { + warn!(%error, "process lease is fenced; restart required"); + break; + } + Ok(Err(error)) => warn!(%error, "process lease renewal unavailable; retaining original deadline"), + Err(_) => { + warn!("process lease renewal reached its original safety deadline"); + break; + } + } + } + authority.fence(); +} + +fn wall_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) + }) +} diff --git a/app/crowdb-chunk-kv-server/src/serving/split.rs b/app/crowdb-chunk-kv-server/src/serving/split.rs index fa4a6cd49..4efd375dd 100644 --- a/app/crowdb-chunk-kv-server/src/serving/split.rs +++ b/app/crowdb-chunk-kv-server/src/serving/split.rs @@ -107,13 +107,6 @@ impl SplitStateMachine { return Err(MonitorError::PlanFailed("split parent was not preparing".into())); } let mut next = self.transition.clone(); - if next.retained_parent_artifact.tree_id != next.parent_artifact.tree_id - || next.retained_parent_artifact.stream_name != next.parent_artifact.stream_name - { - next.retained_parent_artifact - .tail_overlay - .clone_from(&proof.retained_parent_tail_overlay); - } next.child.artifact.tail_overlay = Some(proof.child_tail_overlay.clone()); next.readiness_proof = Some(proof); next.phase = SplitPhase::ChildPrepared; diff --git a/app/crowdb-chunk-kv-server/src/serving/worker.rs b/app/crowdb-chunk-kv-server/src/serving/worker.rs index 22d7ee731..2a433c4a3 100644 --- a/app/crowdb-chunk-kv-server/src/serving/worker.rs +++ b/app/crowdb-chunk-kv-server/src/serving/worker.rs @@ -366,16 +366,7 @@ impl TransitionExecutor { if artifact.child.applied_seq != artifact.cutover_seq { return Err(plan_error("split child does not share the cutover frontier")); } - let mut retained_parent_tail_overlay = None; - let mut retained_parent_artifact = transition.retained_parent_artifact.clone(); - if retained_parent_artifact.tree_id != transition.parent_artifact.tree_id - || retained_parent_artifact.stream_name != transition.parent_artifact.stream_name - { - retained_parent_tail_overlay = Some(split_tail_overlay(&artifact, &artifact.retained_parent)); - retained_parent_artifact - .tail_overlay - .clone_from(&retained_parent_tail_overlay); - } + let retained_parent_artifact = transition.retained_parent_artifact.clone(); info!( transition_id_high = transition.transition_id.high, transition_id_low = transition.transition_id.low, @@ -399,7 +390,6 @@ impl TransitionExecutor { child_applied_seq: artifact.child.applied_seq, child_tree_manifest: artifact.child.tree_manifest, child_root_manifest_generation: artifact.child.root_manifest_generation, - retained_parent_tail_overlay, child_tail_overlay: split_tail_overlay(&artifact, &artifact.child), }) } @@ -492,7 +482,7 @@ fn split_tail_overlay( }, source_epoch: split.parent_epoch, source_stream_name: child.parent_stream_name, - source_stream_manifest_generation: child.parent_stream_manifest_generation, + source_stream_generation: child.parent_stream_generation, replay_offset: child.parent_replay_offset, cutover_offset: child.parent_cutover_offset, base_root_manifest_generation: child.root_manifest_generation, diff --git a/app/crowdb-chunk-kv-server/src/storage.rs b/app/crowdb-chunk-kv-server/src/storage.rs index b9ed9e6c3..ccd64353a 100644 --- a/app/crowdb-chunk-kv-server/src/storage.rs +++ b/app/crowdb-chunk-kv-server/src/storage.rs @@ -17,7 +17,7 @@ use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, PartitionArtifact, SplitChildAssignment, SplitTransition, TailOverlayArtifact, TransferTransition, }; -use crowdb_protocol::chunk_stream::{StreamBinding, StreamBindingState}; +use crowdb_protocol::chunk_stream::{StreamRecord, StreamRecordState}; use crowdb_tree_ffi::{ ChunkPageStoreOptions, ChunkRootCatalog, ChunkTransport, OwnedChunkRpcTransportOptions, PageStore, }; @@ -30,6 +30,7 @@ mod root_catalog; use bootstrap::{initialize_bootstrap_root, recover_bootstrap_partition}; mod split_handoff; mod split_recovery; +mod streams; use root_catalog::KvRootCatalogStore; #[derive(Debug, Error)] @@ -55,6 +56,12 @@ pub struct ChunkKvStorage { metadata_store_id: u64, } +struct TreeStorageGeometry { + mirror_copies: u32, + chunk_capacity_bytes: u64, + strip_capacity_bytes: u32, +} + impl ChunkKvStorage { /// Discovers KV, `ChunkDB`, and `DiskIO` endpoints and creates the shared /// stream runtime used by hosted partitions. @@ -77,29 +84,30 @@ impl ChunkKvStorage { ) .await .map_err(|error| StorageRuntimeError::ChunkIo(error.to_string()))?; - Self::from_parts_with_mirror_copies( + Self::from_parts_with_geometry( kv, chunk_io, config.storage.metadata_store_id, config.storage.stream_writer_lease_ms, - config.storage.stream_mirror_copies, - config.storage.tree_chunk_capacity_bytes, + TreeStorageGeometry { + mirror_copies: config.storage.stream_mirror_copies, + chunk_capacity_bytes: config.storage.tree_chunk_capacity_bytes, + strip_capacity_bytes: config.storage.tree_strip_capacity_bytes, + }, StreamConfig { chunk_capacity_bytes: config.storage.stream_chunk_capacity_bytes, - extent_page_entries: config.storage.stream_extent_page_entries, ..StreamConfig::default() }, ) .await } - async fn from_parts_with_mirror_copies( + async fn from_parts_with_geometry( kv: Arc, chunk_io: ChunkIoClient, metadata_store_id: u64, writer_lease_ms: u64, - stream_mirror_copies: u32, - tree_chunk_capacity_bytes: u64, + tree: TreeStorageGeometry, stream_config: StreamConfig, ) -> Result { let streams = Arc::new( @@ -109,20 +117,11 @@ impl ChunkKvStorage { writer_lease_ms, ChunkReadPolicy::default(), stream_config, - stream_mirror_copies, + tree.mirror_copies, ) .map_err(|error| StorageRuntimeError::Stream(error.to_string()))?, ); - Self::assemble( - kv, - chunk_io, - streams, - metadata_store_id, - writer_lease_ms, - stream_mirror_copies, - tree_chunk_capacity_bytes, - ) - .await + Self::assemble(kv, chunk_io, streams, metadata_store_id, writer_lease_ms, tree).await } /// Assembles production adapters from already connected process clients. @@ -137,13 +136,16 @@ impl ChunkKvStorage { writer_lease_ms: u64, stream_mirror_copies: u32, ) -> Result { - Self::from_parts_with_mirror_copies( + Self::from_parts_with_geometry( kv, chunk_io, metadata_store_id, writer_lease_ms, - stream_mirror_copies, - 256 * 1024 * 1024, + TreeStorageGeometry { + mirror_copies: stream_mirror_copies, + chunk_capacity_bytes: 256 * 1024 * 1024, + strip_capacity_bytes: 0, + }, StreamConfig::default(), ) .await @@ -155,8 +157,7 @@ impl ChunkKvStorage { streams: Arc, metadata_store_id: u64, writer_lease_ms: u64, - mirror_copies: u32, - tree_chunk_capacity_bytes: u64, + tree: TreeStorageGeometry, ) -> Result { let (chunkdb, disks) = chunk_io .native_storage_routes(writer_lease_ms) @@ -177,7 +178,8 @@ impl ChunkKvStorage { writer_lease_ms, rpc_timeout_ms: writer_lease_ms, completion_capacity: 1_024, - mirror_copies, + mirror_copies: tree.mirror_copies, + strip_capacity_bytes: tree.strip_capacity_bytes, }) .map_err(|error| StorageRuntimeError::Tree(error.to_string()))?, ); @@ -186,8 +188,8 @@ impl ChunkKvStorage { chunk_io, streams, tree_transport, - tree_mirror_copies: mirror_copies, - tree_chunk_capacity_bytes, + tree_mirror_copies: tree.mirror_copies, + tree_chunk_capacity_bytes: tree.chunk_capacity_bytes, metadata_store_id, }) } @@ -254,7 +256,7 @@ impl ChunkKvStorage { ) } - /// Initializes the stream selected by an Active catalog binding. + /// Initializes the stream selected by an Active catalog record. /// /// # Errors /// @@ -291,29 +293,38 @@ impl ChunkKvStorage { /// /// # Errors /// - /// Returns an error when the stream binding is absent, either durable + /// Returns an error when the stream record is absent, either durable /// artifact cannot be reopened exactly, or WAL replay fails. pub async fn recover_partition( &self, entry: &ChunkKvRangeCatalogEntry, ) -> Result { let stream_name = entry.artifact.stream_name; - let binding = self + let record = self .streams .registry() .load(stream_name) .await .map_err(|error| StorageRuntimeError::Stream(error.to_string()))? - .ok_or_else(|| StorageRuntimeError::Stream("assigned stream binding does not exist".into()))?; - let stream = self - .open_stream(stream_name, entry.owner_epoch) + .ok_or_else(|| StorageRuntimeError::Stream("assigned stream record does not exist".into()))?; + let stream = if streams::is_transfer_target(entry) { + self.open_or_create_stream( + stream_name, + entry.partition_id, + entry.owner_epoch, + record.metadata_group_id, + ) .await - .map_err(|error| { - StorageRuntimeError::Stream(format!( - "target stream {stream_name:?} at owner epoch {}: {error}", - entry.owner_epoch - )) - })?; + .map_err(|error| StorageRuntimeError::Stream(error.to_string())) + } else { + self.open_stream(stream_name, entry.owner_epoch).await + } + .map_err(|error| { + StorageRuntimeError::Stream(format!( + "target stream {stream_name:?} at owner epoch {}: {error}", + entry.owner_epoch + )) + })?; let open_generation = entry .artifact .tail_overlay @@ -333,7 +344,7 @@ impl ChunkKvStorage { let catalog = KvRootCatalogStore::for_assignment( self.kv.clone(), self.metadata_store_id, - binding.metadata_group_id, + record.metadata_group_id, entry, ) .await?; @@ -407,7 +418,7 @@ impl ChunkKvStorage { .load(entry.artifact.stream_name) .await .map_err(|error| storage_plan_error(&error.to_string()))? - .ok_or_else(|| storage_plan_error("target stream binding is absent"))? + .ok_or_else(|| storage_plan_error("target stream record is absent"))? .metadata_group_id, entry, ) @@ -446,13 +457,18 @@ impl ChunkKvStorage { &self, config: &BootstrapPartitionConfig, ) -> Result { - let binding = StreamBinding { + let record = StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, stream_name: config.stream_name, metadata_group_id: config.metadata_group_id, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: Some(format!( + "{:016x}{:016x}", + config.partition_id.high, config.partition_id.low + )), + created_at_ms: StreamRecord::creation_time_ms(), }; let existing = self .streams @@ -461,27 +477,30 @@ impl ChunkKvStorage { .await .map_err(|error| StorageRuntimeError::Stream(error.to_string()))?; let fresh = match existing { - Some(observed) if observed != binding => { + Some(observed) if !observed.same_identity(&record) || observed.state != record.state => { return Err(StorageRuntimeError::Stream( - "bootstrap stream binding conflicts".into(), + "bootstrap stream record conflicts".into(), )); } Some(_) => false, None => { self.streams .registry() - .create(binding) + .create(record) .await .map_err(|error| StorageRuntimeError::Stream(error.to_string()))?; true } }; - let stream = if fresh { - self.create_registered_stream(config.stream_name, config.owner_epoch) - .await? - } else { - self.open_stream(config.stream_name, config.owner_epoch).await? - }; + let stream = self + .open_or_create_stream( + config.stream_name, + config.partition_id, + config.owner_epoch, + config.metadata_group_id, + ) + .await + .map_err(|error| StorageRuntimeError::Stream(error.to_string()))?; let page_store = self .open_durable_tree_page_store( ChunkPageStoreOptions { @@ -531,7 +550,7 @@ impl ChunkKvStorage { /// /// # Errors /// - /// Returns an error for conflicting stream bindings, stale tree authority, + /// Returns an error for conflicting Stream records, stale tree authority, /// invalid transition identity, or an incomplete R142 split preparation. pub async fn prepare_split( &self, @@ -559,15 +578,15 @@ impl ChunkKvStorage { parent .release_generation_pin(split_plan(transition).transition_id) .map_err(|error| storage_plan_error(&error.to_string()))?; - let parent_binding = self + let parent_record = self .streams .registry() .load(transition.parent_artifact.stream_name) .await .map_err(|error| storage_plan_error(&error.to_string()))? - .ok_or_else(|| storage_plan_error("split parent stream binding does not exist"))?; + .ok_or_else(|| storage_plan_error("split parent stream record does not exist"))?; let child = self - .split_target(&transition.child, parent_binding.metadata_group_id) + .split_target(&transition.child, parent_record.metadata_group_id) .await?; if child.journal.tail() != 0 { return Err(storage_plan_error( @@ -592,11 +611,12 @@ impl ChunkKvStorage { Ok(prepared) } - /// Pins a live source checkpoint and creates the target-owned empty WAL. + /// Pins a live source checkpoint and registers the target WAL identity. + /// The target process initializes its WAL without a source-owned writer lease. /// /// # Errors /// - /// Returns an error for a missing source binding, checkpoint failure, or target stream conflict. + /// Returns an error for a missing source record, checkpoint failure, or target stream conflict. pub async fn prepare_transfer_source( &self, source: &Partition, @@ -608,18 +628,17 @@ impl ChunkKvStorage { low: transition.transition_id.low, }) .map_err(|error| storage_plan_error(&error.to_string()))?; - let binding = self + let record = self .streams .registry() .load(transition.artifact.stream_name) .await .map_err(|error| storage_plan_error(&error.to_string()))? - .ok_or_else(|| storage_plan_error("transfer source stream binding does not exist"))?; - self.open_or_create_stream( + .ok_or_else(|| storage_plan_error("transfer source stream record does not exist"))?; + self.register_stream( transition.target_artifact.stream_name, - transition.target_epoch, - binding.metadata_group_id, - true, + transition.partition_id, + record.metadata_group_id, ) .await?; let checkpoint = source @@ -635,20 +654,23 @@ impl ChunkKvStorage { checkpoint.root_manifest_generation, ) .map_err(|error| storage_plan_error(&error.to_string()))?; - let snapshot = source.snapshot(); + let frontier = source + .capture_journal_frontier(transition.source_epoch) + .await + .map_err(|error| storage_plan_error(&error.to_string()))?; let mut artifact = transition.target_artifact.clone(); artifact.tail_overlay = Some(TailOverlayArtifact { source_partition_id: transition.partition_id, source_epoch: transition.source_epoch, source_stream_name: transition.artifact.stream_name, - source_stream_manifest_generation: checkpoint.stream_manifest_generation, + source_stream_generation: frontier.stream_generation, replay_offset: checkpoint.replay_offset, - cutover_offset: snapshot.journal_durable_offset, + cutover_offset: frontier.offset, base_tree_manifest: checkpoint.tree_manifest, base_root_manifest_generation: checkpoint.root_manifest_generation, base_applied_seq: checkpoint.applied_seq, - cutover_seq: snapshot.journal_durable_seq, - target_stream_start_seq: snapshot.journal_durable_seq.saturating_add(1), + cutover_seq: frontier.mutation_seq, + target_stream_start_seq: frontier.mutation_seq.saturating_add(1), }); Ok(artifact) } @@ -658,18 +680,24 @@ impl ChunkKvStorage { child: &SplitChildAssignment, metadata_group_id: u64, ) -> Result { - self.split_target_artifact(&child.artifact, child.owner_epoch, metadata_group_id) - .await + self.split_target_artifact( + &child.artifact, + child.partition_id, + child.owner_epoch, + metadata_group_id, + ) + .await } async fn split_target_artifact( &self, artifact: &PartitionArtifact, + partition_id: crowdb_protocol::chunk_kv::Id128, owner_epoch: u64, metadata_group_id: u64, ) -> Result { let stream = self - .open_or_create_stream(artifact.stream_name, owner_epoch, metadata_group_id, false) + .open_or_create_stream(artifact.stream_name, partition_id, owner_epoch, metadata_group_id) .await?; let page_store = self .open_durable_tree_page_store( @@ -700,56 +728,6 @@ impl ChunkKvStorage { ), }) } - - async fn open_or_create_stream( - &self, - stream_name: StreamName, - writer_epoch: u64, - metadata_group_id: u64, - require_empty: bool, - ) -> Result { - let registry = self.streams.registry(); - let expected = StreamBinding { - purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, - stream_name, - metadata_group_id, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), - }; - match registry - .load(stream_name) - .await - .map_err(|error| storage_plan_error(&error.to_string()))? - { - Some(observed) if observed != expected => { - return Err(storage_plan_error("split child stream binding conflicts")); - } - Some(_) => {} - None => { - if let Err(error) = registry.create(expected.clone()).await { - let reconciled = registry - .load(stream_name) - .await - .map_err(|error| storage_plan_error(&error.to_string()))?; - if reconciled.as_ref() != Some(&expected) { - return Err(storage_plan_error(&error.to_string())); - } - } - } - } - let stream = match self.create_registered_stream(stream_name, writer_epoch).await { - Ok(stream) => stream, - Err(_) => self - .open_stream(stream_name, writer_epoch) - .await - .map_err(|error| storage_plan_error(&error.to_string()))?, - }; - if require_empty && stream.tail() != 0 { - return Err(storage_plan_error("target WAL is not empty")); - } - Ok(stream) - } } fn prepared_overlay_artifact( @@ -777,7 +755,7 @@ fn prepared_overlay_artifact( }, parent_epoch: overlay.source_epoch, parent_stream_name: overlay.source_stream_name, - parent_stream_manifest_generation: overlay.source_stream_manifest_generation, + parent_stream_generation: overlay.source_stream_generation, parent_replay_offset: overlay.replay_offset, parent_cutover_offset: overlay.cutover_offset, applied_seq: overlay.cutover_seq, diff --git a/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs index 95198a078..6b56650fa 100644 --- a/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs +++ b/app/crowdb-chunk-kv-server/src/storage/root_catalog.rs @@ -156,7 +156,7 @@ impl KvRootCatalogStore { let assigned = pages.iter().flat_map(|page| &page.entries).any(|entry| { entry.partition_id == prepared.partition_id && entry.range == prepared.range - && entry.owner == prepared.owner + && entry.owner.instance_id == prepared.owner.instance_id && entry.owner_epoch == prepared.owner_epoch && entry.artifact.tree_id == prepared.artifact.tree_id && entry.artifact.stream_name == prepared.artifact.stream_name @@ -530,7 +530,7 @@ fn same_recovery_base(entry: &ChunkKvRangeCatalogEntry, prepared: &ChunkKvRangeC current.source_partition_id == base.source_partition_id && current.source_epoch == base.source_epoch && current.source_stream_name == base.source_stream_name - && current.source_stream_manifest_generation == base.source_stream_manifest_generation + && current.source_stream_generation == base.source_stream_generation && current.replay_offset == base.replay_offset && current.base_root_manifest_generation == base.base_root_manifest_generation && current.base_tree_manifest == base.base_tree_manifest diff --git a/app/crowdb-chunk-kv-server/src/storage/split_handoff.rs b/app/crowdb-chunk-kv-server/src/storage/split_handoff.rs index 72abc5af5..ea94bb82b 100644 --- a/app/crowdb-chunk-kv-server/src/storage/split_handoff.rs +++ b/app/crowdb-chunk-kv-server/src/storage/split_handoff.rs @@ -32,7 +32,7 @@ impl SplitHandoffStore for SplitHandoffCommit { source_partition_id: current.parent_id, source_epoch: current.parent_epoch, source_stream_name: artifact.parent_stream_name, - source_stream_manifest_generation: artifact.parent_stream_manifest_generation, + source_stream_generation: artifact.parent_stream_generation, replay_offset: artifact.parent_replay_offset, cutover_offset: artifact.parent_cutover_offset, base_tree_manifest: artifact.tree_manifest, diff --git a/app/crowdb-chunk-kv-server/src/storage/streams.rs b/app/crowdb-chunk-kv-server/src/storage/streams.rs new file mode 100644 index 000000000..6bd13f93e --- /dev/null +++ b/app/crowdb-chunk-kv-server/src/storage/streams.rs @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Stable WAL identity registration and initialization by its actual writer. + +use super::{storage_plan_error, ChunkKvStorage, ChunkStream, StreamName, StreamRecord, StreamRecordState}; +use crowdb_chunk_stream::StreamRegistry; +use crowdb_protocol::chunk_kv::{ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogPartitionState, Id128}; + +impl ChunkKvStorage { + pub(super) async fn register_stream( + &self, + stream_name: StreamName, + partition_id: Id128, + metadata_group_id: u64, + ) -> Result<(), crate::MonitorError> { + let registry = self.streams.registry(); + let expected = StreamRecord { + purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, + stream_name, + metadata_group_id, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: Some(format!("{:016x}{:016x}", partition_id.high, partition_id.low)), + created_at_ms: StreamRecord::creation_time_ms(), + }; + match registry + .load(stream_name) + .await + .map_err(|error| storage_plan_error(&error.to_string()))? + { + Some(observed) if !observed.same_identity(&expected) || observed.state != expected.state => { + return Err(storage_plan_error("WAL Stream record conflicts")); + } + Some(_) => {} + None => { + if let Err(error) = registry.create(expected.clone()).await { + let reconciled = registry + .load(stream_name) + .await + .map_err(|error| storage_plan_error(&error.to_string()))?; + if reconciled.as_ref().map_or(true, |record| { + !record.same_identity(&expected) || record.state != expected.state + }) { + return Err(storage_plan_error(&error.to_string())); + } + } + } + } + Ok(()) + } + + pub(super) async fn open_or_create_stream( + &self, + stream_name: StreamName, + partition_id: Id128, + writer_epoch: u64, + metadata_group_id: u64, + ) -> Result { + self.register_stream(stream_name, partition_id, metadata_group_id) + .await?; + match self.create_registered_stream(stream_name, writer_epoch).await { + Ok(stream) => Ok(stream), + Err(initialization) => self + .open_stream(stream_name, writer_epoch) + .await + .map_err(|recovery| { + storage_plan_error(&format!( + "stream initialization failed: {initialization}; recovery failed: {recovery}" + )) + }), + } + } +} + +pub(super) fn is_transfer_target(entry: &ChunkKvRangeCatalogEntry) -> bool { + matches!( + entry.state, + ChunkKvRangeCatalogPartitionState::Prepared | ChunkKvRangeCatalogPartitionState::TargetCatchingUp + ) && entry.artifact.tail_overlay.is_some() + && entry.transition_id.is_some_and(|id| { + entry.artifact.stream_name + == StreamName { + high: id.high, + low: id.low, + } + }) +} diff --git a/app/crowdb-chunk-kv-server/tests/bootstrap_recovery_test.rs b/app/crowdb-chunk-kv-server/tests/bootstrap_recovery_test.rs index c6fe105a8..348a6bfed 100644 --- a/app/crowdb-chunk-kv-server/tests/bootstrap_recovery_test.rs +++ b/app/crowdb-chunk-kv-server/tests/bootstrap_recovery_test.rs @@ -7,7 +7,7 @@ use crowdb_chunk_kv::{ use crowdb_chunk_kv_server::{BootstrapPartitionConfig, ChunkKvStorage}; use crowdb_chunk_stream::memory::MemoryStreamStore; use crowdb_chunk_stream::{ - ChunkStream, StreamBinding, StreamBindingState, StreamConfig, StreamName, StreamPurpose, + ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, StreamRecordState, }; use crowdb_protocol::chunk_kv::Id128; use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, PageStore}; @@ -25,13 +25,15 @@ async fn bootstrap_resume_finishes_its_first_checkpoint_and_does_not_reinitializ let name = StreamName { high: 2, low: 11 }; let store = Arc::new(MemoryStreamStore::new(4096)); let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: StreamPurpose::Wal, stream_name: name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 4, StreamConfig::default(), diff --git a/app/crowdb-chunk-kv-server/tests/catalog_transition_test.rs b/app/crowdb-chunk-kv-server/tests/catalog_transition_test.rs index ae31e5cb1..45e0b365d 100644 --- a/app/crowdb-chunk-kv-server/tests/catalog_transition_test.rs +++ b/app/crowdb-chunk-kv-server/tests/catalog_transition_test.rs @@ -118,7 +118,7 @@ fn prepared_transfer() -> TransferTransition { source_partition_id: id(1), source_epoch: 3, source_stream_name: source_artifact.stream_name, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 40, base_root_manifest_generation: 1, @@ -205,7 +205,7 @@ fn prepared_split() -> SplitTransition { source_partition_id: id(1), source_epoch: 3, source_stream_name: artifact(11).stream_name, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 55, base_root_manifest_generation: 1, @@ -214,8 +214,7 @@ fn prepared_split() -> SplitTransition { cutover_seq: 55, target_stream_start_seq: 56, }; - let mut retained_parent_artifact = artifact(12); - retained_parent_artifact.tail_overlay = Some(overlay.clone()); + let retained_parent_artifact = artifact(11); let mut child_artifact = artifact(14); child_artifact.tail_overlay = Some(overlay.clone()); SplitTransition { @@ -254,7 +253,6 @@ fn prepared_split() -> SplitTransition { child_applied_seq: 55, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay: Some(overlay.clone()), child_tail_overlay: overlay, }), failure: None, @@ -282,7 +280,7 @@ async fn split_shrinks_parent_and_adds_exact_child_in_one_generation() { transition .readiness_proof .as_ref() - .and_then(|proof| proof.retained_parent_tail_overlay.clone()) + .and_then(|proof| proof.retained_parent_artifact.tail_overlay.clone()) ); assert_eq!( pages[0].entries[1].artifact.tail_overlay, @@ -326,3 +324,36 @@ async fn cutover_rejects_unprepared_or_stale_transition() { Err(ChunkKvRangeCatalogError::TransitionConflict) ); } + +#[tokio::test] +async fn fixed_transfer_accepts_recovered_source_address_with_the_same_owner() { + let (store, _, mut pages) = seeded_catalog().await; + pages[0].generation = 2; + pages[0].entries[0].owner.rpc_endpoint = "192.0.2.101:7401".into(); + pages[0].seal().unwrap(); + let mut rebound = head(2, &pages); + rebound.previous_generation = Some(1); + rebound.seal().unwrap(); + let publisher = ChunkKvRangeCatalogPublisher::new(store.clone()); + publisher.publish(rebound, pages).await.unwrap(); + let mut transition = prepared_transfer(); + transition.phase = TransferPhase::TargetCatchingUp; + transition.catchup_proof = None; + transition.validate().unwrap(); + let cutover = ChunkKvRangeCatalogCutover::new(store.clone()); + assert_eq!(cutover.publish_transfer(&transition).await.unwrap(), 3); + let (_, current_pages) = publisher.load_current().await.unwrap().unwrap(); + let entry = ¤t_pages[0].entries[0]; + assert_eq!(entry.owner, transition.target); + assert_eq!(entry.owner_epoch, transition.target_epoch); + assert_eq!(entry.artifact, transition.target_artifact); + let mut moved = current_pages; + moved[0].generation = 4; + moved[0].entries[0].owner.rpc_endpoint = "192.0.2.102:7401".into(); + moved[0].seal().unwrap(); + let mut rebound = head(4, &moved); + rebound.previous_generation = Some(3); + rebound.seal().unwrap(); + publisher.publish(rebound, moved).await.unwrap(); + assert_eq!(cutover.publish_transfer(&transition).await.unwrap(), 4); +} diff --git a/app/crowdb-chunk-kv-server/tests/common/process.rs b/app/crowdb-chunk-kv-server/tests/common/process.rs new file mode 100644 index 000000000..c0c9885b8 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/process.rs @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv_client::{StreamProcessAuthority, StreamProcessLeaseTiming}; +use crowdb_protocol::chunk_stream::{StreamProcessIdentity, StreamProcessLease, StreamPurpose}; +use std::{sync::Arc, time::Instant}; + +pub struct TestProcess; + +impl TestProcess { + pub fn authority(instance_id: u64) -> Arc { + Arc::new( + StreamProcessAuthority::new( + StreamProcessLease { + identity: StreamProcessIdentity { + purpose: StreamPurpose::Wal, + instance_id, + process_epoch: 1, + boot_nonce: [1; 16], + }, + expires_at_ms: 3_601_000, + prior_exclusion_deadline_ms: 0, + grant_sequence: 0, + grant_expires_at_ms: 0, + }, + StreamProcessLeaseTiming::default(), + 1_000, + Instant::now(), + ) + .unwrap(), + ) + } +} diff --git a/app/crowdb-chunk-kv-server/tests/common/process_kv.rs b/app/crowdb-chunk-kv-server/tests/common/process_kv.rs new file mode 100644 index 000000000..22629a46a --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/process_kv.rs @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv::cluster::{ + group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::chunk_kv::{DomainFailurePolicy, DomainMonitorDescriptor}; +use std::sync::Arc; + +pub struct TestProcessKv { + store: Arc, + pub kv: Arc, +} + +impl TestProcessKv { + pub async fn start() -> Self { + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new( + 0, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + store.start().await.unwrap(); + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + kv.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); + Self { store, kv } + } + + pub fn policy() -> DomainMonitorDescriptor { + DomainMonitorDescriptor { + domain: "chunk-kv".into(), + service_registry_name: "chunk-kv".into(), + driver_version: 1, + capability_version: 1, + heartbeat_interval_ms: 20, + suspect_after_ms: 60, + dead_after_ms: 100, + lease_duration_ms: 1_000, + max_clock_skew_ms: 1, + self_fence_margin_ms: 1, + failure_policy: DomainFailurePolicy::OperatorOnly, + balance_policy: "count-first-v1".into(), + chunk_kv_range_balance: None, + } + } +} + +impl Drop for TestProcessKv { + fn drop(&mut self) { + self.store.stop(); + } +} diff --git a/app/crowdb-chunk-kv-server/tests/common/serving_lease.rs b/app/crowdb-chunk-kv-server/tests/common/serving_lease.rs new file mode 100644 index 000000000..559428ee2 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/common/serving_lease.rs @@ -0,0 +1,156 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{catalog, policy, routing, EPOCH, INSTANCE_ID}; +use crowdb_chunk_kv::memory::MemoryPartitionTree; +use crowdb_chunk_kv::{ + Partition, PartitionConfig, PartitionId, PartitionJournal, PartitionRange, PartitionTree, + StreamPartitionJournal, +}; +use crowdb_chunk_kv_server::ChunkKvService; +use crowdb_chunk_stream::memory::MemoryStreamStore; +use crowdb_chunk_stream::{ + ChunkStream, StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, StreamRecord, + StreamRecordState, StreamRegistry, +}; +use crowdb_protocol::chunk_kv::{ + BatchMutationItem, BatchMutationRequest, Id128, PartitionRouting, PointOperation, ServingAssignment, + ServingGrant, +}; +use std::sync::Arc; + +pub struct TestServingLease { + pub service: ChunkKvService, + pub partition: Partition, + pub store: Arc, +} + +impl TestServingLease { + pub async fn start() -> Self { + let stream_name = StreamName { high: 30, low: 31 }; + let store = Arc::new(MemoryStreamStore::new(4_096)); + let stream = ChunkStream::create( + StreamRecord { + purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, + stream_name, + metadata_group_id: 7, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + EPOCH, + StreamConfig::default(), + store.clone() as Arc, + store.clone() as Arc, + store.clone() as Arc, + ) + .await + .unwrap(); + let tree: Arc = Arc::new(MemoryPartitionTree::default()); + let journal: Arc = + Arc::new(StreamPartitionJournal::new(stream, stream_name).unwrap()); + let partition = Partition::open( + PartitionId { high: 1, low: 2 }, + PartitionRange { + start: Some(Vec::new()), + end: None, + }, + EPOCH, + PartitionConfig::default(), + tree, + journal, + ) + .unwrap(); + + let (head, page) = catalog(Id128 { high: 1, low: 2 }, stream_name, EPOCH, 1, None); + + let service = + ChunkKvService::new(super::test_process::TestProcess::authority(INSTANCE_ID), 8).unwrap(); + service + .install_catalog_and_reconcile(&head, &[page], std::slice::from_ref(&partition)) + .unwrap(); + let mut grant = ServingGrant { + instance_id: INSTANCE_ID, + process_epoch: 1, + lease_sequence: 1, + catalog_generation: 1, + issued_at_ms: 1_000, + expires_at_ms: 13_000, + assignments: vec![ServingAssignment { + partition_id: Id128 { high: 1, low: 2 }, + owner_epoch: EPOCH, + }], + assignment_digest: [0; 32], + }; + grant.seal(); + service + .authority() + .install(grant, &policy(), 1_000, 50_000) + .unwrap(); + Self { + service, + partition, + store, + } + } + pub fn replace_authority(&self) { + self.service.authority().clear(); + self.renew_authority(); + } + + pub fn renew_authority(&self) { + let mut grant = ServingGrant { + instance_id: INSTANCE_ID, + process_epoch: 1, + lease_sequence: 2, + catalog_generation: 1, + issued_at_ms: 1_100, + expires_at_ms: 13_100, + assignments: vec![ServingAssignment { + partition_id: Id128 { high: 1, low: 2 }, + owner_epoch: EPOCH, + }], + assignment_digest: [0; 32], + }; + grant.seal(); + self.service + .authority() + .install(grant, &policy(), 1_100, 50_100) + .unwrap(); + } + pub fn batch_request() -> BatchMutationRequest { + BatchMutationRequest { + routing: PartitionRouting { + map_revision: 1, + partition_id: Id128 { high: 1, low: 2 }, + owner_epoch: EPOCH, + deadline_ms: Some(2_000), + }, + operations: (1_u8..=2) + .map(|sequence| BatchMutationItem { + request_id: routing(u64::from(sequence)).request_id, + operation: PointOperation::Put { + key: vec![b'k', sequence], + value: b"value".to_vec(), + }, + }) + .collect(), + } + } + + pub async fn wait_for_write( + &self, + mut pending: impl std::future::Future + Unpin, + ) { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + tokio::select! { + () = self.store.wait_for_write() => {} + response = &mut pending => panic!("request completed before its gate: {response:?}"), + } + }) + .await + .unwrap(); + } +} diff --git a/app/crowdb-chunk-kv-server/tests/common/split_storage.rs b/app/crowdb-chunk-kv-server/tests/common/split_storage.rs index 35125a9e1..063ae1000 100644 --- a/app/crowdb-chunk-kv-server/tests/common/split_storage.rs +++ b/app/crowdb-chunk-kv-server/tests/common/split_storage.rs @@ -6,7 +6,7 @@ use crowdb_chunk_kv::{ StreamPartitionJournal, }; use crowdb_chunk_stream::{ - memory::MemoryStreamStore, ChunkStream, StreamBinding, StreamBindingState, StreamConfig, StreamName, + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamRecord, StreamRecordState, }; use std::sync::Arc; @@ -23,13 +23,15 @@ impl TestSplitStorage { pub async fn journal(&self, stream_name: StreamName, epoch: u64) -> Arc { let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, epoch, StreamConfig::default(), diff --git a/app/crowdb-chunk-kv-server/tests/config_test.rs b/app/crowdb-chunk-kv-server/tests/config_test.rs index cfae7cfcd..03292e9d7 100644 --- a/app/crowdb-chunk-kv-server/tests/config_test.rs +++ b/app/crowdb-chunk-kv-server/tests/config_test.rs @@ -25,8 +25,8 @@ fn defaults_close_the_documented_timing_contract() { assert_eq!(config.storage.metadata_store_id, 1); assert_eq!(config.storage.stream_writer_lease_ms, 30_000); assert_eq!(config.storage.tree_chunk_capacity_bytes, 256 * 1024 * 1024); + assert_eq!(config.storage.tree_strip_capacity_bytes, 0); assert_eq!(config.storage.stream_chunk_capacity_bytes, 256 * 1024 * 1024); - assert_eq!(config.storage.stream_extent_page_entries, 1_024); assert_eq!(config.storage.diskio_connections_per_endpoint, 1); assert_eq!(config.storage.diskio_rpc_workers, 2); assert_eq!(config.rpc_workers, 2); @@ -107,18 +107,35 @@ fn management_discovery_requires_explicit_routable_identity() { } #[test] -fn stream_directory_geometry_survives_config_round_trip() { +fn stream_chunk_geometry_survives_config_round_trip() { let mut config = ChunkKvServerConfig { instance_id: 1, ..ChunkKvServerConfig::default() }; config.storage.stream_chunk_capacity_bytes = 1024 * 1024; - config.storage.stream_extent_page_entries = 1; + config.storage.tree_strip_capacity_bytes = 1024 * 1024; config.validate().unwrap(); let restored: ChunkKvServerConfig = toml::from_str(&toml::to_string(&config).unwrap()).unwrap(); assert_eq!(restored.storage, config.storage); - config.storage.stream_extent_page_entries = 0; - assert!(config.validate().is_err()); +} + +#[test] +fn explicit_tree_strips_require_supported_allocation_units() { + let mut config = ChunkKvServerConfig { + instance_id: 1, + ..ChunkKvServerConfig::default() + }; + for bytes in [1, 1024, 1024 * 1024 + 1, 257 * 1024 * 1024] { + config.storage.tree_strip_capacity_bytes = bytes; + assert!( + config.validate().is_err(), + "accepted invalid strip capacity {bytes}" + ); + } + for bytes in [0, 1024 * 1024, 256 * 1024 * 1024] { + config.storage.tree_strip_capacity_bytes = bytes; + config.validate().unwrap(); + } } #[test] diff --git a/app/crowdb-chunk-kv-server/tests/control_store_test.rs b/app/crowdb-chunk-kv-server/tests/control_store_test.rs index f955bcc53..f692e5c26 100644 --- a/app/crowdb-chunk-kv-server/tests/control_store_test.rs +++ b/app/crowdb-chunk-kv-server/tests/control_store_test.rs @@ -170,7 +170,7 @@ fn transfer() -> TransferTransition { source_partition_id: Id128 { high: 1, low: 2 }, source_epoch: 3, source_stream_name: source_artifact.stream_name, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 0, base_root_manifest_generation: 1, @@ -235,8 +235,8 @@ fn split() -> SplitTransition { tail_overlay: None, }, retained_parent_artifact: PartitionArtifact { - tree_id: 8, - stream_name: StreamName { high: 6, low: 8 }, + tree_id: 5, + stream_name: StreamName { high: 6, low: 7 }, tail_overlay: None, }, parent_next_epoch: 4, @@ -271,7 +271,7 @@ fn split_overlay(cutover_seq: u64) -> TailOverlayArtifact { source_partition_id: Id128 { high: 1, low: 2 }, source_epoch: 3, source_stream_name: StreamName { high: 6, low: 7 }, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: cutover_seq, base_root_manifest_generation: 1, @@ -400,8 +400,7 @@ fn split_readiness_extends_handoff_tail_without_replacing_base() { overlay.cutover_seq = 43; overlay.cutover_offset = 43; overlay.target_stream_start_seq = 44; - let mut retained = committed.retained_parent_artifact.clone(); - retained.tail_overlay = Some(overlay.clone()); + let retained = committed.retained_parent_artifact.clone(); let proof = SplitReadinessProof { cutover_seq: 43, parent_next_epoch: committed.parent_next_epoch, @@ -412,7 +411,6 @@ fn split_readiness_extends_handoff_tail_without_replacing_base() { child_applied_seq: 43, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay: Some(overlay.clone()), child_tail_overlay: overlay, }; let mut wrong = proof.clone(); @@ -486,6 +484,7 @@ async fn group0_serving_grant_load_rejects_invalid_authority() { let store = Group0ControlStore::new(kv.clone()); let mut grant = ServingGrant { instance_id: 7, + process_epoch: 1, lease_sequence: 2, catalog_generation: 3, issued_at_ms: 1_000, @@ -600,9 +599,7 @@ async fn group0_split_store_resumes_prepared_child_before_catalog_cutover() { let revision = store.persist_split_transition(&planned, 0).await.unwrap(); let mut machine = SplitStateMachine::restore(planned).unwrap(); machine.begin_parent_prepare().unwrap(); - let retained_parent_tail_overlay = split_overlay(41); - let mut retained_parent_artifact = split().retained_parent_artifact; - retained_parent_artifact.tail_overlay = Some(retained_parent_tail_overlay.clone()); + let retained_parent_artifact = split().retained_parent_artifact; machine .record_child_ready(SplitReadinessProof { cutover_seq: 41, @@ -614,7 +611,6 @@ async fn group0_split_store_resumes_prepared_child_before_catalog_cutover() { child_applied_seq: 41, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay: Some(retained_parent_tail_overlay), child_tail_overlay: split_overlay(41), }) .unwrap(); diff --git a/app/crowdb-chunk-kv-server/tests/lease_test.rs b/app/crowdb-chunk-kv-server/tests/lease_test.rs index c868aec82..f95cb6250 100644 --- a/app/crowdb-chunk-kv-server/tests/lease_test.rs +++ b/app/crowdb-chunk-kv-server/tests/lease_test.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/process.rs"] +mod test_process; + use crowdb_chunk_kv_server::{classify_instance, replacement_may_activate, AuthorityError, ServingAuthority}; use crowdb_protocol::chunk_kv::{ DomainFailurePolicy, DomainMonitorDescriptor, Id128, InstanceHealth, ServingAssignment, ServingGrant, @@ -34,6 +37,7 @@ fn assignment(epoch: u64) -> ServingAssignment { fn grant(sequence: u64, epoch: u64) -> ServingGrant { let mut grant = ServingGrant { instance_id: 7, + process_epoch: 1, lease_sequence: sequence, catalog_generation: 3, issued_at_ms: 1_000, @@ -45,13 +49,44 @@ fn grant(sequence: u64, epoch: u64) -> ServingGrant { grant } +#[test] +fn a_live_grant_for_another_boot_cannot_authorize_the_same_instance() { + let authority = ServingAuthority::new(test_process::TestProcess::authority(7)); + let mut successor = grant(99, 11); + successor.process_epoch = 2; + assert_eq!( + authority.install(successor, &policy(), 1_000, 8_000), + Err(AuthorityError::WrongProcess) + ); + assert!(!authority.has_live_grant(3, 8_001)); +} + +#[test] +fn process_revocation_invalidates_captured_grants_and_prevents_reinstallation() { + let authority = ServingAuthority::new(test_process::TestProcess::authority(7)); + authority.install(grant(1, 11), &policy(), 1_000, 8_000).unwrap(); + let context = authority + .capture(3, assignment(11).partition_id, 11, 8_001) + .unwrap(); + authority.process().fence(); + assert_eq!( + authority.validate_context(&context, 8_002), + Err(AuthorityError::LeaseExpired) + ); + assert!(!authority.has_live_grant(3, 8_002)); + assert_eq!( + authority.install(grant(2, 11), &policy(), 1_100, 8_100), + Err(AuthorityError::LeaseExpired) + ); +} + #[test] fn default_timing_fences_before_replacement_can_activate() { let policy = policy(); assert_eq!(classify_instance(1_000, 7_000, &policy), InstanceHealth::Suspect); assert_eq!(classify_instance(1_000, 11_000, &policy), InstanceHealth::Dead); - let authority = ServingAuthority::new(7); + let authority = ServingAuthority::new(test_process::TestProcess::authority(7)); authority.install(grant(1, 9), &policy, 1_000, 50_000).unwrap(); assert!(authority .authorize(3, assignment(9).partition_id, 9, 59_999) @@ -67,7 +102,7 @@ fn default_timing_fences_before_replacement_can_activate() { #[test] fn grant_authorizes_only_exact_catalog_partition_and_epoch() { - let authority = ServingAuthority::new(7); + let authority = ServingAuthority::new(test_process::TestProcess::authority(7)); authority.install(grant(4, 11), &policy(), 1_000, 8_000).unwrap(); let partition = assignment(11).partition_id; assert!(authority.has_live_grant(3, 8_001)); @@ -85,7 +120,7 @@ fn grant_authorizes_only_exact_catalog_partition_and_epoch() { #[test] fn renewal_is_monotonic_and_epoch_change_revokes_old_assignment() { - let authority = ServingAuthority::new(7); + let authority = ServingAuthority::new(test_process::TestProcess::authority(7)); authority.install(grant(4, 11), &policy(), 1_000, 8_000).unwrap(); authority.install(grant(5, 12), &policy(), 1_100, 8_100).unwrap(); assert_eq!( @@ -99,3 +134,73 @@ fn renewal_is_monotonic_and_epoch_change_revokes_old_assignment() { ); assert!(authority.authorize(3, partition, 12, 8_201).is_ok()); } + +#[test] +fn clearing_authority_preserves_the_sequence_fence() { + let authority = ServingAuthority::new(test_process::TestProcess::authority(7)); + authority.install(grant(4, 11), &policy(), 1_000, 8_000).unwrap(); + authority.clear(); + assert_eq!( + authority.install(grant(4, 11), &policy(), 1_100, 8_100), + Err(AuthorityError::StaleGrant) + ); + assert!(!authority.has_live_grant(3, 8_101)); + authority.install(grant(5, 11), &policy(), 1_200, 8_200).unwrap(); + assert!(authority + .authorize(3, assignment(11).partition_id, 11, 8_201) + .is_ok()); +} + +#[test] +fn continuous_renewal_preserves_acknowledgment_but_not_the_original_io_deadline() { + let authority = ServingAuthority::new(test_process::TestProcess::authority(7)); + authority.install(grant(4, 11), &policy(), 1_000, 8_000).unwrap(); + let context = authority + .capture(3, assignment(11).partition_id, 11, 8_001) + .unwrap(); + assert_eq!(context.deadline_ms(), 18_000); + let mut renewed = grant(5, 11); + renewed.issued_at_ms = 1_100; + renewed.expires_at_ms = 13_100; + renewed.seal(); + authority.install(renewed, &policy(), 1_100, 8_100).unwrap(); + assert!(authority.validate_context(&context, 18_001).is_ok()); + assert_eq!(context.deadline_ms(), 18_000); +} + +#[test] +fn renewing_after_a_gap_does_not_resurrect_an_admitted_context() { + let authority = ServingAuthority::new(test_process::TestProcess::authority(7)); + authority.install(grant(4, 11), &policy(), 1_000, 8_000).unwrap(); + let context = authority + .capture(3, assignment(11).partition_id, 11, 8_001) + .unwrap(); + let mut renewed = grant(5, 11); + renewed.issued_at_ms = 12_000; + renewed.expires_at_ms = 25_000; + renewed.seal(); + authority.install(renewed, &policy(), 12_000, 19_000).unwrap(); + assert_eq!( + authority.validate_context(&context, 19_001), + Err(AuthorityError::LeaseExpired) + ); + let fresh = authority + .capture(3, assignment(11).partition_id, 11, 19_001) + .unwrap(); + assert!(authority.validate_context(&fresh, 19_002).is_ok()); +} + +#[test] +fn restoring_an_assignment_cannot_restore_its_old_context() { + let authority = ServingAuthority::new(test_process::TestProcess::authority(7)); + authority.install(grant(4, 11), &policy(), 1_000, 8_000).unwrap(); + let context = authority + .capture(3, assignment(11).partition_id, 11, 8_001) + .unwrap(); + authority.install(grant(5, 12), &policy(), 1_100, 8_100).unwrap(); + authority.install(grant(6, 11), &policy(), 1_200, 8_200).unwrap(); + assert_eq!( + authority.validate_context(&context, 8_201), + Err(AuthorityError::LeaseExpired) + ); +} diff --git a/app/crowdb-chunk-kv-server/tests/load_sampling_test.rs b/app/crowdb-chunk-kv-server/tests/load_sampling_test.rs index 5dd527ca9..2b14bd441 100644 --- a/app/crowdb-chunk-kv-server/tests/load_sampling_test.rs +++ b/app/crowdb-chunk-kv-server/tests/load_sampling_test.rs @@ -1,13 +1,16 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/process.rs"] +mod test_process; + use crowdb_chunk_kv::{ memory::MemoryPartitionTree, MutationOperation, Partition, PartitionConfig, PartitionId, PartitionRange, RequestId, StreamPartitionJournal, }; use crowdb_chunk_kv_server::ChunkKvService; use crowdb_chunk_stream::{ - memory::MemoryStreamStore, ChunkStream, StreamBinding, StreamBindingState, StreamConfig, StreamName, + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamRecord, StreamRecordState, }; use std::{ sync::Arc, @@ -24,13 +27,15 @@ async fn partition_with_value_size(id: u64, count: u64, value_size: usize) -> Pa )); let name = StreamName { high: 1, low: id }; let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, stream_name: name, metadata_group_id: 1, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 1, StreamConfig::default(), @@ -71,7 +76,7 @@ async fn partition_with_value_size(id: u64, count: u64, value_size: usize) -> Pa #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn large_values_still_supply_two_live_split_witnesses() { - let service = ChunkKvService::new(1, 8).unwrap(); + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 8).unwrap(); let partition = partition_with_value_size(3, 4, 64 * 1024).await; service.install_partition(&partition).unwrap(); tokio::time::timeout(Duration::from_secs(1), async { @@ -94,7 +99,7 @@ async fn large_values_still_supply_two_live_split_witnesses() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn observation_bounds_samples_and_does_not_starve_other_partitions() { - let service = ChunkKvService::new(1, 8).unwrap(); + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 8).unwrap(); // One unsplittable partition must not stop observation of a larger valid range. let one = partition(1, 1).await; let many = partition(2, 1000).await; diff --git a/app/crowdb-chunk-kv-server/tests/management_test.rs b/app/crowdb-chunk-kv-server/tests/management_test.rs index 8200cfd26..3034dc0c7 100644 --- a/app/crowdb-chunk-kv-server/tests/management_test.rs +++ b/app/crowdb-chunk-kv-server/tests/management_test.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/process.rs"] +mod test_process; + use std::sync::Arc; use axum::body::{to_bytes, Body}; @@ -9,7 +12,7 @@ use crowdb_chunk_kv_server::{management_router, ChunkKvService, ManagementState} use tower::ServiceExt; fn app() -> axum::Router { - let service = Arc::new(ChunkKvService::new(7, 8).unwrap()); + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(7), 8).unwrap()); management_router(ManagementState::new(service)) } @@ -23,6 +26,8 @@ async fn health_is_live_while_unleased_server_is_not_ready() { let body = to_bytes(health.into_body(), usize::MAX).await.unwrap(); let body: serde_json::Value = serde_json::from_slice(&body).unwrap(); assert_eq!(body["instance_id"], 7); + assert_eq!(body["process_lease"]["lease"]["identity"]["process_epoch"], 1); + assert_eq!(body["process_lease"]["fenced"], false); assert_eq!(body["lifecycle"], "prepared"); assert_eq!(body["ready"], false); diff --git a/app/crowdb-chunk-kv-server/tests/partition_observation_test.rs b/app/crowdb-chunk-kv-server/tests/partition_observation_test.rs index cdbc19c11..b42284c07 100644 --- a/app/crowdb-chunk-kv-server/tests/partition_observation_test.rs +++ b/app/crowdb-chunk-kv-server/tests/partition_observation_test.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/process.rs"] +mod test_process; + use std::sync::Arc; use axum::{ @@ -13,7 +16,7 @@ use crowdb_chunk_kv::{ }; use crowdb_chunk_kv_server::{management_router, ChunkKvService, ManagementState}; use crowdb_chunk_stream::{ - memory::MemoryStreamStore, ChunkStream, StreamBinding, StreamBindingState, StreamConfig, + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamRecord, StreamRecordState, }; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, @@ -32,13 +35,15 @@ async fn journal() -> StreamPartitionJournal { let store = Arc::new(MemoryStreamStore::new(4096)); let stream_name = StreamName { high: 2, low: 3 }; let stream = ChunkStream::create( - StreamBinding { + StreamRecord { stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, + record_version: 1, + state: StreamRecordState::Active, purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, - owner_kind: Some("chunk-kv-partition".into()), + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, u64::MAX, StreamConfig::default(), @@ -110,7 +115,7 @@ async fn get(app: axum::Router, id: &str, generation: u64, epoch: u64) -> (Statu #[tokio::test] async fn observation_keeps_exact_fences_and_does_not_confuse_serving_with_live_authority() { - let service = Arc::new(ChunkKvService::new(7, 8).unwrap()); + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(7), 8).unwrap()); let (head, page) = catalog(); service.install_catalog(&head, &[page]).unwrap(); let id = format!("{:016x}{:016x}", ID.high, ID.low); @@ -170,8 +175,13 @@ async fn observation_keeps_exact_fences_and_does_not_confuse_serving_with_live_a assert_eq!(advanced["journal_durable_seq"], "1"); assert_eq!(advanced["applied_seq"], "1"); assert_eq!(advanced["tree"]["checkpoint_applied_seq"], "1"); - assert_eq!(advanced["journal"]["extent_pages"].as_array().unwrap().len(), 1); - assert!(advanced["journal"]["active"]["chunk_id"].is_string()); + assert_eq!(advanced["journal"]["chunk_record"]["state"], "Active"); + assert_eq!(advanced["journal"]["published_tail"], "0"); + assert_eq!( + advanced["journal"]["generation"], body["journal"]["generation"], + "append keeps the persisted chunk record unchanged" + ); + assert!(advanced["journal"]["chunk_record"]["chunk_id"].is_string()); assert_eq!(advanced["data_pages_read"], 0); let stale = app .clone() @@ -179,7 +189,12 @@ async fn observation_keeps_exact_fences_and_does_not_confuse_serving_with_live_a Request::get(format!( "/partitions/{id}/observation?generation=9&epoch={}&stream_generation={}", u64::MAX, - body["journal"]["generation"].as_str().unwrap() + body["journal"]["generation"] + .as_str() + .unwrap() + .parse::() + .unwrap() + + 1 )) .body(Body::empty()) .unwrap(), @@ -264,7 +279,7 @@ async fn native_page_windows_preserve_bytes_and_reject_changed_cursors() { .unwrap(); } tree.checkpoint(0).await.unwrap(); - let service = Arc::new(ChunkKvService::new(7, 8).unwrap()); + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(7), 8).unwrap()); let (head, catalog) = catalog(); service.install_catalog(&head, &[catalog]).unwrap(); service.install_partition(&partition).unwrap(); @@ -358,3 +373,65 @@ async fn native_page_partition() -> (Arc, .unwrap(); (tree, partition) } + +#[tokio::test] +async fn terminal_apply_uncertainty_requests_restart_without_reactivating_writer() { + use crowdb_chunk_kv::{ChunkKvError, MutationOperation, PartitionLifecycle, RequestId}; + let service = ChunkKvService::new(test_process::TestProcess::authority(7), 8).unwrap(); + let (head, page) = catalog(); + service.install_catalog(&head, &[page]).unwrap(); + assert!( + service.recovery_required().is_empty(), + "idle instance needs no restart" + ); + let tree = Arc::new(MemoryPartitionTree::with_tree_id(1)); + let partition = Partition::open( + PartitionId { + high: ID.high, + low: ID.low, + }, + PartitionRange::default(), + u64::MAX, + PartitionConfig::default(), + tree.clone(), + Arc::new(journal().await), + ) + .unwrap(); + service.install_partition(&partition).unwrap(); + assert!(service.recovery_required().is_empty()); + tree.fail_next_apply(); + assert_eq!( + partition + .mutate( + u64::MAX, + RequestId { + client_high: 7, + client_low: 8, + client_sequence: 1 + }, + MutationOperation::Put { + key: b"key".to_vec(), + value: b"value".to_vec() + } + ) + .await, + Err(ChunkKvError::ApplyStateUnknown) + ); + let failed = tokio::time::timeout(std::time::Duration::from_secs(2), service.wait_for_recovery()) + .await + .unwrap(); + assert_eq!(failed.len(), 1); + assert_eq!(failed[0].partition_id, ID); + assert_eq!(failed[0].owner_epoch, u64::MAX); + assert_eq!(failed[0].lifecycle, PartitionLifecycle::Recovering); + assert_eq!((failed[0].durable_seq, failed[0].applied_seq), (1, 0)); + assert!(service.health(0).range_readiness[0].detail.contains("Recovering")); + assert!(service.activate_recovered_partition(ID, u64::MAX).is_err()); + assert_eq!(partition.lifecycle(), PartitionLifecycle::Recovering); + service.begin_drain(); + assert_eq!( + service.recovery_required(), + failed, + "drain cannot erase replay evidence" + ); +} diff --git a/app/crowdb-chunk-kv-server/tests/perf_log_test.rs b/app/crowdb-chunk-kv-server/tests/perf_log_test.rs new file mode 100644 index 000000000..da464ab31 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/perf_log_test.rs @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_kv_server::metrics::start_stream_perf_log; +use crowdb_test_harness::test_dirs::tempdir_in_test_data; + +#[tokio::test] +async fn service_perf_log_flushes_registered_stream_histograms_on_shutdown() { + let dir = tempdir_in_test_data("stream-perf-log"); + let path = dir.path().to_str().unwrap(); + assert!(start_stream_perf_log(path, 0, 10, 2).unwrap().is_none()); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 0); + let mut runner = start_stream_perf_log(path, 1, 10, 2).unwrap().unwrap(); + assert!(runner + .registry() + .lock() + .unwrap() + .snapshot_named("stream.checkpoint.cursor.lh", 1.0) + .is_some()); + runner + .registry() + .lock() + .unwrap() + .register_histogram("stream.checkpoint.cursor.lh") + .observe(2_000_000); + runner.stop().await; + let entry = std::fs::read_dir(dir.path()).unwrap().next().unwrap().unwrap(); + let log = std::fs::read_to_string(entry.path()).unwrap(); + assert!( + log.contains("stream.checkpoint.cursor.lh"), + "missing stream phase: {log}" + ); + assert!(log.contains("p50(us)")); + assert!(log.contains("p99(us)")); +} diff --git a/app/crowdb-chunk-kv-server/tests/process_session_test.rs b/app/crowdb-chunk-kv-server/tests/process_session_test.rs new file mode 100644 index 000000000..e0cf5edef --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/process_session_test.rs @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/process_kv.rs"] +mod fixture; + +use crowdb_chunk_kv_server::serving::process::ProcessLeaseSession; +use crowdb_kv_client::StreamProcessLeaseClient; +use crowdb_protocol::chunk_stream::StreamPurpose; +use fixture::TestProcessKv; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +#[tokio::test] +async fn process_renewal_runs_without_data_requests_and_drop_permanently_fences_it() { + let fixture = TestProcessKv::start().await; + let session = ProcessLeaseSession::start(fixture.kv.clone(), 7, &TestProcessKv::policy()) + .await + .unwrap(); + let authority = session.authority(); + let admitted = authority.capture(Instant::now()).unwrap(); + let expiry = authority.status(Instant::now()).lease.expires_at_ms; + tokio::time::timeout(Duration::from_secs(2), async { + while authority.status(Instant::now()).lease.expires_at_ms <= expiry { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + assert!(authority.capture(Instant::now()).unwrap().deadline() > admitted.deadline()); + assert!(authority.validate_context(&admitted, Instant::now()).is_ok()); + drop(session); + assert!(authority.capture(Instant::now()).is_err()); + assert!(authority.status(Instant::now()).fenced); +} + +#[tokio::test] +async fn background_renewal_detects_a_replacement_and_cannot_reclaim_it() { + let fixture = TestProcessKv::start().await; + let session = ProcessLeaseSession::start(fixture.kv.clone(), 7, &TestProcessKv::policy()) + .await + .unwrap(); + let authority = session.authority(); + let old = authority.capture(Instant::now()).unwrap(); + let client = StreamProcessLeaseClient::new(fixture.kv.clone()); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + .try_into() + .unwrap(); + let replacement = client + .claim(StreamPurpose::Wal, 7, [2; 16], Some(old.identity()), now, 1_000) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(2), async { + while !authority.status(Instant::now()).fenced { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap(); + assert!(authority.validate_context(&old, Instant::now()).is_err()); + tokio::time::timeout(Duration::from_secs(1), session.wait_for_fence()) + .await + .unwrap(); + assert_eq!( + client.load(StreamPurpose::Wal, 7).await.unwrap().unwrap(), + replacement + ); +} diff --git a/app/crowdb-chunk-kv-server/tests/rpc_test.rs b/app/crowdb-chunk-kv-server/tests/rpc_test.rs index 388a23428..249e8d3aa 100644 --- a/app/crowdb-chunk-kv-server/tests/rpc_test.rs +++ b/app/crowdb-chunk-kv-server/tests/rpc_test.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/process.rs"] +mod test_process; + use std::sync::Arc; use std::time::Duration; @@ -17,7 +20,7 @@ use crowdb_rpc_ffi::{Buffer, RpcClient, RpcServer}; #[tokio::test] async fn point_request_crosses_real_rpc_boundary_with_typed_response() { crowdb_rpc_ffi::init_test_logging(); - let service = Arc::new(ChunkKvService::new(7, 8).unwrap()); + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(7), 8).unwrap()); let rpc_service = Arc::new(ChunkKvRpcService::new(service, tokio::runtime::Handle::current())); let server = Arc::new(RpcServer::new(None)); server.listen("127.0.0.1", 0).unwrap(); diff --git a/app/crowdb-chunk-kv-server/tests/server_test.rs b/app/crowdb-chunk-kv-server/tests/server_test.rs index f98fc7b57..c37982748 100644 --- a/app/crowdb-chunk-kv-server/tests/server_test.rs +++ b/app/crowdb-chunk-kv-server/tests/server_test.rs @@ -1,19 +1,22 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/process.rs"] +mod test_process; + use std::sync::Arc; use crowdb_chunk_kv::memory::MemoryPartitionTree; use crowdb_chunk_kv::{ Checkpoint, Partition, PartitionConfig, PartitionId, PartitionJournal, PartitionLifecycle, - PartitionRange, PartitionTree, PreparedSplitWriterArtifact, SplitArtifact, SplitChild, SplitPlan, + PartitionRange, PreparedSplitWriterArtifact, SplitArtifact, SplitChild, SplitPlan, StreamPartitionJournal, TransitionId, }; use crowdb_chunk_kv_server::{ChunkKvService, ServerLifecycle, TransferStateMachine}; use crowdb_chunk_stream::memory::MemoryStreamStore; use crowdb_chunk_stream::{ - ChunkStream, StreamBinding, StreamBindingState, StreamChunkStore, StreamConfig, StreamMetadataStore, - StreamName, StreamRegistry, + ChunkStream, StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, StreamRecord, + StreamRecordState, StreamRegistry, }; use crowdb_protocol::chunk_kv::{ AuthorityReleaseProof, ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, @@ -120,7 +123,7 @@ fn prepared_writer_artifact(writer: &Partition) -> PreparedSplitWriterArtifact { parent_id: PartitionId { high: 1, low: 2 }, parent_epoch: EPOCH, parent_stream_name: StreamName { high: 30, low: 31 }, - parent_stream_manifest_generation: 1, + parent_stream_generation: 1, parent_replay_offset: 0, parent_cutover_offset: 0, applied_seq: 0, @@ -155,13 +158,15 @@ async fn prepared_partition_range( ) -> Partition { let store = Arc::new(MemoryStreamStore::new(4_096)); let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, owner_epoch, StreamConfig::default(), @@ -181,7 +186,7 @@ async fn prepared_partition_range( root_manifest_generation: 1, applied_seq: 0, stream_name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), @@ -199,13 +204,15 @@ async fn pending_transfer_fixture() -> (Partition, PreparedSplitWriterArtifact, let open_journal = |stream: ChunkStream, name| -> Arc { Arc::new(StreamPartitionJournal::new(stream, name).unwrap()) }; - let binding = |stream_name| StreamBinding { + let binding = |stream_name| StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }; let source = ChunkStream::create( binding(source_name), @@ -242,7 +249,7 @@ async fn pending_transfer_fixture() -> (Partition, PreparedSplitWriterArtifact, parent_id: PartitionId { high: 9, low: 9 }, parent_epoch: EPOCH, parent_stream_name: source_name, - parent_stream_manifest_generation: 1, + parent_stream_generation: 1, parent_replay_offset: 0, parent_cutover_offset: 0, applied_seq: 0, @@ -257,7 +264,7 @@ async fn pending_transfer_fixture() -> (Partition, PreparedSplitWriterArtifact, root_manifest_generation: 1, applied_seq: 0, stream_name: target_name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), @@ -290,7 +297,7 @@ fn committed_transfer(artifact: &PreparedSplitWriterArtifact) -> TransferTransit }, source_epoch: artifact.parent_epoch, source_stream_name: artifact.parent_stream_name, - source_stream_manifest_generation: artifact.parent_stream_manifest_generation, + source_stream_generation: artifact.parent_stream_generation, replay_offset: artifact.parent_replay_offset, cutover_offset: artifact.parent_cutover_offset, base_root_manifest_generation: artifact.root_manifest_generation, @@ -431,6 +438,7 @@ fn install_local_split_catalog( .unwrap(); let mut grant = ServingGrant { instance_id: INSTANCE_ID, + process_epoch: 1, lease_sequence: 2, catalog_generation: 2, issued_at_ms: 1_100, @@ -577,6 +585,7 @@ async fn assert_old_parent_scan_continuation(service: &ChunkKvService) { async fn assert_retained_seek_needs_no_child_grant(service: &ChunkKvService) { let mut grant = ServingGrant { instance_id: INSTANCE_ID, + process_epoch: 1, lease_sequence: 3, catalog_generation: 2, issued_at_ms: 1_200, @@ -651,55 +660,28 @@ fn assert_hosts_exact_child_assignment(service: &ChunkKvService, child: &Partiti } async fn fixture() -> (ChunkKvService, Partition) { - let stream_name = StreamName { high: 30, low: 31 }; - let store = Arc::new(MemoryStreamStore::new(4_096)); - let stream = ChunkStream::create( - StreamBinding { - purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, - stream_name, - metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), - }, - EPOCH, - StreamConfig::default(), - store.clone() as Arc, - store.clone() as Arc, - store as Arc, - ) - .await - .unwrap(); - let tree: Arc = Arc::new(MemoryPartitionTree::default()); - let journal: Arc = - Arc::new(StreamPartitionJournal::new(stream, stream_name).unwrap()); - let partition = Partition::open( - PartitionId { high: 1, low: 2 }, - PartitionRange { - start: Some(Vec::new()), - end: None, - }, - EPOCH, - PartitionConfig::default(), - tree, - journal, - ) - .unwrap(); - - let (head, page) = catalog(Id128 { high: 1, low: 2 }, stream_name, EPOCH, 1, None); + let fixture = serving_fixture::TestServingLease::start().await; + (fixture.service, fixture.partition) +} - let service = ChunkKvService::new(INSTANCE_ID, 8).unwrap(); +#[tokio::test] +async fn matching_grant_activation_promotes_a_replayed_partition() { + let stream_name = StreamName { high: 30, low: 32 }; + let partition = prepared_partition(PartitionId { high: 1, low: 3 }, stream_name, EPOCH).await; + let service = ChunkKvService::new(test_process::TestProcess::authority(INSTANCE_ID), 4).unwrap(); + let (head, page) = catalog(Id128 { high: 1, low: 3 }, stream_name, EPOCH, 1, None); service .install_catalog_and_reconcile(&head, &[page], std::slice::from_ref(&partition)) .unwrap(); let mut grant = ServingGrant { instance_id: INSTANCE_ID, + process_epoch: 1, lease_sequence: 1, catalog_generation: 1, issued_at_ms: 1_000, expires_at_ms: 13_000, assignments: vec![ServingAssignment { - partition_id: Id128 { high: 1, low: 2 }, + partition_id: Id128 { high: 1, low: 3 }, owner_epoch: EPOCH, }], assignment_digest: [0; 32], @@ -709,18 +691,14 @@ async fn fixture() -> (ChunkKvService, Partition) { .authority() .install(grant, &policy(), 1_000, 50_000) .unwrap(); - (service, partition) -} - -#[tokio::test] -async fn matching_grant_activation_promotes_a_replayed_partition() { - let stream_name = StreamName { high: 30, low: 32 }; - let partition = prepared_partition(PartitionId { high: 1, low: 3 }, stream_name, EPOCH).await; - let service = ChunkKvService::new(INSTANCE_ID, 4).unwrap(); - let (head, page) = catalog(Id128 { high: 1, low: 3 }, stream_name, EPOCH, 1, None); - service - .install_catalog_and_reconcile(&head, &[page], std::slice::from_ref(&partition)) - .unwrap(); + assert_eq!(service.health(50_100).lifecycle, ServerLifecycle::Serving); + assert!(service.health(50_100).range_readiness[0] + .detail + .contains("Prepared")); + assert!( + !service.health(50_100).ready, + "a live grant cannot make an inactive writer ready" + ); assert_eq!(partition.snapshot().lifecycle, PartitionLifecycle::Prepared); assert!(!service.registry_observation(1_024, 0).hosted[0].recovering); assert!(matches!( @@ -734,6 +712,34 @@ async fn matching_grant_activation_promotes_a_replayed_partition() { .unwrap(); assert_eq!(partition.snapshot().lifecycle, PartitionLifecycle::Serving); + assert!(service.health(50_100).ready); + assert!(service.health(50_100).range_readiness[0] + .detail + .contains("current authority")); + assert!( + !service.health(65_000).ready, + "expired authority cannot report ready" + ); +} + +#[test] +fn an_idle_stateless_instance_needs_no_owned_range_or_serving_grant() { + let service = ChunkKvService::new(test_process::TestProcess::authority(INSTANCE_ID + 1), 4).unwrap(); + assert!(!service.health(50_100).ready, "A catalog must first be loaded"); + let (head, page) = catalog( + Id128 { high: 1, low: 3 }, + StreamName { high: 30, low: 32 }, + EPOCH, + 1, + None, + ); + service + .install_catalog_and_reconcile(&head, &[page], &[]) + .unwrap(); + let health = service.health(50_100); + assert!(health.ready); + assert!(health.range_readiness.is_empty()); + assert!(health.detail.contains("no tree-range ownership assigned")); } #[tokio::test] @@ -746,7 +752,10 @@ async fn committed_transfer_activates_a_recovered_overlay() { entries: vec![ChunkKvRangeCatalogEntry { partition_id: transition.partition_id, range: transition.range.clone(), - owner: transition.target.clone(), + owner: crowdb_protocol::chunk_kv::OwnerDescriptor { + rpc_endpoint: "192.0.2.12:7401".into(), + ..transition.target.clone() + }, owner_epoch: transition.target_epoch, state: ChunkKvRangeCatalogPartitionState::Serving, artifact: transition.target_artifact.clone(), @@ -767,7 +776,7 @@ async fn committed_transfer_activates_a_recovered_overlay() { checksum: [0; 32], }; head.seal().unwrap(); - let service = ChunkKvService::new(INSTANCE_ID, 4).unwrap(); + let service = ChunkKvService::new(test_process::TestProcess::authority(INSTANCE_ID), 4).unwrap(); service .install_catalog_and_reconcile(&head, &[page], std::slice::from_ref(&partition)) .unwrap(); @@ -817,6 +826,10 @@ async fn matching_grant_refresh_keeps_a_preparing_parent_active() { .unwrap(); assert_eq!(partition.snapshot().lifecycle, PartitionLifecycle::SplitPreparing); + assert!( + service.health(50_100).ready, + "an active split dispatcher still serves its published range" + ); assert!( !service.registry_observation(1_024, 0).hosted[0].recovering, "a preparing parent must remain eligible for serving-grant renewal" @@ -900,6 +913,7 @@ async fn prepared_split_lineage_suppresses_catalog_recovery() { assert_all_partition_loads_recoverable(&service, true).await; assert_old_parent_ordered_reads(&service).await; assert_old_parent_scan_continuation(&service).await; + split_scan::assert_old_parent_right_interval(&service).await; assert_retained_seek_needs_no_child_grant(&service).await; service.remove_partition(Id128 { high: 1, low: 2 }); @@ -957,7 +971,7 @@ async fn catching_up_target_appends_unconditional_write_before_initialization() }, source_epoch: final_artifact.parent_epoch, source_stream_name: final_artifact.parent_stream_name, - source_stream_manifest_generation: final_artifact.parent_stream_manifest_generation, + source_stream_generation: final_artifact.parent_stream_generation, replay_offset: final_artifact.parent_replay_offset, cutover_offset: final_artifact.parent_cutover_offset, base_root_manifest_generation: final_artifact.root_manifest_generation, @@ -1035,7 +1049,9 @@ async fn catching_up_target_appends_unconditional_write_before_initialization() #[tokio::test] async fn catching_up_read_waits_are_time_and_count_bounded() { let target = pending_transfer_partition().await; - let service = Arc::new(ChunkKvService::new_with_limits(7, 4, 1_024, 1, 20).unwrap()); + let service = Arc::new( + ChunkKvService::new_with_limits(test_process::TestProcess::authority(7), 4, 1_024, 1, 20).unwrap(), + ); let (head, mut page) = catalog( Id128 { high: 1, low: 2 }, target.snapshot().stream_name, @@ -1334,3 +1350,11 @@ async fn ordered_seek_and_scan_preserve_inclusive_range_start() { #[path = "server_test/bounded_scan.rs"] mod bounded_scan; + +#[path = "server_test/lease.rs"] +mod lease; +#[path = "common/serving_lease.rs"] +mod serving_fixture; + +#[path = "server_test/split_scan.rs"] +mod split_scan; diff --git a/app/crowdb-chunk-kv-server/tests/server_test/lease.rs b/app/crowdb-chunk-kv-server/tests/server_test/lease.rs new file mode 100644 index 000000000..7fdb4a7fe --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/server_test/lease.rs @@ -0,0 +1,299 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use serving_fixture::TestServingLease; + +#[tokio::test] +async fn graceful_drain_waits_for_wal_io_and_publishes_its_complete_prefix() { + let fixture = TestServingLease::start().await; + fixture.store.pause_writes(); + let request = PointRequest { + routing: routing(1), + operation: PointOperation::Put { + key: b"key".to_vec(), + value: b"value".to_vec(), + }, + }; + let pending = fixture.service.handle_point(request, 1_000, 50_000); + tokio::pin!(pending); + fixture.wait_for_write(pending.as_mut()).await; + let draining = fixture.service.drain_writers(); + tokio::pin!(draining); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(10), draining.as_mut()) + .await + .is_err() + ); + fixture.store.resume_writes(); + let (_, drained) = tokio::join!(pending, draining); + drained.unwrap(); + assert_eq!( + fixture.partition.lifecycle(), + PartitionLifecycle::TransferQuiesced + ); + let name = fixture.partition.snapshot().stream_name; + let published = fixture.store.load_tail_for_tests(name).await.unwrap().unwrap(); + assert_eq!( + published.record().logical_end, + Some(fixture.partition.snapshot().journal_durable_offset) + ); + assert!(published.record().logical_end.unwrap() > 0); + assert_eq!( + fixture + .store + .record_store(7) + .unwrap() + .candidates(name) + .await + .unwrap() + .len(), + 0 + ); + fixture.service.drain_writers().await.unwrap(); +} + +#[tokio::test] +async fn process_revocation_during_wal_io_prevents_acknowledgment() { + let fixture = TestServingLease::start().await; + fixture.store.pause_writes(); + let request = PointRequest { + routing: routing(1), + operation: PointOperation::Put { + key: b"key".to_vec(), + value: b"value".to_vec(), + }, + }; + let pending = fixture.service.handle_point(request.clone(), 1_000, 50_000); + tokio::pin!(pending); + fixture.wait_for_write(pending.as_mut()).await; + fixture.service.authority().process().fence(); + fixture.store.resume_writes(); + let response = pending.await; + assert!( + matches!(response.result, Err(ref error) if error.code == ChunkKvRpcErrorCode::LeaseExpired), + "{response:?}" + ); + assert!(response.journal_position.is_none()); + assert!(fixture + .partition + .get(EPOCH, b"key", None) + .await + .unwrap() + .is_some()); + let retry = fixture.service.handle_point(request, 1_100, 50_100).await; + assert!( + matches!(retry.result, Err(ref error) if error.code == ChunkKvRpcErrorCode::LeaseExpired), + "{retry:?}" + ); +} + +#[tokio::test] +async fn revoked_inflight_wal_write_cannot_acknowledge_under_a_replacement_grant() { + let fixture = TestServingLease::start().await; + fixture.store.pause_writes(); + let request = PointRequest { + routing: routing(1), + operation: PointOperation::Put { + key: b"key".to_vec(), + value: b"value".to_vec(), + }, + }; + let pending = fixture.service.handle_point(request.clone(), 1_000, 50_000); + tokio::pin!(pending); + fixture.wait_for_write(pending.as_mut()).await; + fixture.replace_authority(); + fixture.store.resume_writes(); + let response = pending.await; + assert!( + matches!(response.result, Err(ref error) if error.code == ChunkKvRpcErrorCode::LeaseExpired), + "{response:?}" + ); + assert!(response.journal_position.is_none()); + assert!(fixture + .partition + .get(EPOCH, b"key", None) + .await + .unwrap() + .is_some()); + let retry = fixture.service.handle_point(request, 1_100, 50_100).await; + assert!(retry.result.is_ok(), "{retry:?}"); + assert!(retry.journal_position.is_some()); +} + +#[tokio::test] +async fn revoked_batch_does_not_admit_remaining_members_under_a_new_grant() { + let fixture = TestServingLease::start().await; + fixture.store.pause_writes(); + let request = TestServingLease::batch_request(); + let pending = fixture + .service + .handle_batch_mutation(request.clone(), 1_000, 50_000); + tokio::pin!(pending); + fixture.wait_for_write(pending.as_mut()).await; + fixture.replace_authority(); + fixture.store.resume_writes(); + let response = pending.await; + assert!( + response.result.as_ref().unwrap().iter().all(|item| { + matches!(&item.result, Err(error) if error.code == ChunkKvRpcErrorCode::LeaseExpired) + && item.journal_position.is_none() + }), + "{response:?}" + ); + assert!(fixture + .partition + .get(EPOCH, &[b'k', 1], None) + .await + .unwrap() + .is_some()); + assert!(fixture + .partition + .get(EPOCH, &[b'k', 2], None) + .await + .unwrap() + .is_none()); + let retried = fixture + .service + .handle_batch_mutation(request, 1_100, 50_100) + .await; + assert!(retried + .result + .unwrap() + .iter() + .all(|item| item.result.is_ok() && item.journal_position.is_some())); +} + +#[tokio::test] +async fn continuous_renewal_preserves_a_batch_waiting_on_its_first_wal_write() { + let fixture = TestServingLease::start().await; + fixture.store.pause_writes(); + let request = TestServingLease::batch_request(); + let pending = fixture.service.handle_batch_mutation(request, 1_000, 50_000); + tokio::pin!(pending); + fixture.wait_for_write(pending.as_mut()).await; + fixture.renew_authority(); + fixture.store.resume_writes(); + let response = pending.await; + assert!( + response + .result + .as_ref() + .unwrap() + .iter() + .all(|item| { item.result.is_ok() && item.journal_position.is_some() }), + "{response:?}" + ); + for sequence in 1..=2 { + assert!(fixture + .partition + .get(EPOCH, &[b'k', sequence], None) + .await + .unwrap() + .is_some()); + } +} + +#[tokio::test] +async fn graceful_drain_releases_writers_during_both_split_stages() { + for finalizing in [false, true] { + let fixture = TestServingLease::start().await; + let transition_id = TransitionId { high: 22, low: 23 }; + fixture + .partition + .begin_split(SplitPlan { + transition_id, + parent_id: PartitionId { high: 1, low: 2 }, + parent_epoch: EPOCH, + parent_range: fixture.partition.snapshot().range, + parent_next_epoch: EPOCH + 1, + split_key: b"m".to_vec(), + child: SplitChild { + partition_id: PartitionId { high: 22, low: 25 }, + range: PartitionRange { + start: Some(b"m".to_vec()), + end: None, + }, + ownership_epoch: EPOCH + 1, + }, + }) + .await + .unwrap(); + if finalizing { + fixture + .partition + .begin_split_finalization(transition_id) + .await + .unwrap(); + } + assert!(fixture.partition.suspend_for_transfer(EPOCH).await.is_err()); + fixture.service.drain_writers().await.unwrap(); + assert_eq!( + fixture.partition.lifecycle(), + PartitionLifecycle::TransferQuiesced + ); + let name = fixture.partition.snapshot().stream_name; + assert!(fixture + .store + .record_store(7) + .unwrap() + .candidates(name) + .await + .unwrap() + .is_empty()); + fixture.service.drain_writers().await.unwrap(); + } +} + +#[tokio::test] +async fn graceful_drain_releases_an_unactivated_prepared_writer() { + let partition = prepared_partition( + PartitionId { high: 1, low: 3 }, + StreamName { high: 30, low: 32 }, + EPOCH, + ) + .await; + let service = ChunkKvService::new(test_process::TestProcess::authority(INSTANCE_ID), 4).unwrap(); + let (head, page) = catalog( + Id128 { high: 1, low: 3 }, + partition.snapshot().stream_name, + EPOCH, + 1, + None, + ); + service + .install_catalog_and_reconcile(&head, &[page], std::slice::from_ref(&partition)) + .unwrap(); + assert_eq!(partition.lifecycle(), PartitionLifecycle::Prepared); + assert!(partition.suspend_for_transfer(EPOCH).await.is_err()); + service.drain_writers().await.unwrap(); + assert_eq!(partition.lifecycle(), PartitionLifecycle::TransferQuiesced); +} + +#[tokio::test] +async fn a_published_remote_assignment_drains_the_previous_local_wal() { + let fixture = TestServingLease::start().await; + let stream = fixture.partition.snapshot().stream_name; + let (mut head, mut page) = catalog(Id128 { high: 1, low: 2 }, stream, EPOCH + 1, 2, Some(1)); + page.entries[0].owner.instance_id = INSTANCE_ID + 1; + page.entries[0].owner.rpc_endpoint = "127.0.0.1:9901".into(); + page.seal().unwrap(); + head.pages[0].page_checksum = page.checksum; + head.seal().unwrap(); + fixture + .service + .reconcile_catalog(&head, &[page], &[]) + .await + .unwrap(); + assert_eq!( + fixture.partition.lifecycle(), + PartitionLifecycle::TransferQuiesced, + "a removed writer must release ownership while its process serves other partitions" + ); + assert!(fixture.service.health(50_000).partitions.is_empty()); + let published = fixture.store.load_tail_for_tests(stream).await.unwrap().unwrap(); + assert_eq!( + published.record().logical_end, + Some(fixture.partition.snapshot().journal_durable_offset) + ); +} diff --git a/app/crowdb-chunk-kv-server/tests/server_test/split_scan.rs b/app/crowdb-chunk-kv-server/tests/server_test/split_scan.rs new file mode 100644 index 000000000..dcb6761f0 --- /dev/null +++ b/app/crowdb-chunk-kv-server/tests/server_test/split_scan.rs @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +pub(super) async fn assert_old_parent_right_interval(service: &ChunkKvService) { + for (index, direction) in [ScanDirection::Forward, ScanDirection::Reverse] + .into_iter() + .enumerate() + { + let request = ScanRequest { + routing: routing(100 + index as u64), + start: Some(b"s".to_vec()), + end: Some(b"z".to_vec()), + direction, + limit: 8, + continuation: None, + }; + let response = service.handle_scan(request.clone(), 1_500, 50_100).await; + let OperationResult::Scan { items, continuation } = response.result.unwrap() else { + panic!("expected a scan through the historical parent route") + }; + assert_eq!( + items.iter().map(|item| item.key.as_slice()).collect::>(), + [b"t"], + "a stale parent route must scan the right writer when its lower bound is past the split" + ); + assert!(continuation.is_none()); + if direction == ScanDirection::Forward { + let response = service.handle_bounded_scan(request, 1024, 1_500, 50_100).await; + let OperationResult::Scan { items, .. } = response.result.unwrap() else { + panic!("expected a bounded historical parent scan") + }; + assert_eq!( + items.iter().map(|item| item.key.as_slice()).collect::>(), + [b"t"] + ); + } + } + assert_current_assignment_bounds(service).await; +} + +async fn assert_current_assignment_bounds(service: &ChunkKvService) { + let mut request = ScanRequest { + routing: RequestRouting { + map_revision: 2, + owner_epoch: EPOCH + 1, + ..routing(102) + }, + start: Some(b"s".to_vec()), + end: Some(b"z".to_vec()), + direction: ScanDirection::Forward, + limit: 8, + continuation: None, + }; + let response = service.handle_scan(request.clone(), 1_500, 50_100).await; + assert_eq!( + response.result.unwrap_err().code, + ChunkKvRpcErrorCode::NotMyRange, + "the current left assignment must not inherit the historical parent's full range" + ); + request.routing.partition_id = Id128 { high: 22, low: 25 }; + let response = service.handle_scan(request, 1_500, 50_100).await; + let OperationResult::Scan { items, continuation } = response.result.unwrap() else { + panic!("expected the current child assignment to contain the right interval") + }; + assert_eq!( + items.iter().map(|item| item.key.as_slice()).collect::>(), + [b"t"] + ); + assert!(continuation.is_none()); +} diff --git a/app/crowdb-chunk-kv-server/tests/split_current_assignment_test.rs b/app/crowdb-chunk-kv-server/tests/split_current_assignment_test.rs index ffb58143e..17b9c8dd5 100644 --- a/app/crowdb-chunk-kv-server/tests/split_current_assignment_test.rs +++ b/app/crowdb-chunk-kv-server/tests/split_current_assignment_test.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/process.rs"] +mod test_process; + #[path = "common/split_storage.rs"] mod split_storage; @@ -72,6 +75,7 @@ fn publish(service: &ChunkKvService, generation: u64, partitions: &[Partition]) .unwrap(); let mut grant = ServingGrant { instance_id: 1, + process_epoch: 1, lease_sequence: generation, catalog_generation: generation, issued_at_ms: 1000, @@ -129,19 +133,10 @@ async fn write(partition: &Partition, sequence: u64, value: &[u8]) { async fn current_assignment_supersedes_historical_split_dispatcher() { let storage = TestSplitStorage::new(); let id = PartitionId { high: 90, low: 1 }; - let (parent, _) = storage - .partition( - id, - PartitionRange { - start: Some(Vec::new()), - end: None, - }, - 1, - Arc::new(MemoryPartitionTree::with_tree_id(90)), - ) - .await; + let parent_tree = Arc::new(MemoryPartitionTree::with_tree_id(90)); + let parent = open_parent(&storage, id, &parent_tree).await; write(&parent, 1, b"old-view").await; - let service = ChunkKvService::new(1, 8).unwrap(); + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 8).unwrap(); publish(&service, 1, std::slice::from_ref(&parent)); let plan = SplitPlan { transition_id: TransitionId { high: 90, low: 2 }, @@ -227,4 +222,58 @@ async fn current_assignment_supersedes_historical_split_dispatcher() { }; assert_eq!(value.value, b"current-view"); } + assert_failed_compatibility_writer_requires_recovery(&service, &parent, &parent_tree).await; +} + +async fn assert_failed_compatibility_writer_requires_recovery( + service: &ChunkKvService, + parent: &Partition, + parent_tree: &MemoryPartitionTree, +) { + assert!(service.recovery_required().is_empty()); + parent_tree.fail_next_apply(); + let result = parent + .mutate( + 1, + RequestId { + client_high: 1, + client_low: 1, + client_sequence: 99, + }, + MutationOperation::Put { + key: b"b".to_vec(), + value: b"uncertain".to_vec(), + }, + ) + .await; + assert_eq!(result, Err(crowdb_chunk_kv::ChunkKvError::ApplyStateUnknown)); + let failed = service.recovery_required(); + assert!( + !failed.is_empty(), + "historical dispatch writers must not hide terminal uncertainty" + ); + assert!(failed + .iter() + .any(|writer| writer.partition_id == entry(parent).partition_id)); + assert!(!service.health(50100).ready); + assert!(service.health(50100).detail.contains("durable replay")); +} + +async fn open_parent( + storage: &TestSplitStorage, + id: PartitionId, + tree: &Arc, +) -> Partition { + storage + .partition( + id, + PartitionRange { + start: Some(Vec::new()), + end: None, + }, + 1, + tree.clone(), + ) + .await + .0 } diff --git a/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs b/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs index 7c5dde832..c2734a60d 100644 --- a/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs +++ b/app/crowdb-chunk-kv-server/tests/split_recovery_test.rs @@ -1,13 +1,16 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/process.rs"] +mod test_process; + use crowdb_chunk_kv::{ memory::MemoryPartitionTree, Checkpoint, MutationOperation, Partition, PartitionConfig, PartitionId, PartitionLifecycle, PartitionRange, PartitionTree, PreparedSplitWriterArtifact, StreamPartitionJournal, }; use crowdb_chunk_kv_server::ChunkKvService; use crowdb_chunk_stream::{ - memory::MemoryStreamStore, ChunkStream, StreamBinding, StreamBindingState, StreamConfig, StreamName, + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamRecord, StreamRecordState, }; use crowdb_protocol::chunk_kv::{ ChunkKvRangeCatalogEntry, ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage, ChunkKvRangeCatalogPageRef, @@ -23,11 +26,13 @@ async fn committed_split_proves_both_recovered_writers_and_rejects_mismatches() for retained in [true, false] { let entry = entry(&transition, retained); let partition = recovered(&entry).await; - let service = ChunkKvService::new(1, 4).unwrap(); + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 4).unwrap(); install(&service, &entry, &partition); - assert!(service - .activate_recovered_partition(entry.partition_id, 2) - .is_err()); + if !retained { + assert!(service + .activate_recovered_partition(entry.partition_id, 2) + .is_err()); + } let mut uncommitted = transition.clone(); uncommitted.phase = SplitPhase::ChildPrepared; assert!(service @@ -59,7 +64,11 @@ async fn committed_split_proves_both_recovered_writers_and_rejects_mismatches() } else { wrong_artifact.child.artifact.tree_id += 1; } - wrong_artifact.validate().unwrap(); + if retained { + assert!(wrong_artifact.validate().is_err()); + } else { + wrong_artifact.validate().unwrap(); + } assert!(service .activate_recovered_split_partition(entry.partition_id, 2, &wrong_artifact) .is_err()); @@ -77,6 +86,22 @@ async fn committed_split_proves_both_recovered_writers_and_rejects_mismatches() } } +#[tokio::test] +async fn committed_split_recovers_both_writers_after_the_same_owner_changes_address() { + let transition = transition(); + for retained in [true, false] { + let mut entry = entry(&transition, retained); + entry.owner.rpc_endpoint = "192.0.2.12:7401".into(); + let partition = recovered(&entry).await; + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 4).unwrap(); + install(&service, &entry, &partition); + service + .activate_recovered_split_partition(entry.partition_id, entry.owner_epoch, &transition) + .unwrap(); + assert_eq!(partition.lifecycle(), PartitionLifecycle::Serving); + } +} + #[tokio::test] async fn materialized_split_half_recovers_while_sibling_keeps_the_transition_marker() { let transition = transition(); @@ -84,13 +109,13 @@ async fn materialized_split_half_recovers_while_sibling_keeps_the_transition_mar let mut clean = entry(&transition, retained); clean.artifact.tail_overlay = None; let partition = recovered(&clean).await; - let service = ChunkKvService::new(1, 4).unwrap(); + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 4).unwrap(); install(&service, &clean, &partition); assert_eq!(service.catalog_overlay_transition_id(clean.partition_id), None); let sibling = entry(&transition, !retained); assert_eq!( service.catalog_overlay_transition_id(sibling.partition_id), - Some(transition.transition_id), + retained.then_some(transition.transition_id), ); service .activate_recovered_partition(clean.partition_id, clean.owner_epoch) @@ -105,9 +130,9 @@ async fn materialized_split_half_recovers_while_sibling_keeps_the_transition_mar #[tokio::test] async fn materialized_catalog_replaces_unactivated_overlay_with_independent_recovery() { let transition = transition(); - let original = entry(&transition, true); + let original = entry(&transition, false); let overlay = recovered(&original).await; - let service = ChunkKvService::new(1, 4).unwrap(); + let service = ChunkKvService::new(test_process::TestProcess::authority(1), 4).unwrap(); install(&service, &original, &overlay); let mut clean = original; clean.artifact.tail_overlay = None; @@ -126,10 +151,12 @@ async fn materialized_catalog_replaces_unactivated_overlay_with_independent_reco service .reconcile_partitions(std::slice::from_ref(&page), std::slice::from_ref(&plain)) .unwrap(); - let mut sibling = entry(&transition, false); + let mut sibling = entry(&transition, true); sibling.artifact.tail_overlay = None; sibling.transition_id = None; page.entries.push(sibling); + page.entries + .sort_by(|left, right| left.range.start.cmp(&right.range.start)); page.seal().unwrap(); let mut head = ChunkKvRangeCatalogHead { generation: 2, @@ -157,7 +184,7 @@ fn transition() -> SplitTransition { source_partition_id: parent_id, source_epoch: 1, source_stream_name: StreamName { high: 1, low: 1 }, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 0, base_root_manifest_generation: 1, @@ -167,9 +194,9 @@ fn transition() -> SplitTransition { target_stream_start_seq: 2, }; let retained = PartitionArtifact { - tree_id: 2, - stream_name: StreamName { high: 1, low: 2 }, - tail_overlay: Some(overlay.clone()), + tree_id: 1, + stream_name: StreamName { high: 1, low: 1 }, + tail_overlay: None, }; let child = PartitionArtifact { tree_id: 3, @@ -220,7 +247,6 @@ fn transition() -> SplitTransition { child_applied_seq: 1, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay: Some(overlay.clone()), child_tail_overlay: overlay, }), failure: None, @@ -258,13 +284,15 @@ async fn recovered(entry: &ChunkKvRangeCatalogEntry) -> Partition { let store = Arc::new(MemoryStreamStore::new(4096)); let name = entry.artifact.stream_name; let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, stream_name: name, metadata_group_id: 1, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 2, StreamConfig::default(), @@ -314,7 +342,7 @@ async fn recovered(entry: &ChunkKvRangeCatalogEntry) -> Partition { }, parent_epoch: 1, parent_stream_name: overlay.source_stream_name, - parent_stream_manifest_generation: 1, + parent_stream_generation: 1, parent_replay_offset: 0, parent_cutover_offset: 0, applied_seq: 1, @@ -326,7 +354,7 @@ async fn recovered(entry: &ChunkKvRangeCatalogEntry) -> Partition { root_manifest_generation: 1, applied_seq: 1, stream_name: name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), @@ -358,7 +386,7 @@ async fn recover_plain( root_manifest_generation: 1, applied_seq: 1, stream_name: entry.artifact.stream_name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), diff --git a/app/crowdb-chunk-kv-server/tests/transfer_test.rs b/app/crowdb-chunk-kv-server/tests/transfer_test.rs index d8b4b80f6..4087a2293 100644 --- a/app/crowdb-chunk-kv-server/tests/transfer_test.rs +++ b/app/crowdb-chunk-kv-server/tests/transfer_test.rs @@ -21,7 +21,7 @@ fn transition() -> TransferTransition { source_partition_id: Id128 { high: 3, low: 4 }, source_epoch: 6, source_stream_name: source_artifact.stream_name, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 10, base_root_manifest_generation: 1, diff --git a/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs b/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs index 2be74865e..fc33b2666 100644 --- a/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs +++ b/app/crowdb-chunk-kv-server/tests/transition_worker_test.rs @@ -1,6 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#[path = "common/process.rs"] +mod test_process; + use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; @@ -18,8 +21,8 @@ use crowdb_chunk_kv_server::{ }; use crowdb_chunk_stream::memory::MemoryStreamStore; use crowdb_chunk_stream::{ - ChunkStream, StreamBinding, StreamBindingState, StreamChunkStore, StreamConfig, StreamMetadataStore, - StreamName, StreamRegistry, + ChunkStream, StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, StreamRecord, + StreamRecordState, StreamRegistry, }; use crowdb_protocol::chunk_kv::{ AuthorityReleaseProof, ChunkKvRangeCatalogEntry, Id128, KeyRange, OwnerDescriptor, PartitionArtifact, @@ -99,13 +102,15 @@ async fn partition( ) -> Partition { let store = Arc::new(MemoryStreamStore::new(4_096)); let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Wal, stream_name: artifact.stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, epoch, StreamConfig::default(), @@ -136,7 +141,7 @@ async fn partition( root_manifest_generation: 1, applied_seq: 0, stream_name: artifact.stream_name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), @@ -269,7 +274,7 @@ fn transfer(phase: TransferPhase) -> TransferTransition { source_partition_id: id(1), source_epoch: 3, source_stream_name: source_artifact.stream_name, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 0, base_root_manifest_generation: 1, @@ -319,7 +324,7 @@ fn split_transition() -> SplitTransition { parent_owner: owner(1), parent_epoch: 3, parent_artifact: artifact(11), - retained_parent_artifact: artifact(12), + retained_parent_artifact: artifact(11), parent_next_epoch: 4, split_key: b"m".to_vec(), child: SplitChildAssignment { @@ -362,7 +367,7 @@ fn split_artifact(transition: &SplitTransition) -> SplitArtifact { }, parent_epoch: transition.parent_epoch, parent_stream_name: StreamName { high: 5, low: 11 }, - parent_stream_manifest_generation: 1, + parent_stream_generation: 1, parent_replay_offset: 0, parent_cutover_offset: 8, applied_seq: 8, @@ -410,7 +415,7 @@ async fn source_worker_quiesces_before_returning_release_proof() { ) .await; let unused = partition(id(9), KeyRange::default(), 1, &artifact(99), true).await; - let service = Arc::new(ChunkKvService::new(1, 4).unwrap()); + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(1), 4).unwrap()); service.install_partition(&source).unwrap(); let worker = TransitionExecutor::with_storage( 1, @@ -453,7 +458,7 @@ async fn source_worker_quiesces_before_returning_release_proof() { async fn target_worker_recovers_but_does_not_activate_assignment() { let target_artifact = transfer(TransferPhase::TargetPreparing).target_artifact; let recovered = partition(id(1), KeyRange::default(), 4, &target_artifact, true).await; - let service = Arc::new(ChunkKvService::new(2, 4).unwrap()); + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(2), 4).unwrap()); let worker = TransitionExecutor::with_storage( 2, Arc::clone(&service), @@ -553,7 +558,7 @@ async fn final_target_catchup_reuses_the_live_prepared_partition() { true, ) .await; - let service = Arc::new(ChunkKvService::new(2, 4).unwrap()); + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(2), 4).unwrap()); service.install_partition(&recovered).unwrap(); let storage = Arc::new(LiveCatchupStorage { recovered, @@ -581,7 +586,7 @@ async fn split_worker_reports_only_a_common_child_frontier() { ) .await; let unused = partition(id(9), KeyRange::default(), 1, &artifact(99), true).await; - let service = Arc::new(ChunkKvService::new(1, 4).unwrap()); + let service = Arc::new(ChunkKvService::new(test_process::TestProcess::authority(1), 4).unwrap()); service.install_partition(&parent).unwrap(); let worker = TransitionExecutor::with_storage( 1, @@ -698,7 +703,7 @@ async fn service_after_local_split(first: &SplitTransition) -> (Arc u64 { @@ -270,31 +273,9 @@ impl DiskdbClientPool { /// # Errors /// Returns a String error if routing is incomplete or any commit fails. pub async fn commit_blocks(&self, segments: Vec) -> Result<(), String> { - if segments.is_empty() { - return Ok(()); - } - - let grouped = self.group_segments(segments, "commit_blocks")?; - let mut futures = Vec::with_capacity(grouped.len()); - for (dg_id, segs) in grouped { - let endpoint = self.endpoint_for_dg(dg_id).await?; - let transport = Arc::clone(&self.transport); - futures.push(async move { - let req = CommitBlocksRequest { segments: segs }; - transport - .commit_blocks(&endpoint, &req) - .await - .map_err(|e| format!("commit_blocks RPC: {e}")) - }); - } - - let results = futures::future::join_all(futures).await; - let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect(); - if errors.is_empty() { - Ok(()) - } else { - Err(errors.join("; ")) - } + self.commit_blocks_for_recovery(segments) + .await + .map_err(|error| error.to_string()) } /// Free blocks via the diskdb instances that own them. @@ -304,31 +285,9 @@ impl DiskdbClientPool { /// # Errors /// Returns a String error if routing is incomplete or any free RPC fails. pub async fn free_blocks(&self, segments: Vec) -> Result<(), String> { - if segments.is_empty() { - return Ok(()); - } - - let grouped = self.group_segments(segments, "free_blocks")?; - let mut futures = Vec::with_capacity(grouped.len()); - for (dg_id, segs) in grouped { - let endpoint = self.endpoint_for_dg(dg_id).await?; - let transport = Arc::clone(&self.transport); - futures.push(async move { - let req = FreeBlocksRequest { segments: segs }; - transport - .free_blocks(&endpoint, &req) - .await - .map_err(|e| format!("free_blocks RPC: {e}")) - }); - } - - let results = futures::future::join_all(futures).await; - let errors: Vec<_> = results.into_iter().filter_map(Result::err).collect(); - if errors.is_empty() { - Ok(()) - } else { - Err(errors.join("; ")) - } + self.free_blocks_for_recovery(segments) + .await + .map_err(|error| error.to_string()) } /// Mark proven-corrupt exact block incarnations on their DiskDB owners. diff --git a/app/crowdb-chunkdb/src/allocator/pool/recovery.rs b/app/crowdb-chunkdb/src/allocator/pool/recovery.rs new file mode 100644 index 000000000..16b1c8e6a --- /dev/null +++ b/app/crowdb-chunkdb/src/allocator/pool/recovery.rs @@ -0,0 +1,116 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Typed outcomes for replaying exact persisted block incarnations. + +use super::DiskdbClientPool; +use crowdb_diskdb_client::DiskdbClientError; +use crowdb_protocol::diskdb::rpc::{ + CommitBlocksRequest, FreeBlocksRequest, FreeFailureReason, FreeResponse, Segment, +}; +use std::collections::HashMap; + +#[derive(Debug, thiserror::Error)] +pub(crate) enum BlockMutationError { + #[error("invalid persisted segment: {0}")] + InvalidSegment(String), + #[error("block owner discovery pending: {0}")] + Routing(String), + #[error("block mutation RPC: {0}")] + Rpc(#[from] DiskdbClientError), +} + +impl BlockMutationError { + pub(crate) fn is_retryable(&self) -> bool { + matches!( + self, + Self::Routing(_) + | Self::Rpc( + DiskdbClientError::ConnectFailed(_) + | DiskdbClientError::Unreachable(_) + | DiskdbClientError::NotOwner(_) + ) + ) + } +} + +impl DiskdbClientPool { + pub(crate) async fn free_blocks_for_recovery( + &self, + segments: Vec, + ) -> Result<(), BlockMutationError> { + self.mutate_blocks(segments, false).await + } + + pub(crate) async fn commit_blocks_for_recovery( + &self, + segments: Vec, + ) -> Result<(), BlockMutationError> { + self.mutate_blocks(segments, true).await + } + + async fn mutate_blocks(&self, segments: Vec, commit: bool) -> Result<(), BlockMutationError> { + let mut grouped = HashMap::>::new(); + for segment in segments { + let disk = segment + .disk_id + .ok_or_else(|| BlockMutationError::InvalidSegment("missing disk ID".into()))?; + let group = self.dg_for_disk(&disk).ok_or_else(|| { + BlockMutationError::Routing(format!("disk {disk:?} has no disk-group mapping")) + })?; + grouped.entry(group).or_default().push(segment); + } + let mut work = Vec::with_capacity(grouped.len()); + for (group, segments) in grouped { + let endpoint = self + .endpoint_for_dg(group) + .await + .map_err(BlockMutationError::Routing)?; + work.push(async move { + let result = if commit { + self.transport + .commit_blocks(&endpoint, &CommitBlocksRequest { segments }) + .await + .map(|_| ()) + } else { + self.transport + .free_blocks(&endpoint, &FreeBlocksRequest { segments }) + .await + .and_then(|response| validate_free(&response)) + }; + self.refresh_after_owner_failure(result) + .await + .map_err(BlockMutationError::Rpc) + }); + } + // Await every submitted mutation before inspecting outcomes. A terminal + // error must not be hidden by an earlier transient owner failure. + let mut pending = None; + for result in futures::future::join_all(work).await { + if let Err(error) = result { + if !error.is_retryable() { + return Err(error); + } + pending = Some(error); + } + } + pending.map_or(Ok(()), Err) + } +} + +fn validate_free(response: &FreeResponse) -> Result<(), DiskdbClientError> { + // A replay may find an already freed or subsequently reused incarnation. + // Neither permits freeing the current owner, and neither needs more work. + if response.failures.iter().any(|failure| { + !matches!( + failure.reason, + FreeFailureReason::NotBusy | FreeFailureReason::IncarnationMismatch + ) + }) { + return Err(DiskdbClientError::Unreachable(format!( + "block cleanup remains pending: {:?}", + response.failures + ))); + } + Ok(()) +} diff --git a/app/crowdb-chunkdb/src/chunkdb_config.rs b/app/crowdb-chunkdb/src/chunkdb_config.rs index 3f6b390bb..e00cf64fa 100644 --- a/app/crowdb-chunkdb/src/chunkdb_config.rs +++ b/app/crowdb-chunkdb/src/chunkdb_config.rs @@ -24,7 +24,7 @@ pub enum PlacementMode { pub enum DeploymentMode { #[default] Production, - TestSingleNode, + Local, /// Legacy colocated EC fixtures; rejected by release builds. TestUnsafePlacement, } @@ -133,15 +133,15 @@ impl BaseConfig for ChunkdbConfig { return Err("production deployment requires protected placement".into()); } } - DeploymentMode::TestSingleNode => { + DeploymentMode::Local => { if self.deployment.max_node_failures != 0 { - return Err("test_single_node requires max_node_failures = 0".into()); + return Err("local requires max_node_failures = 0".into()); } if self.placement.mode != PlacementMode::UnsafeColocated { - return Err("test_single_node requires explicit unsafe_colocated placement".into()); + return Err("local requires explicit unsafe_colocated placement".into()); } if self.conversion.enabled { - return Err("test_single_node must disable mirror-to-EC conversion".into()); + return Err("local must disable mirror-to-EC conversion".into()); } } DeploymentMode::TestUnsafePlacement => { @@ -341,6 +341,7 @@ impl RepairConfig { /// Background mirror-to-EC conversion policy. #[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] pub struct ConversionConfig { /// Run automatic conversion scans. Manual triggers remain available. pub enabled: bool, @@ -407,8 +408,7 @@ pub struct ServerConfig { #[serde(default = "default_rpc_workers")] pub rpc_workers: u32, pub http_listen_addr: String, - /// crowdb-rpc listen address (R116 migration — runs alongside the - /// HTTP listener). + /// crowdb-rpc listen address, separate from the HTTP listener. #[serde(default = "default_rpc_listen_addr")] pub rpc_listen_addr: String, pub instance_id: Option, diff --git a/app/crowdb-chunkdb/src/conversion/io.rs b/app/crowdb-chunkdb/src/conversion/io.rs index 57dd6eb71..90f356e6a 100644 --- a/app/crowdb-chunkdb/src/conversion/io.rs +++ b/app/crowdb-chunkdb/src/conversion/io.rs @@ -106,13 +106,6 @@ impl ConversionDiskIo { offset: u64, length: u32, ) -> Result { - #[cfg(feature = "test-util")] - if self.client.load().is_none() { - return Ok(Bytes::from(vec![ - 0; - usize::try_from(length).expect("u32 fits usize") - ])); - } let target = target(segment, unit_bytes)?; let client = self.client()?; client @@ -141,6 +134,24 @@ impl ConversionDiskIo { .map_err(|error| ConversionIoError::Io(error.to_string())) } + /// Prove completion and synchronization of earlier writes in this segment. + /// Owner exclusion and late-request age must be established by the caller. + pub async fn drain_segment( + &self, + segment: &Segment, + unit_bytes: u64, + deadline: std::time::Instant, + ) -> Result<(), ConversionIoError> { + let target = target(segment, unit_bytes)?; + let client = self.client()?; + let mut options = client.normal_options().priority(); + options.deadline = deadline; + client + .drain_segment(target, options) + .await + .map_err(|error| ConversionIoError::Io(error.to_string())) + } + pub async fn fsync_segment(&self, segment: &Segment) -> Result<(), ConversionIoError> { let disk_id = disk_id(segment)?; let client = self.client()?; diff --git a/app/crowdb-chunkdb/src/finalize.rs b/app/crowdb-chunkdb/src/finalize.rs index 75a3955a3..3d5417e36 100644 --- a/app/crowdb-chunkdb/src/finalize.rs +++ b/app/crowdb-chunkdb/src/finalize.rs @@ -7,19 +7,17 @@ use std::sync::Arc; use std::time::{SystemTime, UNIX_EPOCH}; use crowdb_protocol::chunk_task::{ChunkTaskValue, FINALIZE_CHUNK_KIND_VERSION, TASK_KIND_FINALIZE_CHUNK}; -use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkStrip, Strip}; -use crowdb_protocol::common::ChunkId; -use crowdb_protocol::frame::{frame_length, parse_frame, parse_header}; +use crowdb_protocol::chunkdb::rpc::ChunkState; use crate::conversion::io::ConversionDiskIo; use crate::lifecycle::LifecycleHandler; use crate::task::executor::TaskFuture; use crate::task::{TaskHandler, TaskOutcome}; -/// Executes the durable liveness task. Empty chunks have no possible frame -/// boundary and can be reclaimed immediately. Nonempty chunks are retained -/// until the DiskIO-backed frame scanner is available to derive their exact -/// sealed boundary. +mod scan; + +/// Executes durable liveness finalization after old requests have aged out. +/// Grouped tails are recovered from complete content, independently of checkpoints. pub struct FinalizeChunkTaskHandler { lifecycle: Arc, io: Arc, @@ -42,206 +40,93 @@ impl TaskHandler for FinalizeChunkTaskHandler { } fn execute<'a>(&'a self, task: &'a ChunkTaskValue) -> TaskFuture<'a> { - Box::pin(async move { - let safe_at = task - .eligible_at_ms - .saturating_add(crowdb_protocol::timing::DEFAULT_MAX_WRITE_REQUEST_AGE_MS) - .saturating_add(crowdb_protocol::timing::DEFAULT_MAX_CLOCK_SKEW_MS) - .saturating_add(crowdb_protocol::timing::DEFAULT_FINALIZER_SCANNER_MARGIN_MS); - let now_ms = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_or(0, |duration| { - u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) - }); - if now_ms < safe_at { - return TaskOutcome::Retry { - delay_ms: safe_at.saturating_sub(now_ms), - error_code: 0, - error: "waiting for pre-expiry writes to age out".into(), - }; - } - let chunk_id = task.partition_id; - let chunk = match self.lifecycle.query_chunk(&chunk_id).await { - Ok(chunk) => chunk, - Err(error) => { - return TaskOutcome::Retry { - delay_ms: 1_000, - error_code: 1, - error: error.to_string(), - }; - } - }; - if chunk.state == ChunkState::Deleted as i32 || chunk.state == ChunkState::Sealed as i32 { - return TaskOutcome::Complete; - } - let boundary = match scan_complete_frame_boundary(&self.io, &chunk, chunk_id).await { - Ok(boundary) => boundary, - Err(error) => { - return TaskOutcome::Retry { - delay_ms: 1_000, - error_code: 3, - error, - }; - } - }; - if boundary == 0 { - return match self.lifecycle.delete_chunk(&chunk_id).await { - Ok(_) => TaskOutcome::Complete, - Err(error) => TaskOutcome::Retry { - delay_ms: 1_000, - error_code: 2, - error: error.to_string(), - }, - }; - } - match self.lifecycle.seal_chunk_bytes(&chunk_id, boundary).await { - Ok(_) => TaskOutcome::Complete, - Err(error) => TaskOutcome::Retry { - delay_ms: 1_000, - error_code: 4, - error: error.to_string(), - }, - } - }) + Box::pin(self.finalize(task)) } } -async fn scan_complete_frame_boundary( - io: &ConversionDiskIo, - chunk: &Chunk, - chunk_id: ChunkId, -) -> Result { - let mut cursor = 0_u64; - loop { - let header = match io - .read_chunk_range(chunk, cursor, crowdb_protocol::frame::FRAME_HEADER_PREFIX_BYTES) - .await - { - Ok(header) => header, - Err(error) => return Err(error.clone()), +impl FinalizeChunkTaskHandler { + async fn finalize(&self, task: &ChunkTaskValue) -> TaskOutcome { + if let Some(waiting) = wait_for_old_requests(task) { + return waiting; + } + let chunk_id = task.partition_id; + let chunk = match self.lifecycle.query_chunk(&chunk_id).await { + Ok(chunk) => chunk, + Err(error) => return retry(1, error.to_string()), }; - let Ok(header) = parse_header(&header) else { - return Ok(cursor); + if chunk.state == ChunkState::Deleted as i32 { + return self.finish_deletion(&chunk_id).await; + } + if chunk.writer_epoch != task.source_revision { + return retry(6, "finalizer owner generation differs from chunk writer epoch"); + } + if chunk.state == ChunkState::Sealed as i32 { + return if chunk.acknowledged_cursor == 0 { + self.finish_empty_chunk(&chunk).await + } else { + TaskOutcome::Complete + }; + } + let boundary = match scan::complete_boundary(Arc::clone(&self.io), Arc::new(chunk.clone())).await { + Ok(boundary) => boundary, + Err(error) => return retry(3, error), }; - let Ok(length) = frame_length(header) else { - return Ok(cursor); + if boundary < chunk.acknowledged_cursor { + return retry(5, format!( + "chunk {chunk_id:?}: scanned frame boundary {boundary} is below acknowledged cursor {}; retaining data and layout; inspect frame integrity and DiskIO reads", + chunk.acknowledged_cursor + )); + } + let sealed = match self.lifecycle.seal_observed_chunk(&chunk, boundary).await { + Ok(sealed) => sealed, + Err(error) => return retry(4, error.to_string()), }; - let frame_end = cursor.saturating_add(u64::try_from(length).expect("frame length fits u64")); - let frame = io.read_chunk_range(chunk, cursor, length).await?; - if parse_frame(&frame, chunk_id).is_err() { - return Ok(cursor); + if boundary == 0 { + return self.finish_empty_chunk(&sealed).await; } - cursor = frame_end; + TaskOutcome::Complete } -} -trait FinalizerRead { - fn read_chunk_range<'a>( - &'a self, - chunk: &'a Chunk, - offset: u64, - length: usize, - ) -> std::pin::Pin, String>> + Send + 'a>>; -} + async fn finish_empty_chunk(&self, chunk: &crowdb_protocol::chunkdb::rpc::Chunk) -> TaskOutcome { + if chunk.capacity != 0 || !chunk.strips.is_empty() || !chunk.cleanup_intents.is_empty() { + return retry(4, "waiting for empty sealed chunk physical cleanup"); + } + let Some(id) = chunk.id else { + return retry(4, "sealed chunk has no identity"); + }; + self.finish_deletion(&id).await + } -impl FinalizerRead for ConversionDiskIo { - fn read_chunk_range<'a>( - &'a self, - chunk: &'a Chunk, - offset: u64, - length: usize, - ) -> std::pin::Pin, String>> + Send + 'a>> { - Box::pin(async move { - let end = offset - .checked_add(u64::try_from(length).map_err(|_| "frame read length overflows")?) - .ok_or("frame read range overflows")?; - let mut cursor = offset; - let mut bytes = Vec::with_capacity(length); - while cursor < end { - let strip = chunk - .strips - .iter() - .find(|strip| strip_contains(strip, cursor)) - .ok_or("chunk layout ends before frame")?; - let strip_start = u64::from(strip.chunk_offset) * 1024; - let strip_end = strip_start.saturating_add(u64::from(strip.capacity) * 1024); - let take = usize::try_from((end.min(strip_end)).saturating_sub(cursor)) - .map_err(|_| "frame part exceeds addressable memory")?; - bytes.extend(read_strip_data(self, strip, cursor - strip_start, take).await?); - cursor = cursor.saturating_add(u64::try_from(take).expect("usize fits u64")); - } - Ok(bytes) - }) + async fn finish_deletion(&self, id: &crowdb_protocol::common::ChunkId) -> TaskOutcome { + match self.lifecycle.delete_chunk(id).await { + Ok(_) => TaskOutcome::Complete, + Err(error) => retry(2, error.to_string()), + } } } -fn strip_contains(strip: &ChunkStrip, offset: u64) -> bool { - let start = u64::from(strip.chunk_offset) * 1024; - let end = start.saturating_add(u64::from(strip.capacity) * 1024); - start <= offset && offset < end +fn retry(error_code: u16, error: impl Into) -> TaskOutcome { + TaskOutcome::Retry { + delay_ms: 1_000, + error_code, + error: error.into(), + } } -async fn read_strip_data( - io: &ConversionDiskIo, - strip: &ChunkStrip, - offset: u64, - length: usize, -) -> Result, String> { - let unit_bytes = u64::from(strip.unit_kb) * 1024; - match strip.strip.as_ref() { - Some(Strip::MirrorStrip(mirror)) => { - let segment = mirror - .segments - .iter() - .find(|segment| !strip.unavailable_segments.contains(segment)) - .ok_or_else(|| format!("all mirrors unavailable for strip {}", strip.strip_sequence))?; - io.read_segment_range( - segment, - unit_bytes, - offset, - u32::try_from(length).map_err(|_| "frame range exceeds RPC size")?, - ) - .await - .map(|bytes| bytes.to_vec()) - .map_err(|error| error.to_string()) - } - Some(Strip::EcStrip(ec)) => { - let data_num = usize::try_from(ec.data_num).map_err(|_| "EC data count overflows")?; - if data_num == 0 || ec.segments.len() < data_num { - return Err(format!("invalid EC strip {}", strip.strip_sequence)); - } - let shard_bytes = u64::from(ec.segments[0].unit_count).saturating_mul(unit_bytes); - if shard_bytes == 0 { - return Err(format!("empty EC shard {}", strip.strip_sequence)); - } - let mut cursor = offset; - let end = offset.saturating_add(u64::try_from(length).expect("usize fits u64")); - let mut bytes = Vec::with_capacity(length); - while cursor < end { - let shard = usize::try_from(cursor / shard_bytes).map_err(|_| "EC shard index overflows")?; - if shard >= data_num { - return Err(format!("EC data range exceeds strip {}", strip.strip_sequence)); - } - let segment = &ec.segments[shard]; - if strip.unavailable_segments.contains(segment) { - return Err(format!("EC data shard {shard} is unavailable")); - } - let local = cursor % shard_bytes; - let take = (end - cursor).min(shard_bytes - local); - bytes.extend( - io.read_segment_range( - segment, - unit_bytes, - local, - u32::try_from(take).map_err(|_| "EC frame range exceeds RPC size")?, - ) - .await - .map_err(|error| error.to_string())?, - ); - cursor = cursor.saturating_add(take); - } - Ok(bytes) - } - None => Err(format!("strip {} has no physical layout", strip.strip_sequence)), - } +fn wait_for_old_requests(task: &ChunkTaskValue) -> Option { + let safe_at = task + .eligible_at_ms + .saturating_add(crowdb_protocol::timing::MAX_WRITE_REQUEST_AGE_MS) + .saturating_add(crowdb_protocol::timing::MAX_WRITE_CLOCK_SKEW_MS) + .saturating_add(crowdb_protocol::timing::DEFAULT_FINALIZER_SCANNER_MARGIN_MS); + let now_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) + }); + (now_ms < safe_at).then(|| TaskOutcome::Retry { + delay_ms: safe_at.saturating_sub(now_ms), + error_code: 0, + error: "waiting for pre-expiry writes to age out".into(), + }) } diff --git a/app/crowdb-chunkdb/src/finalize/scan.rs b/app/crowdb-chunkdb/src/finalize/scan.rs new file mode 100644 index 000000000..b83d8c1de --- /dev/null +++ b/app/crowdb-chunkdb/src/finalize/scan.rs @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Content authority for abandoned grouped chunks; checkpoint authority for objects. + +use std::sync::Arc; +use std::time::Duration; + +use crowdb_chunk_client::{select_frame_prefix, FrameWindowReader, MAX_FRAME_OBSERVATION_REPLICAS}; +use crowdb_protocol::chunk_stream::chunk_owner_key_matches_type; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkType, Strip}; +use crowdb_protocol::frame::{FrameScanConfig, FrameScanMode, MAX_FRAME_SCAN_INPUT_BYTES}; +use tokio::time::Instant; + +use crate::conversion::io::ConversionDiskIo; + +mod drain; +mod read; +use read::FinalizerWindowReader; + +pub(super) async fn complete_boundary(io: Arc, chunk: Arc) -> Result { + let chunk_id = chunk.id.ok_or("finalizer chunk has no identity")?; + let purpose = ChunkType::try_from(chunk.chunk_type).map_err(|()| "invalid finalizer chunk type")?; + if chunk_id.high >> 56 != purpose as u64 || !chunk_owner_key_matches_type(purpose, &chunk.owner_key) { + return Err("finalizer chunk identity or owner does not match purpose".into()); + } + let grouped = [ChunkType::Stream as i32, ChunkType::Wal as i32].contains(&chunk.chunk_type); + let (capacity, replicas) = layout_bounds(&chunk)?; + let started = Instant::now(); + let deadline = started + Duration::from_secs(30); + drain::drain_layout(&io, &chunk, started + Duration::from_secs(15)) + .await + .map_err(|error| { + format!( + "chunk {chunk_id:?}: cannot drain prior writes before observing acknowledged cursor {}: {error}", + chunk.acknowledged_cursor + ) + })?; + // Ordinary object publication already persists its exact physical cursor. + // Only grouped streams have acknowledged data newer than that checkpoint. + if !grouped && chunk.acknowledged_cursor == 0 { + return Ok(0); + } + let config = FrameScanConfig { + chunk_id, + mode: if grouped { + FrameScanMode::Grouped + } else { + FrameScanMode::Ordinary + }, + physical_start: 0, + physical_limit: if grouped { + capacity + } else { + chunk.acknowledged_cursor + }, + required_end: chunk.acknowledged_cursor, + logical_start: 0, + max_append_bytes: capacity, + }; + let readers: Vec> = (0..replicas) + .map(|replica| { + Arc::new(FinalizerWindowReader::new( + Arc::clone(&io), + Arc::clone(&chunk), + replica, + )) as _ + }) + .collect(); + let window = u32::try_from(MAX_FRAME_SCAN_INPUT_BYTES).expect("bounded observation window"); + select_frame_prefix(&readers, config, window, deadline) + .await + .map(|selected| selected.observation().summary().physical_end) + .map_err(|error| { + format!( + "chunk {chunk_id:?}: cannot prove complete prefix covering acknowledged cursor {}: {error}", + chunk.acknowledged_cursor + ) + }) +} + +fn layout_bounds(chunk: &Chunk) -> Result<(u64, usize), String> { + let mut end = 0; + let mut replicas = 1; + for strip in &chunk.strips { + let start = u64::from(strip.chunk_offset) * 1024; + let length = u64::from(strip.capacity) * 1024; + if start != end || length == 0 || strip.unit_kb == 0 { + return Err("finalizer layout has gaps, overlaps or empty strips".into()); + } + let segments = match &strip.strip { + Some(Strip::MirrorStrip(mirror)) => { + if mirror.segments.is_empty() || mirror.segments.len() > MAX_FRAME_OBSERVATION_REPLICAS { + return Err("finalizer mirror count is invalid".into()); + } + replicas = replicas.max(mirror.segments.len()); + &mirror.segments + } + Some(Strip::EcStrip(ec)) if ec.data_num > 0 && ec.segments.len() >= ec.data_num as usize => { + &ec.segments + } + _ => return Err("finalizer strip has no valid physical layout".into()), + }; + for segment in segments { + if segment.disk_id.is_none() || segment.owner_chunk != chunk.id || segment.allocation_ts == 0 { + return Err("finalizer segment has invalid allocation identity".into()); + } + } + end = start.checked_add(length).ok_or("finalizer layout overflows")?; + } + if end == 0 || chunk.acknowledged_cursor > end || end != u64::from(chunk.capacity) * 1024 { + return Err("finalizer checkpoint or capacity exceeds physical layout".into()); + } + Ok((end, replicas)) +} diff --git a/app/crowdb-chunkdb/src/finalize/scan/drain.rs b/app/crowdb-chunkdb/src/finalize/scan/drain.rs new file mode 100644 index 000000000..a2c23068a --- /dev/null +++ b/app/crowdb-chunkdb/src/finalize/scan/drain.rs @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Ordered completion of old physical writes before an abandoned-tail scan. + +use crowdb_protocol::chunkdb::rpc::{Chunk, Strip}; +use futures::{stream, StreamExt, TryStreamExt}; +use tokio::time::{timeout_at, Instant}; + +use crate::conversion::io::ConversionDiskIo; + +pub(super) async fn drain_layout( + io: &ConversionDiskIo, + chunk: &Chunk, + deadline: Instant, +) -> Result<(), String> { + let targets = chunk.strips.iter().flat_map(|strip| { + let segments = match &strip.strip { + Some(Strip::MirrorStrip(mirror)) => mirror.segments.as_slice(), + Some(Strip::EcStrip(ec)) => ec.segments.as_slice(), + None => &[], + }; + segments + .iter() + .copied() + .map(move |segment| (segment, u64::from(strip.unit_kb) * 1024)) + }); + let drain = + stream::iter(targets) + .map(Ok) + .try_for_each_concurrent(8, |(segment, unit_bytes)| async move { + io.drain_segment(&segment, unit_bytes, deadline.into_std()) + .await + .map_err(|error| format!("cannot drain physical segment before finalization: {error}")) + }); + timeout_at(deadline, drain) + .await + .map_err(|_| "physical segment drain deadline expired".to_string())? +} diff --git a/app/crowdb-chunkdb/src/finalize/scan/read.rs b/app/crowdb-chunkdb/src/finalize/scan/read.rs new file mode 100644 index 000000000..6ac38aa4d --- /dev/null +++ b/app/crowdb-chunkdb/src/finalize/scan/read.rs @@ -0,0 +1,122 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::{Bytes, BytesMut}; +use crowdb_chunk_client::{FrameWindowReader, IoError, Result}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkStrip, Strip}; +use crowdb_protocol::diskdb::rpc::Segment; +use crowdb_protocol::frame::MAX_FRAME_SCAN_INPUT_BYTES; + +use crate::conversion::io::ConversionDiskIo; + +pub(super) struct FinalizerWindowReader { + io: Arc, + chunk: Arc, + replica: usize, +} + +impl FinalizerWindowReader { + pub(super) fn new(io: Arc, chunk: Arc, replica: usize) -> Self { + Self { io, chunk, replica } + } + + async fn read_part(&self, strip: &ChunkStrip, offset: u64, length: u32) -> Result { + let unit_bytes = u64::from(strip.unit_kb) * 1024; + let (segment, local, count) = match &strip.strip { + Some(Strip::MirrorStrip(mirror)) => ( + mirror + .segments + .get(self.replica) + .ok_or_else(|| failed("mirror replica absent"))?, + offset, + length, + ), + Some(Strip::EcStrip(ec)) => { + let shard_bytes = ec + .segments + .first() + .map_or(0, |segment| u64::from(segment.unit_count) * unit_bytes); + if shard_bytes == 0 { + return Err(failed("empty EC shard")); + } + let shard = usize::try_from(offset / shard_bytes).map_err(|_| failed("EC shard overflow"))?; + if shard >= ec.data_num as usize { + return Err(failed("EC range exceeds data shards")); + } + let local = offset % shard_bytes; + ( + &ec.segments[shard], + local, + u32::try_from(u64::from(length).min(shard_bytes - local)).expect("bounded window"), + ) + } + None => return Err(failed("missing physical layout")), + }; + self.read_segment(strip, segment, unit_bytes, local, count).await + } + + async fn read_segment( + &self, + strip: &ChunkStrip, + segment: &Segment, + unit_bytes: u64, + offset: u64, + count: u32, + ) -> Result { + if strip.unavailable_segments.contains(segment) { + return Err(failed("physical segment is unavailable")); + } + let bytes = self + .io + .read_segment_range(segment, unit_bytes, offset, count) + .await + .map_err(|error| failed(&error.to_string()))?; + if bytes.len() != count as usize { + return Err(failed("short physical segment read")); + } + Ok(bytes) + } +} + +#[async_trait] +impl FrameWindowReader for FinalizerWindowReader { + async fn read_window(&self, offset: u64, length: u32) -> Result { + let end = offset + .checked_add(u64::from(length)) + .ok_or_else(|| failed("window overflows"))?; + if length == 0 + || length as usize > MAX_FRAME_SCAN_INPUT_BYTES + || end > u64::from(self.chunk.capacity) * 1024 + { + return Err(failed("physical window exceeds layout or budget")); + } + let mut output = BytesMut::with_capacity(length as usize); + let mut cursor = offset; + while cursor < end { + let index = self + .chunk + .strips + .partition_point(|strip| u64::from(strip.chunk_offset) * 1024 <= cursor) + .checked_sub(1) + .ok_or_else(|| failed("window precedes layout"))?; + let strip = &self.chunk.strips[index]; + let start = u64::from(strip.chunk_offset) * 1024; + let strip_end = start + u64::from(strip.capacity) * 1024; + if cursor >= strip_end { + return Err(failed("window has a layout gap")); + } + let count = u32::try_from((end.min(strip_end)) - cursor).expect("bounded window"); + let bytes = self.read_part(strip, cursor - start, count).await?; + cursor += bytes.len() as u64; + output.extend_from_slice(&bytes); + } + Ok(output.freeze()) + } +} + +fn failed(message: &str) -> IoError { + IoError::ReadFailed(message.into()) +} diff --git a/app/crowdb-chunkdb/src/lifecycle.rs b/app/crowdb-chunkdb/src/lifecycle.rs index 3c0c0a9c0..6f8b63ed8 100644 --- a/app/crowdb-chunkdb/src/lifecycle.rs +++ b/app/crowdb-chunkdb/src/lifecycle.rs @@ -4,6 +4,7 @@ //! Chunk lifecycle state, serialization, and orchestration. mod handler; +mod replacement; pub mod state; pub use handler::{ diff --git a/app/crowdb-chunkdb/src/lifecycle/handler.rs b/app/crowdb-chunkdb/src/lifecycle/handler.rs index 06bcfd108..8afeea280 100644 --- a/app/crowdb-chunkdb/src/lifecycle/handler.rs +++ b/app/crowdb-chunkdb/src/lifecycle/handler.rs @@ -15,7 +15,6 @@ use quick_cache::sync::Cache; use tokio::sync::{Mutex, OwnedMutexGuard}; use tracing::{info, warn}; -use crowdb_protocol::chunk_stream::chunk_owner_key_matches_type; use crowdb_protocol::chunk_task::{ ChunkTaskState, ChunkTaskValue, CHUNK_TASK_SCHEMA_VERSION, FINALIZE_CHUNK_KIND_VERSION, TASK_KIND_FINALIZE_CHUNK, @@ -27,8 +26,7 @@ use crowdb_protocol::chunkdb::rpc::{ use crowdb_protocol::common::ChunkId; use crowdb_protocol::common::DiskId; use crowdb_protocol::diskdb::rpc::Segment; -use crowdb_protocol::generate_chunk_id; -use crowdb_protocol::timing::{DEFAULT_MAX_CLOCK_SKEW_MS, DEFAULT_SELF_FENCE_MARGIN_MS}; +use crowdb_protocol::timing::{DEFAULT_SELF_FENCE_MARGIN_MS, MAX_WRITE_CLOCK_SKEW_MS}; use crate::allocator::{AllocError, ChunkAllocator, StripAllocType, StripBatchSpec}; use crate::metrics::ChunkdbMetrics; @@ -40,6 +38,9 @@ use crate::storage::{ChunkStore, StoreError}; use crate::task::TaskStore; use crate::topology::TopologyCache; +mod allocation; + +use super::replacement::validate_replacement_geometry; use super::state::{ChunkState, StateTransitionError}; /// Default lock wait time for `LockPolicy::default()`. @@ -48,7 +49,7 @@ const DEFAULT_LAYOUT_VALIDITY_MS: u64 = 30_000; const FINALIZE_CHUNK_LIVENESS_MS: u64 = 15 * 60 * 1_000; const FINALIZE_CHUNK_RENEWAL_MS: u64 = 12 * 60 * 1_000; const FINALIZE_CHUNK_SELF_FENCE_MS: u64 = - FINALIZE_CHUNK_LIVENESS_MS - DEFAULT_MAX_CLOCK_SKEW_MS - DEFAULT_SELF_FENCE_MARGIN_MS; + FINALIZE_CHUNK_LIVENESS_MS - MAX_WRITE_CLOCK_SKEW_MS - DEFAULT_SELF_FENCE_MARGIN_MS; /// Lifecycle error — maps to crowdb-rpc status codes in the service layer. #[derive(Debug, thiserror::Error)] @@ -79,6 +80,8 @@ pub enum LifecycleError { Commit(String), #[error("diskdb cleanup failed after metadata publication: {0}")] Cleanup(String), + #[error("startup recovery is waiting for a dependency: {0}")] + RecoveryDependency(String), #[error("cluster reservation capacity is exhausted")] ReservationLimit, } @@ -128,6 +131,7 @@ mod admission; mod append; #[path = "lock_map.rs"] mod lock_map; +mod recovery; mod reservation; mod seal; @@ -273,15 +277,15 @@ impl LifecycleHandler { )); } match self.deployment_mode { - Some(DeploymentMode::TestSingleNode) => { + Some(DeploymentMode::Local) => { if strip_type != ProtoStripType::Mirror - || copy_count != 1 + || !matches!(copy_count, 1 | 2) || data_num != 0 || code_num != 0 || capacity_kb != 1024 { return Err(LifecycleError::InvalidRequest( - "test_single_node requires 1 MiB mirror strips with one copy each".into(), + "local requires 1 MiB mirror strips with one or two copies".into(), )); } } @@ -408,201 +412,6 @@ impl LifecycleHandler { Ok(()) } - /// Allocate a new chunk. - #[allow(clippy::too_many_arguments)] - pub async fn allocate_chunk( - &self, - chunk_id: Option, - write_granularity_kb: u32, - strip_count: u32, - strip_type: ProtoStripType, - data_num: u32, - code_num: u32, - copy_count: u32, - chunk_type: ChunkType, - writer_epoch: u64, - writer_lease_ms: u64, - ) -> Result { - self.allocate_chunk_owned( - chunk_id, - write_granularity_kb, - strip_count, - strip_type, - data_num, - code_num, - copy_count, - chunk_type, - writer_epoch, - writer_lease_ms, - Vec::new(), - ) - .await - } - - /// Allocate a new chunk with a stable logical owner identity. - #[allow(clippy::too_many_arguments)] - #[allow(clippy::too_many_lines)] - pub async fn allocate_chunk_owned( - &self, - chunk_id: Option, - write_granularity_kb: u32, - strip_count: u32, - strip_type: ProtoStripType, - data_num: u32, - code_num: u32, - copy_count: u32, - chunk_type: ChunkType, - writer_epoch: u64, - writer_lease_ms: u64, - owner_key: Vec, - ) -> Result { - self.validate_strip_layout(strip_type, data_num, code_num, copy_count, write_granularity_kb)?; - if !chunk_owner_key_matches_type(chunk_type, &owner_key) { - return Err(LifecycleError::InvalidRequest( - "chunk owner key does not match chunk type".into(), - )); - } - let id = match chunk_id { - Some(id) => id, - None => self.generate_owned_chunk_id(chunk_type)?, - }; - if (id.high >> 56) != u64::from(chunk_type as u8) { - return Err(LifecycleError::InvalidRequest( - "chunk id prefix does not match chunk type".into(), - )); - } - self.check_range(&id)?; - let mut allocation_guard = AllocationMetricGuard::new(self.metrics.clone()); - - // Caller-supplied ID: acquire lock + existence check. - // Auto-generated ID: skip lock (UUID collision negligible). - let mut guard = if chunk_id.is_some() { - if let Some(locks) = &self.locks { - let g = locks - .acquire_for_create(&id, &LockPolicy::default(), CacheHint::Cache) - .await?; - // Existence check inside the lock. - match self.store.get_chunk(&id).await { - Ok(_) => return Err(LifecycleError::ChunkAlreadyExists), - Err(StoreError::ChunkNotFound) => {} - Err(e) => return Err(LifecycleError::Storage(e)), - } - Some(g) - } else { - None - } - } else { - None - }; - - let snap = self.topology.snapshot(); - - let mirror_copies = if copy_count == 0 { 2 } else { copy_count as usize }; - let strip_alloc_type = - self.protected_degraded_layout(strip_type, &snap) - .unwrap_or(match strip_type { - ProtoStripType::Mirror => StripAllocType::Mirror { - copy_count: mirror_copies, - }, - ProtoStripType::Ec => StripAllocType::Ec { - data_num: data_num as usize, - code_num: code_num as usize, - }, - }); - - let constraints = self.allocation_constraints(strip_type, &snap); - // Convert write_granularity (KB) to unit_count using the unit - // size from the topology snapshot. Fall back to treating KB as - // units if unit_size_bytes is unavailable (0). - let unit_size_kb = snap.unit_size_bytes() / 1024; - let unit_count = write_granularity_kb - .checked_div(unit_size_kb) - .unwrap_or(write_granularity_kb) - .max(1); - - let mut strips = self - .allocator - .allocate_strips( - &snap, - &id, - StripBatchSpec { - strip_type: strip_alloc_type, - unit_count, - start_sequence: 0, - strip_count, - }, - &constraints, - ) - .await?; - assign_strip_offsets(&mut strips, 0); - - let now_ms = unix_time_ms(); - if writer_epoch != 0 && writer_lease_ms == 0 { - self.allocator.rollback_strips(&strips).await?; - return Err(LifecycleError::InvalidRequest( - "writer_lease_ms must be nonzero for a shared writer".into(), - )); - } - - let chunk = Chunk { - id: Some(id), - modify_ts: 1, - state: ProtoChunkState::Active as i32, - create_ts_ms: now_ms, - sealed_ts_ms: 0, - capacity: strips.iter().map(|s| s.capacity).sum(), - sealed_length: 0, - strips, - chunk_type: chunk_type as i32, - writer_epoch, - acknowledged_cursor: 0, - closed_strip_sequence: None, - writer_lease_deadline_ms: writer_lease_deadline(now_ms, writer_epoch, writer_lease_ms), - next_strip_sequence: strip_count, - cleanup_intents: Vec::new(), - last_strip_replacement: None, - owner_key, - }; - self.create_chunk_with_liveness_task(&chunk).await?; - self.liveness_renewed_at.insert(id, now_ms); - self.admit_placement_repairs(&chunk); - self.commit_strip_segments_background(chunk.strips.clone()); - - // Update cache. - if let Some(ref mut g) = guard { - g.refresh(chunk.clone()); - } else if chunk_id.is_none() { - // Auto-generated ID: populate cache directly (no guard). - if let Some(locks) = &self.locks { - locks.populate_cache(&id, chunk.clone()); - } - } - info!(chunk_id = ?id, strips = strip_count, "chunk allocated"); - allocation_guard.mark_success(); - Ok(chunk) - } - - fn generate_owned_chunk_id(&self, chunk_type: ChunkType) -> Result { - let Some(range_guard) = &self.range_guard else { - return Ok(generate_chunk_id(chunk_type as u8).to_proto()); - }; - if !range_guard.is_ready() { - let candidate = generate_chunk_id(chunk_type as u8).to_proto(); - return self.check_range(&candidate).map(|()| candidate); - } - let attempts = 32 * u64::from(crowdb_protocol::chunk_slot::CHUNK_SLOT_COUNT) - / range_guard.owned_bucket_count().max(1); - for _ in 0..attempts { - let candidate = generate_chunk_id(chunk_type as u8).to_proto(); - if range_guard.check(&candidate).is_ok() { - return Ok(candidate); - } - } - Err(LifecycleError::InvalidRequest( - "failed to generate a chunk id in the owned range".into(), - )) - } - async fn create_chunk_with_liveness_task(&self, chunk: &Chunk) -> Result<(), LifecycleError> { let id = chunk .id @@ -898,36 +707,6 @@ impl LifecycleHandler { Ok(chunk) } - /// Compatibility wrapper for a fenced one-strip range replacement. - pub async fn update_chunk_strip( - &self, - chunk_id: &ChunkId, - strip_index: u32, - new_strip: ChunkStrip, - ) -> Result { - let chunk = self.query_chunk(chunk_id).await?; - let idx = usize::try_from(strip_index).unwrap_or(usize::MAX); - if idx >= chunk.strips.len() { - return Err(LifecycleError::StripIndexOutOfRange { - index: strip_index, - len: chunk.strips.len(), - }); - } - let operation_id = ChunkId { - high: chunk_id.high ^ chunk.modify_ts, - low: chunk_id.low ^ u64::from(strip_index), - }; - self.replace_chunk_strip_range( - chunk_id, - chunk.modify_ts, - strip_index, - std::slice::from_ref(&chunk.strips[idx]), - std::slice::from_ref(&new_strip), - operation_id, - ) - .await - } - /// Atomically replace a capacity-compatible strip range under a revision /// fence. Newly introduced segments are committed before publication; /// removed segments stay allocated until the reader-layout grace expires. @@ -1367,9 +1146,9 @@ impl LifecycleHandler { data_num: u32, code_num: u32, ) -> Result { - if self.deployment_mode == Some(crate::chunkdb_config::DeploymentMode::TestSingleNode) { + if self.deployment_mode == Some(crate::chunkdb_config::DeploymentMode::Local) { return Err(LifecycleError::InvalidRequest( - "test_single_node disables mirror-to-EC conversion".into(), + "local disables mirror-to-EC conversion".into(), )); } self.check_range(chunk_id)?; @@ -1535,146 +1314,6 @@ impl LifecycleHandler { .map_err(LifecycleError::Storage) } - /// Finish durable `Init` allocation intents left by an interrupted commit. - pub async fn reconcile_pending_chunks(&self) -> Result { - let mut start_after = None; - let mut reconciled = 0u64; - loop { - let chunks = self.list_chunks(start_after.as_ref(), 1_000).await?; - if chunks.is_empty() { - break; - } - for mut chunk in chunks.iter().cloned() { - if !chunk.cleanup_intents.is_empty() { - let intent_count = chunk.cleanup_intents.len(); - let current_segments: HashSet<_> = - chunk.strips.iter().flat_map(extract_segments).collect(); - let now_ms = unix_time_ms(); - let mut pending = Vec::new(); - for intent in std::mem::take(&mut chunk.cleanup_intents) { - if intent.not_before_ms > now_ms { - pending.push(intent); - continue; - } - let retired: Vec<_> = intent - .retired_segments - .iter() - .filter(|segment| !current_segments.contains(segment)) - .copied() - .collect(); - self.allocator - .pool() - .free_blocks(retired) - .await - .map_err(LifecycleError::Cleanup)?; - } - chunk.cleanup_intents = pending; - if chunk.cleanup_intents.len() != intent_count { - chunk.modify_ts = chunk.modify_ts.saturating_add(1); - self.store.put_chunk(&chunk).await?; - if let (Some(locks), Some(chunk_id)) = (&self.locks, chunk.id) { - locks.populate_cache(&chunk_id, chunk.clone()); - } - reconciled = reconciled.saturating_add(1); - } - } - match ChunkState::from_proto(chunk.state) { - ChunkState::Init => { - self.commit_strip_segments(&chunk.strips).await?; - chunk.state = ProtoChunkState::Active as i32; - chunk.modify_ts = chunk.modify_ts.saturating_add(1); - } - ChunkState::Deleted if !chunk.strips.is_empty() => { - let segments = chunk.strips.iter().flat_map(extract_segments).collect(); - self.allocator - .pool() - .free_blocks(segments) - .await - .map_err(LifecycleError::Cleanup)?; - chunk.strips.clear(); - chunk.capacity = 0; - chunk.modify_ts = chunk.modify_ts.saturating_add(1); - } - _ => continue, - } - self.store.put_chunk(&chunk).await?; - if let (Some(locks), Some(chunk_id)) = (&self.locks, chunk.id) { - locks.populate_cache(&chunk_id, chunk); - } - reconciled += 1; - } - start_after = chunks.last().and_then(|chunk| chunk.id); - if chunks.len() < 1_000 { - break; - } - } - Ok(reconciled) - } - - /// Seal Active shared chunks whose persisted writer lease expired. - pub async fn seal_expired_writer_chunks(&self) -> Result { - let now_ms = unix_time_ms(); - let mut start_after = None; - let mut sealed = 0_u64; - loop { - let chunks = self.list_chunks(start_after.as_ref(), 1_000).await?; - if chunks.is_empty() { - break; - } - for candidate in &chunks { - if ChunkState::from_proto(candidate.state) != ChunkState::Active - || candidate.writer_epoch == 0 - || candidate.writer_lease_deadline_ms > now_ms - { - continue; - } - let Some(chunk_id) = candidate.id else { - continue; - }; - let mut guard = if let Some(locks) = &self.locks { - Some( - locks - .acquire(&chunk_id, &self.store, &LockPolicy::default(), CacheHint::Cache) - .await?, - ) - } else { - None - }; - let mut chunk = match &guard { - Some(guard) => guard - .chunk() - .unwrap_or_else(|| unreachable!("acquire guarantees chunk on Ok")) - .clone(), - None => self.store.get_chunk(&chunk_id).await?, - }; - if ChunkState::from_proto(chunk.state) != ChunkState::Active - || chunk.writer_epoch == 0 - || chunk.writer_lease_deadline_ms > now_ms - { - continue; - } - self.reclaim_unconsumed_reservations(&chunk_id).await?; - chunk.state = ProtoChunkState::Sealed as i32; - chunk.modify_ts = chunk.modify_ts.saturating_add(1); - chunk.sealed_ts_ms = now_ms; - chunk.sealed_length = - u32::try_from(chunk.acknowledged_cursor.div_ceil(1024)).unwrap_or(u32::MAX); - close_acknowledged_strips(&mut chunk, now_ms); - self.store.put_chunk(&chunk).await?; - if let Some(ref mut guard) = guard { - guard.refresh(chunk.clone()); - } - sealed = sealed.saturating_add(1); - info!(chunk_id = ?chunk_id, cursor = chunk.acknowledged_cursor, "expired shared writer chunk sealed"); - } - start_after = chunks.last().and_then(|chunk| chunk.id); - if chunks.len() < 1_000 { - break; - } - } - Ok(sealed) - } - /// Commit all segments in the given strips to diskdb (two-phase /// commit: mark tentative blocks as permanent after chunk persist). async fn commit_strip_segments(&self, strips: &[ChunkStrip]) -> Result<(), LifecycleError> { @@ -1740,7 +1379,7 @@ impl LifecycleHandler { } /// Extract all segments from a strip (mirror or EC). -fn extract_segments(strip: &ChunkStrip) -> Vec { +pub(super) fn extract_segments(strip: &ChunkStrip) -> Vec { use crowdb_protocol::chunkdb::rpc::Strip; match &strip.strip { Some(Strip::MirrorStrip(m)) => m.segments.clone(), @@ -1780,82 +1419,6 @@ fn conversion_unit_count(old: &[ChunkStrip], chunk_id: &ChunkId) -> Result Result<(), LifecycleError> { - let old_capacity: u32 = old.iter().map(|strip| strip.capacity).sum(); - let new_capacity: u32 = replacement.iter().map(|strip| strip.capacity).sum(); - if old_capacity != new_capacity - || old[0].chunk_offset != replacement[0].chunk_offset - || old[0].strip_sequence != replacement[0].strip_sequence - { - return Err(LifecycleError::InvalidRequest( - "replacement must preserve first offset, first sequence, and total capacity".into(), - )); - } - if replacement - .iter() - .flat_map(extract_segments) - .any(|segment| segment.owner_chunk.as_ref() != Some(chunk_id) || segment.unit_count == 0) - { - return Err(LifecycleError::InvalidRequest( - "replacement segments must be non-empty and owned by the chunk".into(), - )); - } - let mut expected_offset = replacement[0].chunk_offset; - for strip in replacement { - if strip.capacity == 0 || strip.unit_kb == 0 || strip.chunk_offset != expected_offset { - return Err(LifecycleError::InvalidRequest( - "replacement strips must be non-empty and contiguous".into(), - )); - } - expected_offset = expected_offset - .checked_add(strip.capacity) - .ok_or_else(|| LifecycleError::InvalidRequest("replacement strip range overflows u32".into()))?; - let segments = extract_segments(strip); - let shape_is_valid = match &strip.strip { - Some(Strip::MirrorStrip(_)) => { - !segments.is_empty() - && segments - .iter() - .all(|segment| segment.unit_count.saturating_mul(strip.unit_kb) == strip.capacity) - } - Some(Strip::EcStrip(ec)) => { - ec.data_num > 0 - && ec.code_num > 0 - && segments.len() - == usize::try_from(ec.data_num.saturating_add(ec.code_num)).unwrap_or(usize::MAX) - && segments.iter().all(|segment| { - segment - .unit_count - .saturating_mul(strip.unit_kb) - .saturating_mul(ec.data_num) - == strip.capacity - }) - } - None => false, - }; - if !shape_is_valid { - return Err(LifecycleError::InvalidRequest( - "replacement strip has invalid mirror or EC geometry".into(), - )); - } - let segment_set: HashSet<_> = segments.into_iter().collect(); - if strip - .unavailable_segments - .iter() - .any(|segment| !segment_set.contains(segment)) - { - return Err(LifecycleError::InvalidRequest( - "unavailable replicas must belong to the replacement strip".into(), - )); - } - } - Ok(()) -} - fn validate_replacement_sequences( chunk: &Chunk, replaced: std::ops::Range, @@ -1919,29 +1482,6 @@ fn writer_lease_deadline(now_ms: u64, writer_epoch: u64, writer_lease_ms: u64) - } } -fn close_acknowledged_strips(chunk: &mut Chunk, now_ms: u64) { - if chunk.writer_epoch == 0 || chunk.acknowledged_cursor == 0 { - return; - } - let cursor = chunk.acknowledged_cursor; - let mut last_closed = chunk.closed_strip_sequence; - for strip in &mut chunk.strips { - let start = u64::from(strip.chunk_offset) * 1024; - if cursor <= start { - break; - } - let end = u64::from(strip.chunk_offset.saturating_add(strip.capacity)) * 1024; - let length = cursor.min(end).saturating_sub(start); - strip.sealed_length = u32::try_from(length.div_ceil(1024)).unwrap_or(u32::MAX); - strip.sealed_ts_ms = now_ms; - last_closed = Some(strip.strip_sequence); - if cursor < end { - break; - } - } - chunk.closed_strip_sequence = last_closed; -} - fn seal_written_strips(chunk: &mut Chunk, seal_length: u32, now_ms: u64) { let mut remaining = seal_length; for strip in &mut chunk.strips { diff --git a/app/crowdb-chunkdb/src/lifecycle/handler/allocation.rs b/app/crowdb-chunkdb/src/lifecycle/handler/allocation.rs new file mode 100644 index 000000000..66e312f5e --- /dev/null +++ b/app/crowdb-chunkdb/src/lifecycle/handler/allocation.rs @@ -0,0 +1,234 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Chunk allocation and owner-aware initial placement. + +use super::{ + assign_strip_offsets, info, unix_time_ms, writer_lease_deadline, AllocationMetricGuard, CacheHint, Chunk, + ChunkGuard, ChunkId, ChunkStrip, ChunkType, LifecycleError, LifecycleHandler, LockPolicy, + ProtoChunkState, ProtoStripType, StoreError, StripAllocType, +}; +use crate::allocator::StripBatchSpec; +use crowdb_protocol::{chunk_stream::chunk_owner_key_matches_type, generate_chunk_id}; + +impl LifecycleHandler { + /// Allocate a new chunk. + #[allow(clippy::too_many_arguments)] + pub async fn allocate_chunk( + &self, + chunk_id: Option, + write_granularity_kb: u32, + strip_count: u32, + strip_type: ProtoStripType, + data_num: u32, + code_num: u32, + copy_count: u32, + chunk_type: ChunkType, + writer_epoch: u64, + writer_lease_ms: u64, + ) -> Result { + self.allocate_chunk_owned( + chunk_id, + write_granularity_kb, + strip_count, + strip_type, + data_num, + code_num, + copy_count, + chunk_type, + writer_epoch, + writer_lease_ms, + Vec::new(), + ) + .await + } + + /// Allocate a new chunk with a stable logical owner identity. + #[allow(clippy::too_many_arguments)] + pub async fn allocate_chunk_owned( + &self, + chunk_id: Option, + write_granularity_kb: u32, + strip_count: u32, + strip_type: ProtoStripType, + data_num: u32, + code_num: u32, + copy_count: u32, + chunk_type: ChunkType, + writer_epoch: u64, + writer_lease_ms: u64, + owner_key: Vec, + ) -> Result { + self.validate_strip_layout(strip_type, data_num, code_num, copy_count, write_granularity_kb)?; + if !chunk_owner_key_matches_type(chunk_type, &owner_key) { + return Err(LifecycleError::InvalidRequest( + "chunk owner key does not match chunk type".into(), + )); + } + let id = match chunk_id { + Some(id) => id, + None => self.generate_owned_chunk_id(chunk_type)?, + }; + if (id.high >> 56) != u64::from(chunk_type as u8) { + return Err(LifecycleError::InvalidRequest( + "chunk id prefix does not match chunk type".into(), + )); + } + self.check_range(&id)?; + let mut allocation_guard = AllocationMetricGuard::new(self.metrics.clone()); + + let mut guard = self.acquire_allocation_guard(&id, chunk_id.is_some()).await?; + let strips = self + .allocate_initial_strips( + &id, + write_granularity_kb, + strip_count, + (strip_type, data_num, code_num, copy_count), + ) + .await?; + let now_ms = unix_time_ms(); + if writer_epoch != 0 && writer_lease_ms == 0 { + self.allocator.rollback_strips(&strips).await?; + return Err(LifecycleError::InvalidRequest( + "writer_lease_ms must be nonzero for a shared writer".into(), + )); + } + + let chunk = Chunk { + id: Some(id), + modify_ts: 1, + state: ProtoChunkState::Active as i32, + create_ts_ms: now_ms, + sealed_ts_ms: 0, + capacity: strips.iter().map(|s| s.capacity).sum(), + sealed_length: 0, + strips, + chunk_type: chunk_type as i32, + writer_epoch, + acknowledged_cursor: 0, + closed_strip_sequence: None, + writer_lease_deadline_ms: writer_lease_deadline(now_ms, writer_epoch, writer_lease_ms), + next_strip_sequence: strip_count, + cleanup_intents: Vec::new(), + last_strip_replacement: None, + owner_key, + }; + self.create_chunk_with_liveness_task(&chunk).await?; + self.liveness_renewed_at.insert(id, now_ms); + self.admit_placement_repairs(&chunk); + self.commit_strip_segments_background(chunk.strips.clone()); + + // Update cache. + if let Some(ref mut g) = guard { + g.refresh(chunk.clone()); + } else if chunk_id.is_none() { + // Auto-generated ID: populate cache directly (no guard). + if let Some(locks) = &self.locks { + locks.populate_cache(&id, chunk.clone()); + } + } + info!(chunk_id = ?id, strips = strip_count, "chunk allocated"); + allocation_guard.mark_success(); + Ok(chunk) + } + + fn generate_owned_chunk_id(&self, chunk_type: ChunkType) -> Result { + let Some(range_guard) = &self.range_guard else { + return Ok(generate_chunk_id(chunk_type as u8).to_proto()); + }; + if !range_guard.is_ready() { + let candidate = generate_chunk_id(chunk_type as u8).to_proto(); + return self.check_range(&candidate).map(|()| candidate); + } + let attempts = 32 * u64::from(crowdb_protocol::chunk_slot::CHUNK_SLOT_COUNT) + / range_guard.owned_bucket_count().max(1); + for _ in 0..attempts { + let candidate = generate_chunk_id(chunk_type as u8).to_proto(); + if range_guard.check(&candidate).is_ok() { + return Ok(candidate); + } + } + Err(LifecycleError::InvalidRequest( + "failed to generate a chunk id in the owned range".into(), + )) + } + + async fn acquire_allocation_guard( + &self, + id: &ChunkId, + caller_supplied: bool, + ) -> Result, LifecycleError> { + let guard = if caller_supplied { + if let Some(locks) = &self.locks { + let g = locks + .acquire_for_create(id, &LockPolicy::default(), CacheHint::Cache) + .await?; + // Existence check inside the lock. + match self.store.get_chunk(id).await { + Ok(_) => return Err(LifecycleError::ChunkAlreadyExists), + Err(StoreError::ChunkNotFound) => {} + Err(e) => return Err(LifecycleError::Storage(e)), + } + Some(g) + } else { + None + } + } else { + None + }; + + Ok(guard) + } + + async fn allocate_initial_strips( + &self, + id: &ChunkId, + write_granularity_kb: u32, + strip_count: u32, + layout: (ProtoStripType, u32, u32, u32), + ) -> Result, LifecycleError> { + let (strip_type, data_num, code_num, copy_count) = layout; + let snap = self.topology.snapshot(); + + let mirror_copies = if copy_count == 0 { 2 } else { copy_count as usize }; + let strip_alloc_type = + self.protected_degraded_layout(strip_type, &snap) + .unwrap_or(match strip_type { + ProtoStripType::Mirror => StripAllocType::Mirror { + copy_count: mirror_copies, + }, + ProtoStripType::Ec => StripAllocType::Ec { + data_num: data_num as usize, + code_num: code_num as usize, + }, + }); + + let constraints = self.allocation_constraints(strip_type, &snap); + // Convert write_granularity (KB) to unit_count using the unit + // size from the topology snapshot. Fall back to treating KB as + // units if unit_size_bytes is unavailable (0). + let unit_size_kb = snap.unit_size_bytes() / 1024; + let unit_count = write_granularity_kb + .checked_div(unit_size_kb) + .unwrap_or(write_granularity_kb) + .max(1); + + let mut strips = self + .allocator + .allocate_strips( + &snap, + id, + StripBatchSpec { + strip_type: strip_alloc_type, + unit_count, + start_sequence: 0, + strip_count, + }, + &constraints, + ) + .await?; + assign_strip_offsets(&mut strips, 0); + + Ok(strips) + } +} diff --git a/app/crowdb-chunkdb/src/lifecycle/handler/append.rs b/app/crowdb-chunkdb/src/lifecycle/handler/append.rs index 405a14e38..cd9255ef1 100644 --- a/app/crowdb-chunkdb/src/lifecycle/handler/append.rs +++ b/app/crowdb-chunkdb/src/lifecycle/handler/append.rs @@ -132,7 +132,7 @@ impl LifecycleHandler { return Err(error); } }; - if current.modify_ts != chunk.modify_ts { + if !same_allocation_basis(&chunk, ¤t) { drop(guard); self.allocator.rollback_strips(&appended).await?; return Ok(AppendChunkOutcome { @@ -198,3 +198,27 @@ impl LifecycleHandler { Ok(()) } } + +/// Cursor/lease progress does not invalidate an allocation made against the +/// same owner and strip layout. Every other field remains an exact fence. +fn same_allocation_basis(observed: &super::Chunk, current: &super::Chunk) -> bool { + if current.modify_ts < observed.modify_ts + || current.acknowledged_cursor < observed.acknowledged_cursor + || current.writer_lease_deadline_ms < observed.writer_lease_deadline_ms + { + return false; + } + let mut basis = observed.clone(); + basis.modify_ts = current.modify_ts; + basis.acknowledged_cursor = current.acknowledged_cursor; + basis.writer_lease_deadline_ms = current.writer_lease_deadline_ms; + basis == *current +} + +#[cfg(feature = "test-util")] +impl LifecycleHandler { + #[must_use] + pub fn append_basis_unchanged_for_tests(observed: &super::Chunk, current: &super::Chunk) -> bool { + same_allocation_basis(observed, current) + } +} diff --git a/app/crowdb-chunkdb/src/lifecycle/handler/recovery.rs b/app/crowdb-chunkdb/src/lifecycle/handler/recovery.rs new file mode 100644 index 000000000..c20b1d03a --- /dev/null +++ b/app/crowdb-chunkdb/src/lifecycle/handler/recovery.rs @@ -0,0 +1,102 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Replay persisted block intents without admitting requests before completion. + +use super::{extract_segments, unix_time_ms, ChunkState, LifecycleError, LifecycleHandler, ProtoChunkState}; +use crate::allocator::pool::BlockMutationError; +use std::collections::HashSet; + +impl LifecycleHandler { + /// Finish durable `Init` allocation intents left by an interrupted commit. + pub async fn reconcile_pending_chunks(&self) -> Result { + let mut start_after = None; + let mut reconciled = 0u64; + loop { + let chunks = self.list_chunks(start_after.as_ref(), 1_000).await?; + if chunks.is_empty() { + break; + } + for mut chunk in chunks.iter().cloned() { + if !chunk.cleanup_intents.is_empty() { + let intent_count = chunk.cleanup_intents.len(); + let current_segments: HashSet<_> = + chunk.strips.iter().flat_map(extract_segments).collect(); + let now_ms = unix_time_ms(); + let mut pending = Vec::new(); + for intent in std::mem::take(&mut chunk.cleanup_intents) { + if intent.not_before_ms > now_ms { + pending.push(intent); + continue; + } + let retired: Vec<_> = intent + .retired_segments + .iter() + .filter(|segment| !current_segments.contains(segment)) + .copied() + .collect(); + self.allocator + .pool() + .free_blocks_for_recovery(retired) + .await + .map_err(|error| recovery_error(&error, false))?; + } + chunk.cleanup_intents = pending; + if chunk.cleanup_intents.len() != intent_count { + chunk.modify_ts = chunk.modify_ts.saturating_add(1); + self.store.put_chunk(&chunk).await?; + if let (Some(locks), Some(chunk_id)) = (&self.locks, chunk.id) { + locks.populate_cache(&chunk_id, chunk.clone()); + } + reconciled = reconciled.saturating_add(1); + } + } + match ChunkState::from_proto(chunk.state) { + ChunkState::Init => { + self.allocator + .pool() + .commit_blocks_for_recovery( + chunk.strips.iter().flat_map(extract_segments).collect(), + ) + .await + .map_err(|error| recovery_error(&error, true))?; + chunk.state = ProtoChunkState::Active as i32; + chunk.modify_ts = chunk.modify_ts.saturating_add(1); + } + ChunkState::Deleted if !chunk.strips.is_empty() => { + let segments = chunk.strips.iter().flat_map(extract_segments).collect(); + self.allocator + .pool() + .free_blocks_for_recovery(segments) + .await + .map_err(|error| recovery_error(&error, false))?; + chunk.strips.clear(); + chunk.capacity = 0; + chunk.modify_ts = chunk.modify_ts.saturating_add(1); + } + _ => continue, + } + self.store.put_chunk(&chunk).await?; + if let (Some(locks), Some(chunk_id)) = (&self.locks, chunk.id) { + locks.populate_cache(&chunk_id, chunk); + } + reconciled += 1; + } + start_after = chunks.last().and_then(|chunk| chunk.id); + if chunks.len() < 1_000 { + break; + } + } + Ok(reconciled) + } +} + +fn recovery_error(error: &BlockMutationError, commit: bool) -> LifecycleError { + if error.is_retryable() { + LifecycleError::RecoveryDependency(error.to_string()) + } else if commit { + LifecycleError::Commit(error.to_string()) + } else { + LifecycleError::Cleanup(error.to_string()) + } +} diff --git a/app/crowdb-chunkdb/src/lifecycle/handler/reservation.rs b/app/crowdb-chunkdb/src/lifecycle/handler/reservation.rs index 039cc66b6..fbc720224 100644 --- a/app/crowdb-chunkdb/src/lifecycle/handler/reservation.rs +++ b/app/crowdb-chunkdb/src/lifecycle/handler/reservation.rs @@ -180,7 +180,7 @@ impl LifecycleHandler { rollback.push(group.strips[index].clone()); } StripReservationState::Consumed => { - if group.planned_cursors[index] != 0 && now_ms >= reuse_not_before_ms { + if now_ms >= reuse_not_before_ms { // Consume is durable before data I/O, so a crashed // writer cannot prove the mirrors reached stable // storage. Keep its blocks allocated beyond the RPC @@ -201,8 +201,7 @@ impl LifecycleHandler { self.reservation_admission.release(usage.0, usage.1); } } - // Retain legacy consumed reservations indefinitely and newer consumed - // reservations until their reuse grace period has elapsed. + // Retain consumed reservations until their reuse grace has elapsed. if group.states.contains(&(StripReservationState::Consumed as i32)) { return Ok(ReservationRecovery::Reconciled); } @@ -260,58 +259,6 @@ impl LifecycleHandler { Ok(()) } - pub(super) async fn reclaim_unconsumed_reservations( - &self, - chunk_id: &ChunkId, - ) -> Result { - let groups = self.store.list_reservation_groups(chunk_id).await?; - let mut reclaimed = 0_u64; - for mut group in groups { - validate_group_shape(&group)?; - let mut rollback = Vec::new(); - let mut newly_cancelled = Vec::new(); - for (index, state) in group.states.iter_mut().enumerate() { - if *state == StripReservationState::Reserved as i32 { - *state = StripReservationState::Cancelled as i32; - rollback.push(group.strips[index].clone()); - newly_cancelled.push(group.strips[index].clone()); - } else if *state == StripReservationState::Cancelled as i32 { - rollback.push(group.strips[index].clone()); - } - } - if !rollback.is_empty() { - self.store.put_reservation_group(&group).await?; - self.allocator.rollback_strips(&rollback).await?; - for strip in &newly_cancelled { - let usage = strip_usage(strip); - self.reservation_admission.release(usage.0, usage.1); - } - reclaimed = reclaimed.saturating_add(rollback.len() as u64); - } - let has_consumed = group.states.contains(&(StripReservationState::Consumed as i32)); - if !has_consumed - && group.states.iter().all(|state| { - *state == StripReservationState::Confirmed as i32 - || *state == StripReservationState::Cancelled as i32 - }) - { - if !group.parity_segments.is_empty() { - self.allocator - .pool() - .free_blocks(group.parity_segments.clone()) - .await - .map_err(LifecycleError::Cleanup)?; - let usage = parity_usage(&group); - self.reservation_admission.release(usage.0, usage.1); - } - if let Some(group_id) = group.group_id { - self.store.delete_reservation_group(chunk_id, &group_id).await?; - } - } - } - Ok(reclaimed) - } - #[allow(clippy::too_many_lines)] pub async fn mutate_strip_reservation( &self, @@ -738,6 +685,16 @@ pub(super) fn validate_group_shape(group: &StripReservationGroup) -> Result<(), "reservation state and strip counts differ".into(), )); } + if group + .states + .iter() + .zip(&group.planned_cursors) + .any(|(state, cursor)| *state == StripReservationState::Consumed as i32 && *cursor == 0) + { + return Err(LifecycleError::InvalidRequest( + "consumed reservation must record a planned cursor".into(), + )); + } Ok(()) } diff --git a/app/crowdb-chunkdb/src/lifecycle/handler/seal.rs b/app/crowdb-chunkdb/src/lifecycle/handler/seal.rs index 81ac39957..dcd237f40 100644 --- a/app/crowdb-chunkdb/src/lifecycle/handler/seal.rs +++ b/app/crowdb-chunkdb/src/lifecycle/handler/seal.rs @@ -4,23 +4,43 @@ //! Exact byte sealing and retirement of unused strips. use super::{ - close_acknowledged_strips, extract_segments, reservation, seal_written_strips, unix_time_ms, CacheHint, - Chunk, ChunkId, ChunkState, LifecycleError, LifecycleHandler, LockPolicy, ProtoChunkState, Segment, - StripCleanupIntent, StripReservationState, + extract_segments, reservation, seal_written_strips, unix_time_ms, CacheHint, Chunk, ChunkId, ChunkState, + LifecycleError, LifecycleHandler, LockPolicy, ProtoChunkState, Segment, StripCleanupIntent, + StripReservationState, }; +use crowdb_protocol::chunkdb::rpc::{ChunkStrip, ChunkType, StripReservationGroup}; use tracing::info; impl LifecycleHandler { - /// Seal a chunk — no more appends allowed. - pub async fn seal_chunk(&self, chunk_id: &ChunkId, seal_length: u32) -> Result { - self.seal_chunk_bytes(chunk_id, u64::from(seal_length) * 1024) + /// Seal a chunk at its exact written byte boundary. + pub async fn seal_chunk_bytes( + &self, + chunk_id: &ChunkId, + seal_bytes: u64, + ) -> Result { + self.seal_chunk_at_boundary(chunk_id, seal_bytes, None).await + } + + /// Seal only the exact metadata snapshot whose physical bytes were observed. + /// The caller must independently establish writer exclusion and IO completion. + pub async fn seal_observed_chunk( + &self, + observed: &Chunk, + seal_bytes: u64, + ) -> Result { + let chunk_id = observed + .id + .as_ref() + .ok_or_else(|| LifecycleError::InvalidRequest("observed chunk has no identity".into()))?; + self.seal_chunk_at_boundary(chunk_id, seal_bytes, Some(observed)) .await } - pub async fn seal_chunk_bytes( + async fn seal_chunk_at_boundary( &self, chunk_id: &ChunkId, seal_bytes: u64, + observed: Option<&Chunk>, ) -> Result { let seal_length = u32::try_from(seal_bytes.div_ceil(1024)) .map_err(|_| LifecycleError::InvalidRequest("seal size exceeds capacity field".into()))?; @@ -43,39 +63,14 @@ impl LifecycleHandler { .clone(), None => self.store.get_chunk(chunk_id).await?, }; + if observed.is_some_and(|observed| observed != &chunk) { + return Err(LifecycleError::StateConflict); + } let current_state = ChunkState::from_proto(chunk.state); current_state.check_can_seal()?; - if seal_length > chunk.capacity { - return Err(LifecycleError::InvalidRequest(format!( - "seal_length {seal_length} exceeds chunk capacity {}", - chunk.capacity - ))); - } + validate_seal_bounds(&chunk, seal_bytes, seal_length)?; - let reservation_groups = self.store.list_reservation_groups(chunk_id).await?; - if reservation_groups - .iter() - .any(|group| group.states.contains(&(StripReservationState::Consumed as i32))) - { - return Err(LifecycleError::StateConflict); - } - let mut reserved_strips = Vec::new(); - let mut reserved_parity = Vec::new(); - let mut reserved_parity_groups = Vec::new(); - for mut group in reservation_groups { - reservation::validate_group_shape(&group)?; - reserved_parity.extend_from_slice(&group.parity_segments); - for (index, state) in group.states.iter_mut().enumerate() { - if *state == StripReservationState::Reserved as i32 { - *state = StripReservationState::Cancelled as i32; - reserved_strips.push(group.strips[index].clone()); - } else if *state == StripReservationState::Cancelled as i32 { - reserved_strips.push(group.strips[index].clone()); - } - } - self.store.put_reservation_group(&group).await?; - reserved_parity_groups.push(group); - } + let reservations = self.cancel_seal_reservations(chunk_id).await?; let now_ms = unix_time_ms(); let (unused_segments, cleanup_operation) = trim_sealed_strips(&mut chunk, chunk_id, seal_length); @@ -93,7 +88,7 @@ impl LifecycleHandler { not_before_ms: now_ms, }); } - self.rollback_reserved_resources(&reserved_strips, reserved_parity, &reserved_parity_groups) + self.rollback_reserved_resources(&reservations.strips, reservations.parity, &reservations.groups) .await?; for group in self.store.list_reservation_groups(chunk_id).await? { if let Some(group_id) = group.group_id { @@ -126,6 +121,45 @@ impl LifecycleHandler { info!(chunk_id = ?chunk_id, seal_length, "chunk sealed"); Ok(chunk) } + + async fn cancel_seal_reservations(&self, chunk_id: &ChunkId) -> Result { + let reservation_groups = self.store.list_reservation_groups(chunk_id).await?; + if reservation_groups + .iter() + .any(|group| group.states.contains(&(StripReservationState::Consumed as i32))) + { + return Err(LifecycleError::StateConflict); + } + let mut reserved_strips = Vec::new(); + let mut reserved_parity = Vec::new(); + let mut reserved_parity_groups = Vec::new(); + for mut group in reservation_groups { + reservation::validate_group_shape(&group)?; + reserved_parity.extend_from_slice(&group.parity_segments); + for (index, state) in group.states.iter_mut().enumerate() { + if *state == StripReservationState::Reserved as i32 { + *state = StripReservationState::Cancelled as i32; + reserved_strips.push(group.strips[index].clone()); + } else if *state == StripReservationState::Cancelled as i32 { + reserved_strips.push(group.strips[index].clone()); + } + } + self.store.put_reservation_group(&group).await?; + reserved_parity_groups.push(group); + } + + Ok(SealReservations { + strips: reserved_strips, + parity: reserved_parity, + groups: reserved_parity_groups, + }) + } +} + +struct SealReservations { + strips: Vec, + parity: Vec, + groups: Vec, } fn trim_sealed_strips( @@ -146,3 +180,45 @@ fn trim_sealed_strips( }); (unused_segments, operation) } + +fn validate_seal_bounds(chunk: &Chunk, seal_bytes: u64, seal_length: u32) -> Result<(), LifecycleError> { + if (chunk.chunk_type == ChunkType::Wal as i32 || chunk.chunk_type == ChunkType::Stream as i32) + && seal_bytes < chunk.acknowledged_cursor + { + return Err(LifecycleError::InvalidRequest(format!( + "stream seal boundary {seal_bytes} precedes persisted cursor {}", + chunk.acknowledged_cursor + ))); + } + if seal_length > chunk.capacity { + return Err(LifecycleError::InvalidRequest(format!( + "seal_length {seal_length} exceeds chunk capacity {}", + chunk.capacity + ))); + } + + Ok(()) +} + +fn close_acknowledged_strips(chunk: &mut Chunk, now_ms: u64) { + if chunk.writer_epoch == 0 || chunk.acknowledged_cursor == 0 { + return; + } + let cursor = chunk.acknowledged_cursor; + let mut last_closed = chunk.closed_strip_sequence; + for strip in &mut chunk.strips { + let start = u64::from(strip.chunk_offset) * 1024; + if cursor <= start { + break; + } + let end = u64::from(strip.chunk_offset.saturating_add(strip.capacity)) * 1024; + let length = cursor.min(end).saturating_sub(start); + strip.sealed_length = u32::try_from(length.div_ceil(1024)).unwrap_or(u32::MAX); + strip.sealed_ts_ms = now_ms; + last_closed = Some(strip.strip_sequence); + if cursor < end { + break; + } + } + chunk.closed_strip_sequence = last_closed; +} diff --git a/app/crowdb-chunkdb/src/lifecycle/replacement.rs b/app/crowdb-chunkdb/src/lifecycle/replacement.rs new file mode 100644 index 000000000..50c671c46 --- /dev/null +++ b/app/crowdb-chunkdb/src/lifecycle/replacement.rs @@ -0,0 +1,114 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Validate replacement coverage before publishing any physical layout. + +use super::handler::{extract_segments, LifecycleError}; +use crowdb_protocol::{ + chunkdb::rpc::{ChunkStrip, Strip}, + common::ChunkId, +}; +use std::collections::HashSet; + +pub(super) fn validate_replacement_geometry( + old: &[ChunkStrip], + replacement: &[ChunkStrip], + chunk_id: &ChunkId, +) -> Result<(), LifecycleError> { + let old_capacity: u32 = old.iter().map(|strip| strip.capacity).sum(); + let new_capacity: u32 = replacement.iter().map(|strip| strip.capacity).sum(); + if old_capacity != new_capacity + || old[0].chunk_offset != replacement[0].chunk_offset + || old[0].strip_sequence != replacement[0].strip_sequence + { + return Err(LifecycleError::InvalidRequest( + "replacement must preserve first offset, first sequence, and total capacity".into(), + )); + } + if replacement + .iter() + .flat_map(extract_segments) + .any(|segment| segment.owner_chunk.as_ref() != Some(chunk_id) || segment.unit_count == 0) + { + return Err(LifecycleError::InvalidRequest( + "replacement segments must be non-empty and owned by the chunk".into(), + )); + } + let mut expected_offset = replacement[0].chunk_offset; + for strip in replacement { + if strip.capacity == 0 || strip.unit_kb == 0 || strip.chunk_offset != expected_offset { + return Err(LifecycleError::InvalidRequest( + "replacement strips must be non-empty and contiguous".into(), + )); + } + expected_offset = expected_offset + .checked_add(strip.capacity) + .ok_or_else(|| LifecycleError::InvalidRequest("replacement strip range overflows u32".into()))?; + let segments = extract_segments(strip); + validate_physical_ranges(&segments)?; + let shape_is_valid = match &strip.strip { + Some(Strip::MirrorStrip(_)) => { + !segments.is_empty() + && segments + .iter() + .all(|segment| segment.unit_count.saturating_mul(strip.unit_kb) == strip.capacity) + } + Some(Strip::EcStrip(ec)) => { + ec.data_num > 0 + && ec.code_num > 0 + && segments.len() + == usize::try_from(ec.data_num.saturating_add(ec.code_num)).unwrap_or(usize::MAX) + && segments.iter().all(|segment| { + segment + .unit_count + .saturating_mul(strip.unit_kb) + .saturating_mul(ec.data_num) + == strip.capacity + }) + } + None => false, + }; + if !shape_is_valid { + return Err(LifecycleError::InvalidRequest( + "replacement strip has invalid mirror or EC geometry".into(), + )); + } + let segment_set: HashSet<_> = segments.into_iter().collect(); + if strip + .unavailable_segments + .iter() + .any(|segment| !segment_set.contains(segment)) + { + return Err(LifecycleError::InvalidRequest( + "unavailable replicas must belong to the replacement strip".into(), + )); + } + } + Ok(()) +} + +fn validate_physical_ranges( + segments: &[crowdb_protocol::diskdb::rpc::Segment], +) -> Result<(), LifecycleError> { + let mut ranges = Vec::with_capacity(segments.len()); + for segment in segments { + let disk = segment.disk_id.ok_or_else(|| { + LifecycleError::InvalidRequest("replacement segment has no disk identity".into()) + })?; + let end = segment + .unit_offset + .checked_add(u64::from(segment.unit_count)) + .ok_or_else(|| LifecycleError::InvalidRequest("replacement segment range overflows".into()))?; + ranges.push((disk.high, disk.low, segment.zone_index, segment.unit_offset, end)); + } + ranges.sort_unstable(); + if ranges.windows(2).any(|pair| { + let (left, right) = (&pair[0], &pair[1]); + (left.0, left.1, left.2) == (right.0, right.1, right.2) && left.4 > right.3 + }) { + return Err(LifecycleError::InvalidRequest( + "replacement copies overlap the same physical allocation".into(), + )); + } + Ok(()) +} diff --git a/app/crowdb-chunkdb/src/main.rs b/app/crowdb-chunkdb/src/main.rs index 47bad0678..d9d6b7a9d 100644 --- a/app/crowdb-chunkdb/src/main.rs +++ b/app/crowdb-chunkdb/src/main.rs @@ -451,7 +451,7 @@ async fn main() { rpc_service.register_handlers(&rpc_server); rpc_server.start(); prepared.store(true, std::sync::atomic::Ordering::Release); - info!(%rpc_listen_addr, "crowdb-rpc server listening (R116 migration)"); + info!(%rpc_listen_addr, "crowdb-rpc server listening"); // Start HTTP health + metrics + cache invalidation server. let readiness = HttpReadiness { @@ -797,7 +797,7 @@ async fn validate_voting_topology(kv: &Arc, mode: DeploymentMode let nodes = voters.remove(&(store_id, group_id)).unwrap_or_default(); let valid = match mode { DeploymentMode::Production => nodes.len() >= 3, - DeploymentMode::TestSingleNode => nodes.len() == 1, + DeploymentMode::Local => matches!(nodes.len(), 1 | 3), DeploymentMode::TestUnsafePlacement => { unreachable!("test fixtures bypass voting topology validation") } diff --git a/app/crowdb-chunkdb/src/metrics.rs b/app/crowdb-chunkdb/src/metrics.rs index 8113cdc27..84089f1d5 100644 --- a/app/crowdb-chunkdb/src/metrics.rs +++ b/app/crowdb-chunkdb/src/metrics.rs @@ -24,7 +24,7 @@ pub enum RequestKind { SealChunk, DeleteChunk, DeleteChunkRange, - UpdateChunkStrip, + LayoutMutation, ListChunks, AdHocEcRecovery, } @@ -40,7 +40,7 @@ impl RequestKind { Self::SealChunk, Self::DeleteChunk, Self::DeleteChunkRange, - Self::UpdateChunkStrip, + Self::LayoutMutation, Self::ListChunks, Self::AdHocEcRecovery, ]; @@ -56,7 +56,7 @@ impl RequestKind { Self::SealChunk => "seal_chunk", Self::DeleteChunk => "delete_chunk", Self::DeleteChunkRange => "delete_chunk_range", - Self::UpdateChunkStrip => "update_chunk_strip", + Self::LayoutMutation => "layout_mutation", Self::ListChunks => "list_chunks", Self::AdHocEcRecovery => "ad_hoc_ec_recovery", } diff --git a/app/crowdb-chunkdb/src/runtime.rs b/app/crowdb-chunkdb/src/runtime.rs index d938523f8..85322d6b5 100644 --- a/app/crowdb-chunkdb/src/runtime.rs +++ b/app/crowdb-chunkdb/src/runtime.rs @@ -25,6 +25,7 @@ use crowdb_protocol::chunk_domain::ChunkDomain; mod background; mod executor; +mod recovery; #[derive(Clone)] pub struct RuntimeContext { @@ -103,7 +104,7 @@ impl RuntimeContext { .with_layout_validity(Duration::from_millis(config.lifecycle.layout_validity_ms)), ); handler.rebuild_reservation_admission().await?; - handler.reconcile_pending_chunks().await?; + recovery::recover(&handler, self.stop_rx.clone()).await?; // Build the crowdb-rpc server. The RpcServer listens on the RPC // port and dispatches to ChunkdbRpcService handlers. let rpc_rt_handle = tokio::runtime::Handle::current(); @@ -115,7 +116,7 @@ impl RuntimeContext { let relocation = Arc::new(RelocationCoordinator::new(Arc::clone(&task_manager))); let conversion = Arc::new( ConversionCoordinator::new(Arc::clone(&handler), Arc::clone(&task_store)) - .with_enabled(config.deployment.mode != DeploymentMode::TestSingleNode) + .with_enabled(config.deployment.mode != DeploymentMode::Local) .with_wake(task_manager.wake_handle()) .with_policy( config.conversion.data_num, diff --git a/app/crowdb-chunkdb/src/runtime/executor.rs b/app/crowdb-chunkdb/src/runtime/executor.rs index b0374bca4..823ce087f 100644 --- a/app/crowdb-chunkdb/src/runtime/executor.rs +++ b/app/crowdb-chunkdb/src/runtime/executor.rs @@ -91,7 +91,7 @@ impl RuntimeContext { Arc::clone(&task_manager), )), ]; - if config.deployment.mode != DeploymentMode::TestSingleNode { + if config.deployment.mode != DeploymentMode::Local { task_handlers.push(conversion_task_handler); } let executor = Arc::new( diff --git a/app/crowdb-chunkdb/src/runtime/recovery.rs b/app/crowdb-chunkdb/src/runtime/recovery.rs new file mode 100644 index 000000000..a9b057e4d --- /dev/null +++ b/app/crowdb-chunkdb/src/runtime/recovery.rs @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Startup remains initializing while durable block intents await their owners. + +use crate::lifecycle::{LifecycleError, LifecycleHandler}; +use std::time::Duration; +use tokio::sync::watch; + +pub(super) async fn recover( + handler: &LifecycleHandler, + mut stop: watch::Receiver, +) -> Result<(), LifecycleError> { + let recovery = async { + let mut last_error = None; + loop { + match handler.reconcile_pending_chunks().await { + Ok(_) => return Ok(()), + Err(LifecycleError::RecoveryDependency(error)) => { + if last_error.as_ref() != Some(&error) { + tracing::warn!(%error, "startup recovery waiting for block owner; retaining durable intents"); + last_error = Some(error); + } + } + Err(error) => return Err(error), + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + }; + tokio::select! { + biased; + () = async { + while !*stop.borrow_and_update() { + if stop.changed().await.is_err() { break; } + } + } => Err(LifecycleError::InvalidRequest("startup recovery stopped".into())), + result = recovery => result, + } +} diff --git a/app/crowdb-chunkdb/src/service.rs b/app/crowdb-chunkdb/src/service.rs index 99758c13e..955ce26f3 100644 --- a/app/crowdb-chunkdb/src/service.rs +++ b/app/crowdb-chunkdb/src/service.rs @@ -3,6 +3,5 @@ //! chunkdb service layer — crowdb-rpc handler set. pub mod chunkdb_rpc_service; -pub mod chunkdb_service; pub use chunkdb_rpc_service::ChunkdbRpcService; diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/dispatch.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/dispatch.rs index 3c2efc519..cbc39c30a 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/dispatch.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/dispatch.rs @@ -10,7 +10,7 @@ use crowdb_protocol::chunkdb_fb::{ FBDeleteChunkRangeRequest, FBDeleteChunkRequest, FBDiscardReplacementSegmentRequest, FBMutateStripReservationRequest, FBPrepareMirrorToEcConversionRequest, FBQueryChunkRequest, FBQuerySegmentOwnerRequest, FBRelocateSegmentHandoffRequest, FBReplaceChunkStripRangeRequest, - FBReserveStripGroupRequest, FBSealChunkRequest, FBTriggerConversionRequest, FBUpdateChunkStripRequest, + FBReserveStripGroupRequest, FBSealChunkRequest, FBTriggerConversionRequest, }; use crowdb_protocol::common::ChunkId; use crowdb_protocol::fb::FBMsgType; @@ -40,7 +40,6 @@ pub(super) fn is_system_request(message: u16, control: &[u8]) -> bool { ESealChunkRequest => FBSealChunkRequest, EDeleteChunkRequest => FBDeleteChunkRequest, EDeleteChunkRangeRequest => FBDeleteChunkRangeRequest, - EUpdateChunkStripRequest => FBUpdateChunkStripRequest, EAllocateReplacementSegmentRequest => FBAllocateReplacementSegmentRequest, EDiscardReplacementSegmentRequest => FBDiscardReplacementSegmentRequest, EReplaceChunkStripRangeRequest => FBReplaceChunkStripRangeRequest, diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs index 6bdbe6923..7b96e0081 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/mutations.rs @@ -8,7 +8,7 @@ use super::{ FBCompleteMirrorToEcConversionRequest, FBDeleteChunkRangeRequest, FBDeleteChunkRequest, FBDiscardReplacementSegmentRequest, FBMsgType, FBPrepareMirrorToEcConversionRequest, FBReplaceChunkStripRangeRequest, FBSealChunkRequest, FBTriggerConversionBatchRequest, - FBTriggerConversionRequest, FBUpdateChunkStripRequest, RequestGuard, RpcServer, ServerRequest, + FBTriggerConversionRequest, RequestGuard, RpcServer, ServerRequest, }; use crate::conversion::ConversionError; @@ -913,13 +913,7 @@ impl ChunkdbRpcService { ); return; }; - let seal_length = fb_req.seal_length(); - - let result = if fb_req.seal_bytes() == 0 { - handler.seal_chunk(&chunk_id, seal_length).await - } else { - handler.seal_chunk_bytes(&chunk_id, fb_req.seal_bytes()).await - }; + let result = handler.seal_chunk_bytes(&chunk_id, fb_req.seal_bytes()).await; if result.is_ok() { request.mark_success(); } @@ -1057,91 +1051,6 @@ impl ChunkdbRpcService { ); }); } - - // ── UpdateChunkStrip ────────────────────────────────────────── - - pub(super) fn handle_update_strip( - &self, - req: ServerRequest, - server: &Arc, - mut request: RequestGuard, - ) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let msg_type = FBMsgType::EUpdateChunkStripResponse.0 as u16; - let conn_handle_usize = req.conn_handle as usize; - - let handler = Arc::clone(&self.handler); - let server = Arc::clone(server); - self.spawn(async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBChunkdbRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - - let Some(chunk_id) = fb_req.chunk_id().map(|id| ChunkId { - high: id.high(), - low: id.low(), - }) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBChunkdbRetCode::InvalidArgument, - "missing chunk_id", - ); - return; - }; - let strip_index = fb_req.strip_index(); - let Some(fb_strip) = fb_req.strip() else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBChunkdbRetCode::InvalidArgument, - "missing strip", - ); - return; - }; - let Some(strip) = parse_fb_chunk_strip(&fb_strip) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBChunkdbRetCode::InvalidArgument, - "invalid strip body", - ); - return; - }; - - let result = handler.update_chunk_strip(&chunk_id, strip_index, strip).await; - if result.is_ok() { - request.mark_success(); - } - submit_chunk_result( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - result, - ); - }); - } } fn unix_time_ms() -> u64 { diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/service.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/service.rs index 45756746d..6bf7c6417 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/service.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/service.rs @@ -7,13 +7,10 @@ // by the C++ transport). Confined to `submit_response` calls. #![allow(unsafe_code)] -//! crowdb-rpc handler set for `ChunkdbService` (R116 migration). +//! crowdb-rpc handler set for the current `ChunkDB` service. //! -//! Each handler dispatches by `msg_type` to the existing lifecycle -//! logic — the same bodies as the tonic `ChunkdbService` in -//! `chunkdb_service.rs`. The response is a flatbuffer frame built per -//! `design-crowdb-rpc.md` §6 (build → finish → attach) and submitted via -//! `RpcServer::submit_response`. +//! Each handler dispatches by `msg_type` to lifecycle logic. Responses are +//! `FlatBuffer` frames submitted through `RpcServer::submit_response`. //! //! Handlers run on the C++ I/O worker thread. All chunkdb lifecycle //! methods are async (KV persist, diskdb allocation), so each handler @@ -54,7 +51,7 @@ use crowdb_protocol::chunkdb_fb::{ FBStripReservationAction, FBStripReservationGroup, FBStripReservationGroupArgs, FBStripReservationState, FBStripType, FBTriggerConversionBatchRequest, FBTriggerConversionBatchResponse, FBTriggerConversionBatchResponseArgs, FBTriggerConversionRequest, FBTriggerConversionResponse, - FBTriggerConversionResponseArgs, FBUpdateChunkStripRequest, + FBTriggerConversionResponseArgs, }; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::fb::FBMsgType; @@ -69,9 +66,8 @@ use crate::metrics::{ChunkdbMetrics, RequestGuard, RequestKind}; use crate::relocation::RelocationCoordinator; use crate::task::{SegmentOwnerResolver, TaskStore}; -/// crowdb-rpc handler set for `ChunkdbService`. Holds the same -/// `LifecycleHandler` as the tonic `ChunkdbService`; `register_handlers` -/// wires one handler per request `msg_type` into a `RpcServer`. +/// `ChunkDB` lifecycle RPC handlers. `register_handlers` wires one handler +/// per request `msg_type` into a `RpcServer`. pub struct ChunkdbRpcService { handler: Arc, metrics: Arc, @@ -214,7 +210,7 @@ impl ChunkdbRpcService { Self::make_handler( Arc::clone(self), Arc::clone(server), - RequestKind::UpdateChunkStrip, + RequestKind::LayoutMutation, Self::handle_relocate_segment_handoff, ), ); @@ -245,15 +241,6 @@ impl ChunkdbRpcService { Self::handle_delete_range, ), ); - server.register_handler( - FBMsgType::EUpdateChunkStripRequest.0 as u16, - Self::make_handler( - Arc::clone(self), - Arc::clone(server), - RequestKind::UpdateChunkStrip, - Self::handle_update_strip, - ), - ); server.register_handler( FBMsgType::EListChunksRequest.0 as u16, Self::make_handler( @@ -282,7 +269,7 @@ impl ChunkdbRpcService { Self::make_handler( Arc::clone(self), Arc::clone(server), - RequestKind::UpdateChunkStrip, + RequestKind::LayoutMutation, Self::handle_prepare_conversion, ), ); @@ -291,7 +278,7 @@ impl ChunkdbRpcService { Self::make_handler( Arc::clone(self), Arc::clone(server), - RequestKind::UpdateChunkStrip, + RequestKind::LayoutMutation, Self::handle_complete_conversion, ), ); @@ -300,7 +287,7 @@ impl ChunkdbRpcService { Self::make_handler( Arc::clone(self), Arc::clone(server), - RequestKind::UpdateChunkStrip, + RequestKind::LayoutMutation, Self::handle_trigger_conversion, ), ); @@ -309,7 +296,7 @@ impl ChunkdbRpcService { Self::make_handler( Arc::clone(self), Arc::clone(server), - RequestKind::UpdateChunkStrip, + RequestKind::LayoutMutation, Self::handle_trigger_conversion_batch, ), ); @@ -321,7 +308,7 @@ impl ChunkdbRpcService { Self::make_handler( Arc::clone(self), Arc::clone(server), - RequestKind::UpdateChunkStrip, + RequestKind::LayoutMutation, Self::handle_allocate_replacement, ), ); @@ -330,7 +317,7 @@ impl ChunkdbRpcService { Self::make_handler( Arc::clone(self), Arc::clone(server), - RequestKind::UpdateChunkStrip, + RequestKind::LayoutMutation, Self::handle_replace_range, ), ); @@ -339,7 +326,7 @@ impl ChunkdbRpcService { Self::make_handler( Arc::clone(self), Arc::clone(server), - RequestKind::UpdateChunkStrip, + RequestKind::LayoutMutation, Self::handle_discard_replacement, ), ); diff --git a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs index e89dd1bc9..4f01a2961 100644 --- a/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs +++ b/app/crowdb-chunkdb/src/service/chunkdb_rpc_service/wire.rs @@ -47,9 +47,10 @@ pub(super) fn map_error(e: &LifecycleError) -> (FBChunkdbRetCode, String, u32, u } LifecycleError::Storage(_) => (FBChunkdbRetCode::Internal, e.to_string(), 0, 0), LifecycleError::InvalidRequest(_) => (FBChunkdbRetCode::InvalidArgument, e.to_string(), 0, 0), - LifecycleError::LockBusy | LifecycleError::LockTimeout | LifecycleError::ReservationLimit => { - (FBChunkdbRetCode::Unavailable, e.to_string(), 0, 0) - } + LifecycleError::LockBusy + | LifecycleError::LockTimeout + | LifecycleError::ReservationLimit + | LifecycleError::RecoveryDependency(_) => (FBChunkdbRetCode::Unavailable, e.to_string(), 0, 0), LifecycleError::StripIndexOutOfRange { .. } => { (FBChunkdbRetCode::StripIndexOutOfRange, e.to_string(), 0, 0) } diff --git a/app/crowdb-chunkdb/src/service/chunkdb_service.rs b/app/crowdb-chunkdb/src/service/chunkdb_service.rs deleted file mode 100644 index be2503546..000000000 --- a/app/crowdb-chunkdb/src/service/chunkdb_service.rs +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! chunkdb service — delegates to lifecycle handlers. -//! -//! The legacy tonic server trait impl has been removed. The crowdb-rpc -//! dispatch lives in `chunkdb_rpc_service.rs`. - -use crate::lifecycle::LifecycleHandler; - -/// Suppress unused-import warning — `LifecycleHandler` was used by the -/// former tonic `ChunkdbService` struct. The crowdb-rpc service in -/// `chunkdb_rpc_service.rs` has its own struct with the same handler. -#[allow(dead_code)] -type _DeprecatedChunkdbService = std::sync::Arc; diff --git a/app/crowdb-chunkdb/src/storage.rs b/app/crowdb-chunkdb/src/storage.rs index e21c35f06..ae3fe7254 100644 --- a/app/crowdb-chunkdb/src/storage.rs +++ b/app/crowdb-chunkdb/src/storage.rs @@ -4,11 +4,12 @@ //! KV persistence layer for chunk metadata. //! //! `ChunkStore` reads/writes `Chunk` proto records to CROWDB KV via the -//! routed KV group. Serialization uses prost (no Rust type duplication, -//! design §3.8). The chunk ID (24 bytes) is the KV key. +//! routed KV group. Chunk records have one explicit, strictly decoded envelope. +mod chunk_codec; pub mod reservation; mod scope; +use chunk_codec::{decode_chunk, encode_chunk}; use std::sync::Arc; @@ -17,11 +18,9 @@ use tracing::warn; use bytes::Bytes; use crowdb_kv_client::{BatchOp, CrowdbKvClient, GetOutcome, ReadMode}; use crowdb_protocol::chunk_task::ChunkTaskValue; -use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkStrip, Strip, StripCleanupIntent}; +use crowdb_protocol::chunkdb::rpc::Chunk; use crowdb_protocol::common::ChunkId; -use crowdb_protocol::diskdb::rpc::Segment; use crowdb_protocol::{encode_chunk_task_value, BinaryKey, ChunkTaskKey, FinalizeChunkTaskKey}; -use serde::Deserialize; use crate::range_guard::RangeGuard; use crate::routing::{BindingCache, Route}; @@ -58,131 +57,6 @@ pub struct ChunkStore { scope: Option<(Arc, ChunkDomain)>, } -#[derive(Deserialize)] -struct LegacyChunk { - id: Option, - modify_ts: u64, - state: i32, - create_ts_ms: u64, - sealed_ts_ms: u64, - capacity: u32, - sealed_length: u32, - strips: Vec, - chunk_type: i32, - writer_epoch: u64, - acknowledged_cursor: u64, - closed_strip_sequence: Option, - writer_lease_deadline_ms: u64, - next_strip_sequence: u32, - cleanup_intents: Vec, - last_strip_replacement: Option, -} - -#[derive(Deserialize)] -struct PreviousChunk { - id: Option, - modify_ts: u64, - state: i32, - create_ts_ms: u64, - sealed_ts_ms: u64, - capacity: u32, - sealed_length: u32, - strips: Vec, - chunk_type: i32, - writer_epoch: u64, - acknowledged_cursor: u64, - closed_strip_sequence: Option, - writer_lease_deadline_ms: u64, - next_strip_sequence: u32, - cleanup_intents: Vec, - last_strip_replacement: Option, - owner_key: Vec, -} - -#[derive(Deserialize)] -pub(super) struct LegacyChunkStrip { - chunk_offset: u32, - strip_sequence: u32, - unit_kb: u32, - capacity: u32, - create_ts_ms: u64, - sealed_ts_ms: u64, - sealed_length: u32, - strip_type: i32, - strip: Option, - usage_bitmap: Vec, - unavailable_segments: Vec, -} - -impl From for ChunkStrip { - fn from(strip: LegacyChunkStrip) -> Self { - Self { - chunk_offset: strip.chunk_offset, - strip_sequence: strip.strip_sequence, - unit_kb: strip.unit_kb, - capacity: strip.capacity, - create_ts_ms: strip.create_ts_ms, - sealed_ts_ms: strip.sealed_ts_ms, - sealed_length: strip.sealed_length, - strip_type: strip.strip_type, - strip: strip.strip, - usage_bitmap: strip.usage_bitmap, - unavailable_segments: strip.unavailable_segments, - placement_priority: 0, - placement_assessment: None, - placement_repair_required: false, - } - } -} - -impl From for Chunk { - fn from(chunk: LegacyChunk) -> Self { - Self { - id: chunk.id, - modify_ts: chunk.modify_ts, - state: chunk.state, - create_ts_ms: chunk.create_ts_ms, - sealed_ts_ms: chunk.sealed_ts_ms, - capacity: chunk.capacity, - sealed_length: chunk.sealed_length, - strips: chunk.strips.into_iter().map(ChunkStrip::from).collect(), - chunk_type: chunk.chunk_type, - writer_epoch: chunk.writer_epoch, - acknowledged_cursor: chunk.acknowledged_cursor, - closed_strip_sequence: chunk.closed_strip_sequence, - writer_lease_deadline_ms: chunk.writer_lease_deadline_ms, - next_strip_sequence: chunk.next_strip_sequence, - cleanup_intents: chunk.cleanup_intents, - last_strip_replacement: chunk.last_strip_replacement, - owner_key: Vec::new(), - } - } -} - -impl From for Chunk { - fn from(chunk: PreviousChunk) -> Self { - Self { - id: chunk.id, - modify_ts: chunk.modify_ts, - state: chunk.state, - create_ts_ms: chunk.create_ts_ms, - sealed_ts_ms: chunk.sealed_ts_ms, - capacity: chunk.capacity, - sealed_length: chunk.sealed_length, - strips: chunk.strips.into_iter().map(ChunkStrip::from).collect(), - chunk_type: chunk.chunk_type, - writer_epoch: chunk.writer_epoch, - acknowledged_cursor: chunk.acknowledged_cursor, - closed_strip_sequence: chunk.closed_strip_sequence, - writer_lease_deadline_ms: chunk.writer_lease_deadline_ms, - next_strip_sequence: chunk.next_strip_sequence, - cleanup_intents: chunk.cleanup_intents, - last_strip_replacement: chunk.last_strip_replacement, - owner_key: chunk.owner_key, - } - } -} - impl ChunkStore { #[must_use] pub fn new(kv: Arc, bindings: BindingCache) -> Self { @@ -219,7 +93,7 @@ impl ChunkStore { let ops = [ BatchOp::Put { key: Bytes::from(chunk_key.clone()), - value: Bytes::from(encode_chunk(chunk)), + value: Bytes::from(encode_chunk(chunk)?), }, BatchOp::Put { key: Bytes::from(task_key), @@ -256,7 +130,7 @@ impl ChunkStore { let id = chunk.id.as_ref().expect("chunk has id"); let r = self.route(id)?; let key = chunk_key(id); - let value = encode_chunk(chunk); + let value = encode_chunk(chunk)?; self.put_chunk_at(r.kv_store_id, r.kv_group_id, &key, &value, chunk) .await?; @@ -384,20 +258,14 @@ pub(super) fn chunk_key(id: &ChunkId) -> Vec { key } -/// Encode a `Chunk` to bytes (bincode). -pub(super) fn encode_chunk(chunk: &Chunk) -> Vec { - bincode::serialize(chunk).expect("Chunk serialization") -} - -/// Decode a `Chunk` from bytes (bincode). -fn decode_chunk(data: &[u8]) -> Result { - bincode::deserialize(data) - .or_else(|_| bincode::deserialize::(data).map(Chunk::from)) - .or_else(|_| bincode::deserialize::(data).map(Chunk::from)) - .map_err(|e| StoreError::Serde(e.to_string())) -} - #[cfg(feature = "test-util")] pub fn decode_chunk_for_tests(data: &[u8]) -> Result { decode_chunk(data) } + +/// # Errors +/// Rejects unsupported physical format revisions or serialization failures. +#[cfg(feature = "test-util")] +pub fn encode_chunk_for_tests(chunk: &Chunk) -> Result> { + encode_chunk(chunk) +} diff --git a/app/crowdb-chunkdb/src/storage/chunk_codec.rs b/app/crowdb-chunkdb/src/storage/chunk_codec.rs new file mode 100644 index 000000000..4c84cb24a --- /dev/null +++ b/app/crowdb-chunkdb/src/storage/chunk_codec.rs @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Strict versioned chunk metadata records. + +use super::{Result, StoreError}; +use bincode::Options; +use crowdb_protocol::chunkdb::rpc::Chunk; + +const MAGIC: &[u8; 4] = b"CCHK"; +const VERSION: u16 = 2; +const HEADER: usize = 12; + +pub(super) fn encode_chunk(chunk: &Chunk) -> Result> { + let body = bincode::serialize(chunk).map_err(codec_error)?; + let length = u32::try_from(body.len()).map_err(codec_error)?; + let mut bytes = Vec::with_capacity(HEADER + body.len()); + bytes.extend_from_slice(MAGIC); + bytes.extend_from_slice(&VERSION.to_le_bytes()); + bytes.extend_from_slice(&[0, 0]); + bytes.extend_from_slice(&length.to_le_bytes()); + bytes.extend_from_slice(&body); + Ok(bytes) +} + +pub(super) fn decode_chunk(bytes: &[u8]) -> Result { + let header = bytes + .get(..HEADER) + .ok_or_else(|| codec_error("incomplete chunk envelope"))?; + let version = u16::from_le_bytes([header[4], header[5]]); + let length = u32::from_le_bytes(header[8..12].try_into().map_err(codec_error)?); + if &header[..4] != MAGIC + || version != VERSION + || header[6..8] != [0, 0] + || usize::try_from(length).ok() != Some(bytes.len() - HEADER) + { + return Err(codec_error("unsupported or malformed chunk envelope")); + } + bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(u64::from(length)) + .reject_trailing_bytes() + .deserialize(&bytes[HEADER..]) + .map_err(codec_error) +} + +fn codec_error(error: impl std::fmt::Display) -> StoreError { + StoreError::Serde(format!("chunk metadata codec: {error}")) +} diff --git a/app/crowdb-chunkdb/src/storage/reservation.rs b/app/crowdb-chunkdb/src/storage/reservation.rs index 15f55b940..9fe718177 100644 --- a/app/crowdb-chunkdb/src/storage/reservation.rs +++ b/app/crowdb-chunkdb/src/storage/reservation.rs @@ -5,10 +5,8 @@ use bytes::Bytes; use crowdb_kv_client::{BatchOp, GetOutcome, ReadMode, ScanOutcome}; -use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkStrip, StripReservationGroup}; +use crowdb_protocol::chunkdb::rpc::{Chunk, StripReservationGroup}; use crowdb_protocol::common::ChunkId; -use crowdb_protocol::diskdb::rpc::Segment; -use serde::Deserialize; use crate::routing::Route; @@ -100,7 +98,7 @@ impl ChunkStore { &[ BatchOp::Put { key: Bytes::from(chunk_key(&chunk_id)), - value: Bytes::from(encode_chunk(chunk)), + value: Bytes::from(encode_chunk(chunk)?), }, BatchOp::Put { key: Bytes::from(reservation_key(&chunk_id, &group_id)), @@ -141,7 +139,7 @@ impl ChunkStore { .ok_or_else(|| StoreError::Serde("chunk has no id".into()))?; let mut ops = vec![BatchOp::Put { key: Bytes::from(chunk_key(&chunk_id)), - value: Bytes::from(encode_chunk(chunk)), + value: Bytes::from(encode_chunk(chunk)?), }]; if let Some(group) = group { ops.push(BatchOp::Put { @@ -277,105 +275,11 @@ fn encode_group(group: &StripReservationGroup) -> Result> { } fn decode_group(bytes: &[u8]) -> Result { - let mut group: StripReservationGroup = match bincode::deserialize(bytes) { - Ok(group) => group, - Err(current_error) => match bincode::deserialize::(bytes) { - Ok(previous) => previous.into(), - Err(previous_error) => { - let legacy: LegacyStripReservationGroup = - bincode::deserialize(bytes).map_err(|legacy_error| { - StoreError::Serde(format!( - "reservation decode failed: current={current_error}; previous={previous_error}; legacy={legacy_error}" - )) - })?; - legacy.into() - } - }, - }; - if group.planned_cursors.is_empty() { - group.planned_cursors.resize(group.strips.len(), 0); - } - if group.planned_closed_sequences.is_empty() { - group - .planned_closed_sequences - .resize(group.strips.len(), u32::MAX); - } - Ok(group) -} - -#[derive(Deserialize)] -struct LegacyStripReservationGroup { - group_id: Option, - chunk_id: Option, - writer_epoch: u64, - lease_generation: u64, - lease_deadline_ms: u64, - placement_epoch: u64, - strips: Vec, - states: Vec, - parity_segments: Vec, - preferred_survivors: Vec, - data_num: u32, - code_num: u32, -} - -#[derive(Deserialize)] -struct PreviousStripReservationGroup { - group_id: Option, - chunk_id: Option, - writer_epoch: u64, - lease_generation: u64, - lease_deadline_ms: u64, - placement_epoch: u64, - strips: Vec, - states: Vec, - parity_segments: Vec, - preferred_survivors: Vec, - data_num: u32, - code_num: u32, - planned_cursors: Vec, - planned_closed_sequences: Vec, -} - -impl From for StripReservationGroup { - fn from(previous: PreviousStripReservationGroup) -> Self { - Self { - group_id: previous.group_id, - chunk_id: previous.chunk_id, - writer_epoch: previous.writer_epoch, - lease_generation: previous.lease_generation, - lease_deadline_ms: previous.lease_deadline_ms, - placement_epoch: previous.placement_epoch, - strips: previous.strips.into_iter().map(ChunkStrip::from).collect(), - states: previous.states, - parity_segments: previous.parity_segments, - preferred_survivors: previous.preferred_survivors, - data_num: previous.data_num, - code_num: previous.code_num, - planned_cursors: previous.planned_cursors, - planned_closed_sequences: previous.planned_closed_sequences, - } - } -} - -impl From for StripReservationGroup { - fn from(legacy: LegacyStripReservationGroup) -> Self { - let strip_count = legacy.strips.len(); - Self { - group_id: legacy.group_id, - chunk_id: legacy.chunk_id, - writer_epoch: legacy.writer_epoch, - lease_generation: legacy.lease_generation, - lease_deadline_ms: legacy.lease_deadline_ms, - placement_epoch: legacy.placement_epoch, - strips: legacy.strips.into_iter().map(ChunkStrip::from).collect(), - states: legacy.states, - parity_segments: legacy.parity_segments, - preferred_survivors: legacy.preferred_survivors, - data_num: legacy.data_num, - code_num: legacy.code_num, - planned_cursors: vec![0; strip_count], - planned_closed_sequences: vec![u32::MAX; strip_count], - } - } + use bincode::Options; + bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(bytes.len() as u64) + .reject_trailing_bytes() + .deserialize(bytes) + .map_err(|error| StoreError::Serde(error.to_string())) } diff --git a/app/crowdb-chunkdb/src/task/executor.rs b/app/crowdb-chunkdb/src/task/executor.rs index 71523f0fa..f09d30258 100644 --- a/app/crowdb-chunkdb/src/task/executor.rs +++ b/app/crowdb-chunkdb/src/task/executor.rs @@ -14,6 +14,7 @@ use super::{TaskClaim, TaskManager, TaskManagerError}; pub type TaskFuture<'a> = Pin + Send + 'a>>; +#[derive(Debug)] pub enum TaskOutcome { Complete, Retry { diff --git a/app/crowdb-chunkdb/tests/append_basis_test.rs b/app/crowdb-chunkdb/tests/append_basis_test.rs new file mode 100644 index 000000000..e3415daa2 --- /dev/null +++ b/app/crowdb-chunkdb/tests/append_basis_test.rs @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb::lifecycle::LifecycleHandler; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkStrip}; + +#[test] +fn pending_allocation_preserves_progress_but_rejects_changed_ownership_or_layout() { + let observed = Chunk { + state: ChunkState::Active as i32, + writer_epoch: 9, + owner_key: b"stream/test".to_vec(), + modify_ts: 10, + acknowledged_cursor: 4096, + writer_lease_deadline_ms: 100, + capacity: 1024, + next_strip_sequence: 1, + strips: vec![ChunkStrip { + capacity: 1024, + ..ChunkStrip::default() + }], + ..Chunk::default() + }; + let current = Chunk { + modify_ts: 12, + acknowledged_cursor: 12288, + writer_lease_deadline_ms: 200, + ..observed.clone() + }; + assert!(LifecycleHandler::append_basis_unchanged_for_tests( + &observed, ¤t + )); + let conflicts = [ + Chunk { + writer_epoch: 10, + ..current.clone() + }, + Chunk { + owner_key: b"different".to_vec(), + ..current.clone() + }, + Chunk { + state: ChunkState::Sealed as i32, + ..current.clone() + }, + Chunk { + capacity: 2048, + ..current.clone() + }, + Chunk { + next_strip_sequence: 2, + ..current.clone() + }, + Chunk { + strips: vec![], + ..current.clone() + }, + Chunk { + modify_ts: 9, + ..current.clone() + }, + Chunk { + acknowledged_cursor: 0, + ..current.clone() + }, + Chunk { + writer_lease_deadline_ms: 99, + ..current.clone() + }, + ]; + for conflict in conflicts { + assert!( + !LifecycleHandler::append_basis_unchanged_for_tests(&observed, &conflict), + "{conflict:?}" + ); + } +} diff --git a/app/crowdb-chunkdb/tests/chunk_format_metadata_test.rs b/app/crowdb-chunkdb/tests/chunk_format_metadata_test.rs new file mode 100644 index 000000000..f75e049b8 --- /dev/null +++ b/app/crowdb-chunkdb/tests/chunk_format_metadata_test.rs @@ -0,0 +1,28 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb::storage::{decode_chunk_for_tests, encode_chunk_for_tests}; +use crowdb_protocol::chunkdb::rpc::Chunk; + +#[test] +fn format_metadata_envelope_rejects_unknown_and_malformed_records() { + let chunk = Chunk::default(); + let bytes = encode_chunk_for_tests(&chunk).unwrap(); + assert_eq!(decode_chunk_for_tests(&bytes).unwrap(), chunk); + for offset in [4, 6, 8] { + let mut invalid = bytes.clone(); + invalid[offset] ^= 1; + assert!(decode_chunk_for_tests(&invalid).is_err(), "byte {offset}"); + } + for end in [4, 11, bytes.len() - 1] { + assert!(decode_chunk_for_tests(&bytes[..end]).is_err(), "length {end}"); + } + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(decode_chunk_for_tests(&trailing).is_err()); + let mut trailing_body = trailing.clone(); + let length = u32::try_from(trailing_body.len() - 12).unwrap(); + trailing_body[8..12].copy_from_slice(&length.to_le_bytes()); + assert!(decode_chunk_for_tests(&trailing_body).is_err()); + assert!(decode_chunk_for_tests(&bincode::serialize(&chunk).unwrap()).is_err()); +} diff --git a/app/crowdb-chunkdb/tests/common/diskio_stub.rs b/app/crowdb-chunkdb/tests/common/diskio_stub.rs index a2b391398..6cc10f737 100644 --- a/app/crowdb-chunkdb/tests/common/diskio_stub.rs +++ b/app/crowdb-chunkdb/tests/common/diskio_stub.rs @@ -14,6 +14,7 @@ use crowdb_protocol::fb::{ }; use flatbuffers::FlatBufferBuilder; use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; @@ -23,6 +24,14 @@ type Blocks = Arc>>>; pub struct TestDiskio { worker: JoinHandle<()>, + pub control: Arc, +} + +#[derive(Default)] +pub struct TestDiskioControl { + pub reject_drains: AtomicBool, + pub drain_calls: AtomicUsize, + pub read_calls: AtomicUsize, } impl TestDiskio { @@ -36,13 +45,15 @@ impl TestDiskio { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let blocks = Blocks::default(); + let control = Arc::new(TestDiskioControl::default()); + let serving_control = Arc::clone(&control); let worker = tokio::spawn(async move { let mut connections = JoinSet::new(); loop { tokio::select! { accepted = listener.accept() => { let (stream, _) = accepted.unwrap(); - connections.spawn(serve(stream, Arc::clone(&blocks))); + connections.spawn(serve(stream, Arc::clone(&blocks), Arc::clone(&serving_control))); } finished = connections.join_next(), if !connections.is_empty() => { finished.unwrap().unwrap(); @@ -54,7 +65,7 @@ impl TestDiskio { .heartbeat_diskio_at(instance, &address.to_string(), rack, node, &[group], &[]) .await .unwrap(); - Self { worker } + Self { worker, control } } } @@ -64,7 +75,7 @@ impl Drop for TestDiskio { } } -async fn serve(mut stream: TcpStream, blocks: Blocks) { +async fn serve(mut stream: TcpStream, blocks: Blocks, control: Arc) { loop { let mut header = [0; 14]; if stream.read_exact(&mut header).await.is_err() { @@ -81,7 +92,7 @@ async fn serve(mut stream: TcpStream, blocks: Blocks) { if stream.read_exact(&mut ctrl).await.is_err() || stream.read_exact(&mut data).await.is_err() { return; } - let (reply, payload) = response(kind, &ctrl, data, &blocks); + let (reply, payload) = response(kind, &ctrl, data, &blocks, &control); let mut outgoing = vec![]; outgoing.extend_from_slice(&0xca70_u16.to_le_bytes()); outgoing.extend_from_slice(&(kind + 1).to_le_bytes()); @@ -96,41 +107,45 @@ async fn serve(mut stream: TcpStream, blocks: Blocks) { } } -fn response(kind: u16, ctrl: &[u8], data: Vec, blocks: &Blocks) -> (Vec, Vec) { +fn response( + kind: u16, + ctrl: &[u8], + data: Vec, + blocks: &Blocks, + control: &TestDiskioControl, +) -> (Vec, Vec) { let mut builder = FlatBufferBuilder::new(); let mut payload = vec![]; if kind == FBMsgType::EDiskWriteRequest.0 as u16 { let request = flatbuffers::root::(ctrl).unwrap(); assert_eq!(data.len(), request.size() as usize); - blocks.lock().unwrap().insert( - ( - request.disk_id().unwrap().0, - request.zone_index(), - request.zone_offset(), - ), - data, - ); + let mut sync_completed = request.sync(); + if data.is_empty() && request.sync() { + control.drain_calls.fetch_add(1, Ordering::Relaxed); + sync_completed = !control.reject_drains.load(Ordering::Relaxed); + } else { + blocks.lock().unwrap().insert( + ( + request.disk_id().unwrap().0, + request.zone_index(), + request.zone_offset(), + ), + data, + ); + } let reply = FBDiskWriteResponse::create( &mut builder, &FBDiskWriteResponseArgs { id: request.id(), + sync_completed, ..Default::default() }, ); builder.finish(reply, None); } else if kind == FBMsgType::EDiskReadRequest.0 as u16 { + control.read_calls.fetch_add(1, Ordering::Relaxed); let request = flatbuffers::root::(ctrl).unwrap(); - payload = blocks - .lock() - .unwrap() - .get(&( - request.disk_id().unwrap().0, - request.zone_index(), - request.zone_offset(), - )) - .cloned() - .unwrap_or_else(|| vec![0; request.size() as usize]); - assert_eq!(payload.len(), request.size() as usize); + payload = read_blocks(blocks, &request); let reply = FBDiskReadResponse::create( &mut builder, &FBDiskReadResponseArgs { @@ -163,3 +178,23 @@ fn response(kind: u16, ctrl: &[u8], data: Vec, blocks: &Blocks) -> (Vec, } (builder.finished_data().to_vec(), payload) } + +fn read_blocks(blocks: &Blocks, request: &FBDiskReadRequest<'_>) -> Vec { + let mut bytes = vec![0; request.size() as usize]; + let start = request.zone_offset(); + let end = start + u64::from(request.size()); + for ((disk, zone, offset), data) in blocks.lock().unwrap().iter() { + if *disk != request.disk_id().unwrap().0 || *zone != request.zone_index() { + continue; + } + let begin = start.max(*offset); + let finish = end.min(*offset + u64::try_from(data.len()).unwrap()); + if begin < finish { + let target = usize::try_from(begin - start).unwrap(); + let source = usize::try_from(begin - offset).unwrap(); + let length = usize::try_from(finish - begin).unwrap(); + bytes[target..target + length].copy_from_slice(&data[source..source + length]); + } + } + bytes +} diff --git a/app/crowdb-chunkdb/tests/common/finalization.rs b/app/crowdb-chunkdb/tests/common/finalization.rs new file mode 100644 index 000000000..5e4aa7219 --- /dev/null +++ b/app/crowdb-chunkdb/tests/common/finalization.rs @@ -0,0 +1,163 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::{atomic::Ordering, Arc}; + +use bytes::Bytes; +use crowdb_chunkdb::{ + conversion::io::ConversionDiskIo, + finalize::FinalizeChunkTaskHandler, + routing::{default_binding_table, BindingCache}, + task::{TaskHandler, TaskOutcome, TaskStore}, +}; +use crowdb_protocol::{ + chunk_stream::StreamName, + chunk_task::{ChunkTaskValue, TASK_KIND_FINALIZE_CHUNK}, + chunkdb::rpc::{Chunk, ChunkType, Strip, StripType}, + frame::encode_append_group_views, +}; + +use super::{ + cluster::{seed_hardware, ChunkdbHarness, DiskdbServer, KvCluster, DATA_GROUP_ID, STORE_ID}, + diskio_stub::TestDiskio, +}; + +pub struct TestFinalization { + pub harness: ChunkdbHarness, + pub io: Arc, + pub chunk: Chunk, + pub task: ChunkTaskValue, + diskio: Vec, + _diskdb: DiskdbServer, + _cluster: KvCluster, +} + +impl TestFinalization { + pub async fn start(chunk_type: ChunkType) -> Self { + let cluster = KvCluster::start_pinned().await; + let hardware = cluster.make_hardware_client(); + seed_hardware(&hardware).await; + let diskdb = DiskdbServer::start(&cluster).await; + let service = cluster.make_service_registry_client(); + let mut diskio = Vec::new(); + for index in 0..4 { + diskio + .push(TestDiskio::start(&service, 9000 + index, 100 + index, 10 + index, 1000 + index).await); + } + let harness = ChunkdbHarness::start(&cluster).await; + let io = Arc::new(ConversionDiskIo::connect(&service, &hardware).await.unwrap()); + let chunk = harness + .handler + .allocate_chunk_owned( + None, + 1024, + 1, + StripType::Mirror, + 0, + 0, + 3, + chunk_type, + 19, + 60_000, + if matches!(chunk_type, ChunkType::Stream | ChunkType::Wal) { + StreamName::generate().chunk_owner_key() + } else { + Vec::new() + }, + ) + .await + .unwrap(); + let id = chunk.id.unwrap(); + let bindings = BindingCache::new(); + bindings + .replace(default_binding_table(STORE_ID, DATA_GROUP_ID)) + .unwrap(); + let tasks = TaskStore::new(cluster.make_crowdb_client(), bindings); + let mut task = tasks + .get(&id, TASK_KIND_FINALIZE_CHUNK, &id) + .await + .unwrap() + .unwrap(); + task.eligible_at_ms = 0; + Self { + harness, + io, + chunk, + task, + diskio, + _diskdb: diskdb, + _cluster: cluster, + } + } + + pub fn reject_drains(&self, reject: bool) { + for diskio in &self.diskio { + diskio.control.reject_drains.store(reject, Ordering::Relaxed); + } + } + + pub fn io_counts(&self) -> (usize, usize) { + self.diskio.iter().fold((0, 0), |(drains, reads), diskio| { + ( + drains + diskio.control.drain_calls.load(Ordering::Relaxed), + reads + diskio.control.read_calls.load(Ordering::Relaxed), + ) + }) + } + + pub fn group(&self, offset: u64, payload: &[u8]) -> Vec { + encode_append_group_views( + self.chunk.id.unwrap(), + offset, + &Bytes::copy_from_slice(payload), + 1, + ) + .unwrap() + .concat() + } + + pub async fn write(&self, replica: usize, bytes: Vec) { + let strip = &self.chunk.strips[0]; + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + panic!("expected mirrors") + }; + self.io + .write_segment( + &mirror.segments[replica], + u64::from(strip.unit_kb) * 1024, + Bytes::from(bytes), + ) + .await + .unwrap(); + } + + pub async fn checkpoint(&mut self, cursor: u64) { + self.chunk = self + .harness + .handler + .advance_chunk_write( + &self.chunk.id.unwrap(), + 19, + self.chunk.modify_ts, + cursor, + None, + 60_000, + ) + .await + .unwrap(); + } + + pub async fn run(&self) -> TaskOutcome { + FinalizeChunkTaskHandler::new(Arc::clone(&self.harness.handler), Arc::clone(&self.io)) + .execute(&self.task) + .await + } + + pub async fn current(&self) -> Chunk { + self.harness + .handler + .query_chunk(&self.chunk.id.unwrap()) + .await + .unwrap() + } +} diff --git a/app/crowdb-chunkdb/tests/common/mod.rs b/app/crowdb-chunkdb/tests/common/mod.rs index 55b371412..40502a130 100644 --- a/app/crowdb-chunkdb/tests/common/mod.rs +++ b/app/crowdb-chunkdb/tests/common/mod.rs @@ -5,3 +5,6 @@ pub mod cluster; #[allow(dead_code)] pub mod diskio_stub; + +#[allow(dead_code)] +pub mod finalization; diff --git a/app/crowdb-chunkdb/tests/config_test.rs b/app/crowdb-chunkdb/tests/config_test.rs index 70ed09b69..ea3c89816 100644 --- a/app/crowdb-chunkdb/tests/config_test.rs +++ b/app/crowdb-chunkdb/tests/config_test.rs @@ -36,16 +36,27 @@ fn tracked_config_file_loads_and_validates() { } #[test] -fn single_node_container_declares_test_only_deployment() { +fn container_declares_release_supported_local_deployment() { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("../../container/single-node-container/templates/chunkdb.toml"); let config = crowdb_common::config::load_from_file::(&path).unwrap(); - assert_eq!(config.deployment.mode, DeploymentMode::TestSingleNode); + assert_eq!(config.deployment.mode, DeploymentMode::Local); assert_eq!(config.deployment.max_node_failures, 0); assert_eq!(config.placement.mode, PlacementMode::UnsafeColocated); assert_eq!(config.conversion_io.rpc_workers, 1); } +#[test] +fn local_service_accepts_explicit_disabled_conversion_with_default_geometry() { + let config: ChunkdbConfig = toml::from_str( + "[deployment]\nmode = \"local\"\nmax_node_failures = 0\n[placement]\nmode = \"unsafe_colocated\"\n[conversion]\nenabled = false\n", + ).unwrap(); + config.validate().unwrap(); + assert!(!config.conversion.enabled); + assert!(config.conversion.data_num > 0); + assert!(config.conversion.code_num > 0); +} + #[test] fn conversion_io_transport_rejects_zero_resources() { let mut config = ChunkdbConfig::default(); @@ -98,17 +109,17 @@ fn unsafe_colocated_placement_mode_is_explicit() { assert!(colocated.validate().is_err()); let single: ChunkdbConfig = toml::from_str( - "[deployment]\nmode = \"test_single_node\"\nmax_node_failures = 0\n[placement]\nmode = \"unsafe_colocated\"\n", + "[deployment]\nmode = \"local\"\nmax_node_failures = 0\n[placement]\nmode = \"unsafe_colocated\"\n", ) .expect("explicit test mode parses"); - assert_eq!(single.deployment.mode, DeploymentMode::TestSingleNode); + assert_eq!(single.deployment.mode, DeploymentMode::Local); single.validate().expect("explicit test mode validates"); let mut bad_budget = single; bad_budget.deployment.max_node_failures = 1; assert_eq!( bad_budget.validate(), - Err("test_single_node requires max_node_failures = 0".to_string()) + Err("local requires max_node_failures = 0".to_string()) ); let mut bad_production_budget = ChunkdbConfig::default(); diff --git a/app/crowdb-chunkdb/tests/finalization_test.rs b/app/crowdb-chunkdb/tests/finalization_test.rs new file mode 100644 index 000000000..5a478b911 --- /dev/null +++ b/app/crowdb-chunkdb/tests/finalization_test.rs @@ -0,0 +1,164 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +mod common; + +#[path = "finalization_test/grouped.rs"] +mod grouped; + +use common::cluster::{seed_hardware, ChunkdbHarness, DiskdbServer, KvCluster, DATA_GROUP_ID, STORE_ID}; +use crowdb_chunkdb::{ + conversion::io::ConversionDiskIo, + finalize::FinalizeChunkTaskHandler, + routing::{default_binding_table, BindingCache}, + task::{TaskHandler, TaskOutcome, TaskStore}, +}; +use crowdb_protocol::{ + chunk_task::TASK_KIND_FINALIZE_CHUNK, + chunkdb::rpc::{ChunkState, ChunkType, StripType}, +}; +use std::sync::Arc; + +#[tokio::test] +async fn finalizer_seals_a_readable_acknowledged_tree_frame() { + let cluster = KvCluster::start_pinned().await; + let hardware = cluster.make_hardware_client(); + seed_hardware(&hardware).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let service = cluster.make_service_registry_client(); + let mut diskio = Vec::new(); + for index in 0..4 { + diskio.push( + common::diskio_stub::TestDiskio::start( + &service, + 9000 + index, + 100 + index, + 10 + index, + 1000 + index, + ) + .await, + ); + } + let harness = ChunkdbHarness::start(&cluster).await; + let io = Arc::new(ConversionDiskIo::connect(&service, &hardware).await.unwrap()); + let chunk = harness + .handler + .allocate_chunk( + None, + 1, + 1, + StripType::Mirror, + 0, + 0, + 3, + ChunkType::PageIndex, + 1, + 60_000, + ) + .await + .unwrap(); + let id = chunk.id.unwrap(); + let frame = crowdb_protocol::frame::encode_frame( + crowdb_protocol::frame::FrameMagic::PageIndexV2, + id, + &[0x53; 1024], + 1, + ) + .unwrap(); + let strip = &chunk.strips[0]; + let Some(crowdb_protocol::chunkdb::rpc::Strip::MirrorStrip(mirrors)) = &strip.strip else { + panic!("expected mirrors"); + }; + for segment in &mirrors.segments { + io.write_segment( + segment, + u64::from(strip.unit_kb) * 1024, + bytes::Bytes::from(frame.clone()), + ) + .await + .unwrap(); + } + let cursor = u64::try_from(frame.len()).unwrap(); + harness + .handler + .advance_chunk_write(&id, 1, chunk.modify_ts, cursor, None, 60_000) + .await + .unwrap(); + let bindings = BindingCache::new(); + bindings + .replace(default_binding_table(STORE_ID, DATA_GROUP_ID)) + .unwrap(); + let tasks = TaskStore::new(cluster.make_crowdb_client(), bindings); + let mut task = tasks + .get(&id, TASK_KIND_FINALIZE_CHUNK, &id) + .await + .unwrap() + .unwrap(); + task.eligible_at_ms = 0; + let finalizer = FinalizeChunkTaskHandler::new(Arc::clone(&harness.handler), io); + assert!(matches!(finalizer.execute(&task).await, TaskOutcome::Complete)); + let current = harness.handler.query_chunk(&id).await.unwrap(); + assert_eq!(current.state, ChunkState::Sealed as i32); + assert_eq!(current.acknowledged_cursor, cursor); + assert!(!current.strips.is_empty()); +} + +#[tokio::test] +async fn finalizer_retains_unreadable_acknowledged_frames() { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + let mut chunk = harness + .handler + .allocate_chunk( + None, + 1, + 1, + StripType::Mirror, + 0, + 0, + 3, + ChunkType::PageIndex, + 1, + 60_000, + ) + .await + .unwrap(); + let id = chunk.id.unwrap(); + chunk = harness + .handler + .advance_chunk_write(&id, 1, chunk.modify_ts, 131_174, None, 60_000) + .await + .unwrap(); + let bindings = BindingCache::new(); + bindings + .replace(default_binding_table(STORE_ID, DATA_GROUP_ID)) + .unwrap(); + let tasks = TaskStore::new(cluster.make_crowdb_client(), bindings); + let mut task = tasks + .get(&id, TASK_KIND_FINALIZE_CHUNK, &id) + .await + .unwrap() + .unwrap(); + task.eligible_at_ms = 0; + // Missing DiskIO connectivity is an unavailable observation, never an empty tail. + let finalizer = FinalizeChunkTaskHandler::new( + Arc::clone(&harness.handler), + Arc::new(ConversionDiskIo::empty_for_tests()), + ); + let result = finalizer.execute(&task).await; + match result { + TaskOutcome::Retry { error, .. } => assert!(error.contains("acknowledged"), "{error}"), + TaskOutcome::Complete => { + panic!("finalizer completed after scanning zero readable bytes despite 131174 acknowledged bytes") + } + TaskOutcome::Fail { error, .. } => { + panic!("finalizer must retain acknowledged data and report the inconsistent scan: {error}") + } + } + let current = harness.handler.query_chunk(&id).await.unwrap(); + assert_eq!(current.state, ChunkState::Active as i32); + assert_eq!(current.acknowledged_cursor, 131_174); + assert_eq!(current.strips, chunk.strips); +} diff --git a/app/crowdb-chunkdb/tests/finalization_test/grouped.rs b/app/crowdb-chunkdb/tests/finalization_test/grouped.rs new file mode 100644 index 000000000..5de05113c --- /dev/null +++ b/app/crowdb-chunkdb/tests/finalization_test/grouped.rs @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::common::finalization::TestFinalization; +use crowdb_chunkdb::{ + conversion::io::ConversionDiskIo, + finalize::FinalizeChunkTaskHandler, + task::{TaskHandler, TaskOutcome}, +}; +use crowdb_protocol::{ + chunkdb::rpc::{ChunkState, ChunkType}, + frame::{frame_length, parse_header, FRAME_HEADER_PREFIX_BYTES}, +}; +use std::sync::Arc; + +#[tokio::test] +async fn zero_checkpoint_recovers_complete_groups_and_excludes_partial_large_append() { + for purpose in [ChunkType::Stream, ChunkType::Wal] { + let test = TestFinalization::start(purpose).await; + let prefix = test.group(0, &vec![0x51; 100_000]); + let next = test.group(prefix.len() as u64, &vec![0x62; 180_000]); + let header = parse_header(&next[..FRAME_HEADER_PREFIX_BYTES]).unwrap(); + let mut bytes = prefix.clone(); + bytes.extend_from_slice(&next[..frame_length(header).unwrap()]); + for replica in 0..3 { + test.write(replica, bytes.clone()).await; + } + let outcome = test.run().await; + assert!(matches!(outcome, TaskOutcome::Complete), "{outcome:?}"); + let current = test.current().await; + assert_eq!(current.state, ChunkState::Sealed as i32); + assert_eq!(current.acknowledged_cursor, prefix.len() as u64); + } +} + +#[tokio::test] +async fn shortest_healthy_complete_prefix_wins_without_first_replica_authority() { + let test = TestFinalization::start(ChunkType::Stream).await; + let prefix = test.group(0, b"first"); + let mut longer = prefix.clone(); + longer.extend(test.group(prefix.len() as u64, b"second")); + test.write(0, longer.clone()).await; + test.write(1, prefix.clone()).await; + test.write(2, longer).await; + let outcome = test.run().await; + assert!(matches!(outcome, TaskOutcome::Complete), "{outcome:?}"); + assert_eq!(test.current().await.acknowledged_cursor, prefix.len() as u64); +} + +#[tokio::test] +async fn corrupt_first_replica_cannot_authorize_empty_deletion() { + let test = TestFinalization::start(ChunkType::Stream).await; + let bytes = test.group(0, &vec![0x71; 100_000]); + let mut corrupt = bytes.clone(); + corrupt[40] ^= 1; + test.write(0, corrupt).await; + test.write(1, bytes.clone()).await; + test.write(2, bytes.clone()).await; + let outcome = test.run().await; + assert!(matches!(outcome, TaskOutcome::Complete), "{outcome:?}"); + assert_eq!(test.current().await.acknowledged_cursor, bytes.len() as u64); +} + +#[tokio::test] +async fn conflicting_valid_replicas_retain_the_active_layout() { + let test = TestFinalization::start(ChunkType::Stream).await; + let first = test.group(0, b"first"); + let second = test.group(0, b"other"); + test.write(0, first.clone()).await; + test.write(1, second).await; + test.write(2, first).await; + let outcome = test.run().await; + assert!( + matches!(outcome, TaskOutcome::Retry { ref error, .. } if error.contains("conflict")), + "{outcome:?}" + ); + assert_eq!(test.current().await, test.chunk); +} + +#[tokio::test] +async fn missing_diskio_with_zero_checkpoint_is_not_evidence_of_an_empty_stream() { + let test = TestFinalization::start(ChunkType::Stream).await; + let handler = FinalizeChunkTaskHandler::new( + Arc::clone(&test.harness.handler), + Arc::new(ConversionDiskIo::empty_for_tests()), + ); + let outcome = handler.execute(&test.task).await; + assert!( + matches!(outcome, TaskOutcome::Retry { ref error, .. } if error.contains("not connected")), + "{outcome:?}" + ); + assert_eq!(test.current().await, test.chunk); +} + +#[tokio::test] +async fn a_checkpoint_inside_a_group_cannot_be_used_as_a_seal_boundary() { + let mut test = TestFinalization::start(ChunkType::Wal).await; + let bytes = test.group(0, &vec![0x31; 100_000]); + for replica in 0..3 { + test.write(replica, bytes.clone()).await; + } + test.checkpoint(64 * 1024).await; + let outcome = test.run().await; + assert!(matches!(outcome, TaskOutcome::Retry { .. }), "{outcome:?}"); + assert_eq!(test.current().await, test.chunk); +} + +#[tokio::test] +async fn a_different_owner_generation_cannot_finalize_the_chunk() { + let mut test = TestFinalization::start(ChunkType::Stream).await; + test.task.source_revision += 1; + let outcome = test.run().await; + assert!( + matches!(outcome, TaskOutcome::Retry { error_code: 6, .. }), + "{outcome:?}" + ); + assert_eq!(test.current().await, test.chunk); +} + +#[tokio::test] +async fn missing_drain_proof_retains_empty_and_nonempty_layouts_before_any_scan() { + for (purpose, payload) in [(ChunkType::S3, b"".as_slice()), (ChunkType::Wal, b"durable")] { + let test = TestFinalization::start(purpose).await; + let expected_end = if payload.is_empty() { + 0 + } else { + let data = test.group(0, payload); + for replica in 0..3 { + test.write(replica, data.clone()).await; + } + data.len() as u64 + }; + test.reject_drains(true); + let outcome = test.run().await; + assert!( + matches!(outcome, TaskOutcome::Retry { ref error, .. } if error.contains("completion proof")), + "{outcome:?}" + ); + assert_eq!(test.current().await, test.chunk); + let (drains, reads) = test.io_counts(); + assert!(drains > 0); + assert_eq!(reads, 0); + test.reject_drains(false); + let outcome = test.run().await; + assert!(matches!(outcome, TaskOutcome::Complete), "{outcome:?}"); + let current = test.current().await; + assert_eq!(current.acknowledged_cursor, expected_end); + assert_eq!( + current.state, + if payload.is_empty() { + ChunkState::Deleted + } else { + ChunkState::Sealed + } as i32 + ); + } +} + +#[tokio::test] +async fn a_changed_metadata_snapshot_cannot_commit_an_observed_seal_boundary() { + let mut test = TestFinalization::start(ChunkType::Stream).await; + let observed = test.chunk.clone(); + test.checkpoint(0).await; + let error = test + .harness + .handler + .seal_observed_chunk(&observed, 0) + .await + .unwrap_err(); + assert!(matches!( + error, + crowdb_chunkdb::lifecycle::LifecycleError::StateConflict + )); + assert_eq!(test.current().await, test.chunk); + let sealed = test + .harness + .handler + .seal_observed_chunk(&test.chunk, 0) + .await + .unwrap(); + assert_eq!(sealed.state, ChunkState::Sealed as i32); + assert_eq!(sealed.capacity, 0); + assert!(sealed.strips.is_empty()); +} + +#[tokio::test] +async fn finalization_resumes_empty_deletion_after_a_completed_seal() { + let test = TestFinalization::start(ChunkType::S3).await; + test.harness + .handler + .seal_observed_chunk(&test.chunk, 0) + .await + .unwrap(); + test.reject_drains(true); + let outcome = test.run().await; + assert!(matches!(outcome, TaskOutcome::Complete), "{outcome:?}"); + assert_eq!(test.current().await.state, ChunkState::Deleted as i32); + assert_eq!(test.io_counts(), (0, 0)); +} diff --git a/app/crowdb-chunkdb/tests/full_stack/local_layout.rs b/app/crowdb-chunkdb/tests/full_stack/local_layout.rs new file mode 100644 index 000000000..a88e03622 --- /dev/null +++ b/app/crowdb-chunkdb/tests/full_stack/local_layout.rs @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{seed_hardware, ChunkdbHarness, DeploymentMode, DiskdbServer, KvCluster, LifecycleHandler}; +use crowdb_protocol::chunkdb::rpc::{ChunkType, Strip, StripType}; +use std::{collections::HashSet, sync::Arc}; + +#[tokio::test] +async fn local_layout_allocates_one_or_two_distinct_physical_mirror_copies() { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + let handler = LifecycleHandler::new( + Arc::clone(&harness.store), + Arc::clone(&harness.allocator), + harness.topology.clone(), + ) + .with_deployment_mode(DeploymentMode::Local) + .with_placement_policy(crowdb_chunkdb::selector::FailureDomainPriority::RackFirst, true); + for copies in [1, 2] { + let chunk = handler + .allocate_chunk( + None, + 1024, + 1, + StripType::Mirror, + 0, + 0, + copies, + ChunkType::S3, + 0, + 0, + ) + .await + .unwrap(); + let Some(Strip::MirrorStrip(mirror)) = &chunk.strips[0].strip else { + panic!("local layout must use mirror strips"); + }; + assert_eq!(mirror.segments.len(), copies as usize); + let positions: HashSet<_> = mirror + .segments + .iter() + .map(|segment| { + ( + segment.disk_id, + segment.zone_index, + segment.unit_offset, + segment.unit_count, + ) + }) + .collect(); + assert_eq!(positions.len(), mirror.segments.len()); + } +} diff --git a/app/crowdb-chunkdb/tests/full_stack_test.rs b/app/crowdb-chunkdb/tests/full_stack_test.rs index 44b2b064a..36f09dac5 100644 --- a/app/crowdb-chunkdb/tests/full_stack_test.rs +++ b/app/crowdb-chunkdb/tests/full_stack_test.rs @@ -776,13 +776,15 @@ fn task_value() -> ChunkTaskValue { } } +#[path = "full_stack/local_layout.rs"] +mod local_layout; #[path = "common/reservation_prefetch.rs"] mod reservation_prefetch; struct CompleteTaskHandler; #[tokio::test] -async fn explicit_single_node_mode_rejects_ec_and_extra_copies() { +async fn local_mode_rejects_ec_and_unsupported_mirror_geometry() { let cluster = KvCluster::start_pinned().await; seed_hardware(&cluster.make_hardware_client()).await; let _diskdb = DiskdbServer::start(&cluster).await; @@ -792,7 +794,7 @@ async fn explicit_single_node_mode_rejects_ec_and_extra_copies() { Arc::clone(&harness.allocator), harness.topology.clone(), ) - .with_deployment_mode(DeploymentMode::TestSingleNode) + .with_deployment_mode(DeploymentMode::Local) .with_locks(Arc::new(ChunkLockMap::new( 10_000, Arc::new(LifecycleMetrics::new()), @@ -800,7 +802,7 @@ async fn explicit_single_node_mode_rejects_ec_and_extra_copies() { ))); for (strip_type, copies, size_kb) in [ (StripType::Ec, 0, 1024), - (StripType::Mirror, 2, 1024), + (StripType::Mirror, 3, 1024), (StripType::Mirror, 1, 512), ] { assert!(matches!( @@ -834,7 +836,7 @@ async fn explicit_single_node_mode_rejects_ec_and_extra_copies() { lease_generation: 1, lease_ms: 30_000, }; - for copies in [0, 2] { + for copies in [0, 3] { let group = crowdb_protocol::generate_chunk_id(ChunkType::S3 as u8).to_proto(); assert!(matches!( handler @@ -865,19 +867,20 @@ async fn explicit_single_node_mode_rejects_ec_and_extra_copies() { }; mirror.segments.push(mirror.segments[0]); let operation = crowdb_protocol::generate_chunk_id(ChunkType::S3 as u8).to_proto(); - assert!(matches!( - handler - .replace_chunk_strip_range( - &id, - chunk.modify_ts, - 0, - &chunk.strips, - &[invalid_replacement], - operation, - ) - .await, - Err(LifecycleError::InvalidRequest(_)) - )); + let replacement = handler + .replace_chunk_strip_range( + &id, + chunk.modify_ts, + 0, + &chunk.strips, + &[invalid_replacement], + operation, + ) + .await; + assert!( + matches!(&replacement, Err(LifecycleError::InvalidRequest(_))), + "{replacement:?}" + ); assert_single_node_reservation(&handler, &id, fence).await; } @@ -1199,36 +1202,10 @@ async fn active_chunk_creates_one_deadline_indexed_finalizer() { #[tokio::test] async fn finalizer_reclaims_an_empty_active_chunk() { - let cluster = KvCluster::start_pinned().await; - seed_hardware(&cluster.make_hardware_client()).await; - let _diskdb = DiskdbServer::start(&cluster).await; - let harness = ChunkdbHarness::start(&cluster).await; - let chunk = harness - .handler - .allocate_chunk(None, 1, 1, StripType::Mirror, 0, 0, 3, ChunkType::S3, 19, 60_000) - .await - .unwrap(); - let chunk_id = chunk.id.unwrap(); - let bindings = BindingCache::new(); - bindings - .replace(default_binding_table(STORE_ID, DATA_GROUP_ID)) - .unwrap(); - let tasks = Arc::new(TaskStore::new(cluster.make_crowdb_client(), bindings)); - let mut task = tasks - .get(&chunk_id, TASK_KIND_FINALIZE_CHUNK, &chunk_id) - .await - .unwrap() - .unwrap(); - task.eligible_at_ms = 0; - let finalizer = FinalizeChunkTaskHandler::new( - Arc::clone(&harness.handler), - Arc::new(ConversionDiskIo::empty_for_tests()), - ); - assert!(matches!(finalizer.execute(&task).await, TaskOutcome::Complete)); - assert_eq!( - harness.handler.query_chunk(&chunk_id).await.unwrap().state, - ChunkState::Deleted as i32 - ); + let test = common::finalization::TestFinalization::start(ChunkType::S3).await; + let outcome = test.run().await; + assert!(matches!(outcome, TaskOutcome::Complete), "{outcome:?}"); + assert_eq!(test.current().await.state, ChunkState::Deleted as i32); } #[tokio::test] @@ -1624,7 +1601,7 @@ async fn mirror_range_is_atomically_replaced_by_tentative_ec_strip() { let chunk_id = chunk.id.expect("chunk id"); let chunk = harness .handler - .seal_chunk(&chunk_id, chunk.capacity) + .seal_chunk_bytes(&chunk_id, u64::from(chunk.capacity) * 1024) .await .expect("seal mirror range"); @@ -1697,7 +1674,7 @@ async fn deletion_during_conversion_clears_task_ownership_before_tentative_clean let chunk_id = chunk.id.expect("chunk id"); let chunk = harness .handler - .seal_chunk(&chunk_id, chunk.capacity) + .seal_chunk_bytes(&chunk_id, u64::from(chunk.capacity) * 1024) .await .expect("seal mirror range"); @@ -2505,7 +2482,7 @@ async fn chunkdb_cache_hit_on_second_query() { // Seal (should also be a cache hit after append refreshed the cache). let sealed = harness .handler - .seal_chunk(&chunk_id, 100) + .seal_chunk_bytes(&chunk_id, 100 * 1024) .await .expect("seal_chunk"); assert_eq!(sealed.state, ChunkState::Sealed as i32); @@ -2653,31 +2630,34 @@ async fn reserved_strips_stay_hidden_until_idempotent_confirmation() { .expect("repeat cancellation"); harness .handler - .seal_chunk(&chunk_id, confirmed.chunk.capacity) + .seal_chunk_bytes(&chunk_id, u64::from(confirmed.chunk.capacity) * 1024) .await .expect("seal and remove terminal reservations"); } -async fn assert_legacy_consumed_reservation_is_retained( +async fn assert_consumed_reservation_requires_a_planned_cursor( harness: &ChunkdbHarness, chunk_id: &ChunkId, group_id: &ChunkId, planned_cursor: u64, ) { - let mut legacy = harness + let mut malformed = harness .store .get_reservation_group(chunk_id, group_id) .await .unwrap() .unwrap(); - legacy.planned_cursors[0] = 0; - harness.store.put_reservation_group(&legacy).await.unwrap(); + malformed.planned_cursors[0] = 0; + harness.store.put_reservation_group(&malformed).await.unwrap(); let outcome = harness .handler .recover_expired_reservation_group(chunk_id, group_id, u64::MAX) .await - .unwrap(); - assert_eq!(outcome, ReservationRecovery::Reconciled); + .unwrap_err(); + assert!( + matches!(outcome, LifecycleError::InvalidRequest(ref error) if error.contains("planned cursor")), + "{outcome:?}" + ); let mut retained = harness .store .get_reservation_group(chunk_id, group_id) @@ -2685,6 +2665,7 @@ async fn assert_legacy_consumed_reservation_is_retained( .unwrap() .unwrap(); assert_eq!(retained.states[0], StripReservationState::Consumed as i32); + assert_eq!(retained, malformed); retained.planned_cursors[0] = planned_cursor; harness.store.put_reservation_group(&retained).await.unwrap(); } @@ -2792,7 +2773,8 @@ async fn expired_consumed_reservation_waits_for_reuse_grace() { .await .unwrap(); - assert_legacy_consumed_reservation_is_retained(&harness, &chunk_id, &group_id, planned_cursor).await; + assert_consumed_reservation_requires_a_planned_cursor(&harness, &chunk_id, &group_id, planned_cursor) + .await; assert_consumed_reservation_waits_for_reuse_grace(&harness, &chunk_id, &group_id).await; @@ -3129,7 +3111,7 @@ async fn conversion_reservation_allocates_joint_plan_and_cleans_every_early_tail } let sealed = harness .handler - .seal_chunk(&chunk_id, u32::try_from(cursor / 1024).unwrap()) + .seal_chunk_bytes(&chunk_id, cursor) .await .expect("seal early conversion tail"); assert_eq!(sealed.strips.len(), tail as usize); @@ -3163,8 +3145,8 @@ async fn chunkdb_lock_serializes_concurrent_seal() { let h2 = Arc::clone(&harness.handler); let id1 = chunk_id; let id2 = chunk_id; - let t1 = tokio::spawn(async move { h1.seal_chunk(&id1, 100).await }); - let t2 = tokio::spawn(async move { h2.seal_chunk(&id2, 200).await }); + let t1 = tokio::spawn(async move { h1.seal_chunk_bytes(&id1, 100 * 1024).await }); + let t2 = tokio::spawn(async move { h2.seal_chunk_bytes(&id2, 200 * 1024).await }); let (r1, r2) = tokio::time::timeout(std::time::Duration::from_secs(10), async { (t1.await.expect("task 1"), t2.await.expect("task 2")) }) @@ -3268,7 +3250,7 @@ async fn chunkdb_lock_serializes_concurrent_append_delete() { } #[tokio::test] -async fn chunkdb_shared_writer_cursor_is_fenced_and_orphan_is_sealed() { +async fn chunkdb_shared_writer_cursor_is_fenced_and_renewable() { let cluster = KvCluster::start_pinned().await; let hw = cluster.make_hardware_client(); seed_hardware(&hw).await; @@ -3316,11 +3298,4 @@ async fn chunkdb_shared_writer_cursor_is_fenced_and_orphan_is_sealed() { .await, Err(LifecycleError::StateConflict) )); - tokio::time::sleep(std::time::Duration::from_millis(30)).await; - assert_eq!(harness.handler.seal_expired_writer_chunks().await.unwrap(), 1); - let sealed = harness.handler.query_chunk(&chunk_id).await.unwrap(); - assert_eq!(sealed.state, ChunkState::Sealed as i32); - assert_eq!(sealed.sealed_length, 1024); - assert_eq!(sealed.acknowledged_cursor, 1024 * 1024); - assert_eq!(sealed.closed_strip_sequence, Some(0)); } diff --git a/app/crowdb-chunkdb/tests/journal_handoff_test.rs b/app/crowdb-chunkdb/tests/journal_handoff_test.rs index f6ba666f5..7618a3a11 100644 --- a/app/crowdb-chunkdb/tests/journal_handoff_test.rs +++ b/app/crowdb-chunkdb/tests/journal_handoff_test.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use std::time::Duration; use bytes::Bytes; -use crowdb_chunkdb::storage::decode_chunk_for_tests; +use crowdb_chunkdb::storage::{decode_chunk_for_tests, encode_chunk_for_tests}; use crowdb_kv::cluster::group::{ProposeResult, PxGroup}; use crowdb_kv::cluster::{PxLocalReplica, PxLocalReplicaRole}; use crowdb_kv::kv::{CrowdbTreeConfig, CrowdbTreeEngine, KVEngine}; @@ -22,7 +22,7 @@ fn chunk_key() -> Bytes { } fn metadata(modify_ts: u64, acknowledged_cursor: u64) -> Vec { - bincode::serialize(&Chunk { + encode_chunk_for_tests(&Chunk { id: Some(ChunkId { high: 1, low: 2 }), chunk_type: ChunkType::Wal as i32, modify_ts, diff --git a/app/crowdb-chunkdb/tests/local_service_config_test.rs b/app/crowdb-chunkdb/tests/local_service_config_test.rs new file mode 100644 index 000000000..ea5a8a19a --- /dev/null +++ b/app/crowdb-chunkdb/tests/local_service_config_test.rs @@ -0,0 +1,53 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb::chunkdb_config::{ChunkdbConfig, DeploymentMode, PlacementMode}; +use crowdb_common::config::BaseConfig; +use crowdb_console_shared::{ + config::NodeEntry, + lifecycle::{chunkdb_config, ChunkdbDeployRequest}, +}; + +#[test] +fn local_chunkdb_configuration_is_release_supported_without_ec_or_production_claims() { + let node = NodeEntry { + id: 2, + rack_id: 1, + host: "127.0.0.1".into(), + ssh_port: 2222, + ssh_user: "crowdb".into(), + ssh_credential_ref: None, + ssh_key: None, + ssh_password: None, + }; + let mut request = ChunkdbDeployRequest { + local_deployment: true, + dynamic_ownership: false, + server_id: "chunkdb-2".into(), + instance_id: 2, + http_port: 7300, + rpc_port: 7301, + kv_server_mgmt_seeds: vec!["http://127.0.0.1:10000".into()], + allow_unsafe_ec: false, + rpc_workers: None, + kv_connections: None, + kv_client_rpc_workers: None, + diskdb_connections: None, + diskdb_client_rpc_workers: None, + metrics_interval: None, + }; + let local: ChunkdbConfig = toml::from_str(&chunkdb_config(&request, &node)).unwrap(); + local.validate().unwrap(); + assert_eq!(local.deployment.mode, DeploymentMode::Local); + assert_eq!(local.deployment.max_node_failures, 0); + assert_eq!(local.placement.mode, PlacementMode::UnsafeColocated); + assert!(local.placement.allow_degraded_failure_domains); + assert!(!local.placement.allow_unsafe_ec); + assert!(!local.conversion.enabled); + request.local_deployment = false; + let production: ChunkdbConfig = toml::from_str(&chunkdb_config(&request, &node)).unwrap(); + production.validate().unwrap(); + assert_eq!(production.deployment.mode, DeploymentMode::Production); + assert_eq!(production.placement.mode, PlacementMode::Protected); + assert!(!production.placement.allow_degraded_failure_domains); +} diff --git a/app/crowdb-chunkdb/tests/owner_metadata_test.rs b/app/crowdb-chunkdb/tests/owner_metadata_test.rs index de586a695..4b9ac7402 100644 --- a/app/crowdb-chunkdb/tests/owner_metadata_test.rs +++ b/app/crowdb-chunkdb/tests/owner_metadata_test.rs @@ -1,91 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_chunkdb::storage::decode_chunk_for_tests; -use crowdb_protocol::chunkdb::rpc::{ - Chunk, ChunkStrip, ChunkType, MirrorStrip, PlacementAssessment, PlacementPriority, Strip, - StripCleanupIntent, StripType, -}; +use crowdb_chunkdb::storage::{decode_chunk_for_tests, encode_chunk_for_tests}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkStrip, ChunkType, PlacementAssessment, PlacementPriority}; use crowdb_protocol::common::ChunkId; -use crowdb_protocol::diskdb::rpc::Segment; -use serde::Serialize; - -#[derive(Serialize)] -struct LegacyChunk { - id: Option, - modify_ts: u64, - state: i32, - create_ts_ms: u64, - sealed_ts_ms: u64, - capacity: u32, - sealed_length: u32, - strips: Vec, - chunk_type: i32, - writer_epoch: u64, - acknowledged_cursor: u64, - closed_strip_sequence: Option, - writer_lease_deadline_ms: u64, - next_strip_sequence: u32, - cleanup_intents: Vec, - last_strip_replacement: Option, -} - -#[derive(Serialize)] -struct LegacyChunkStrip { - chunk_offset: u32, - strip_sequence: u32, - unit_kb: u32, - capacity: u32, - create_ts_ms: u64, - sealed_ts_ms: u64, - sealed_length: u32, - strip_type: i32, - strip: Option, - usage_bitmap: Vec, - unavailable_segments: Vec, -} - -#[test] -fn legacy_persisted_chunk_decodes_as_unattributed() { - let legacy = LegacyChunk { - id: Some(ChunkId { high: 1, low: 2 }), - modify_ts: 3, - state: 1, - create_ts_ms: 4, - sealed_ts_ms: 0, - capacity: 1024, - sealed_length: 0, - strips: vec![LegacyChunkStrip { - chunk_offset: 0, - strip_sequence: 0, - unit_kb: 4, - capacity: 4, - create_ts_ms: 4, - sealed_ts_ms: 0, - sealed_length: 0, - strip_type: StripType::Mirror as i32, - strip: Some(Strip::MirrorStrip(MirrorStrip { segments: Vec::new() })), - usage_bitmap: Vec::new(), - unavailable_segments: Vec::new(), - }], - chunk_type: ChunkType::Wal as i32, - writer_epoch: 5, - acknowledged_cursor: 6, - closed_strip_sequence: None, - writer_lease_deadline_ms: 7, - next_strip_sequence: 0, - cleanup_intents: Vec::new(), - last_strip_replacement: None, - }; - let bytes = bincode::serialize(&legacy).unwrap(); - let decoded = decode_chunk_for_tests(&bytes).unwrap(); - assert_eq!(decoded.id, legacy.id); - assert_eq!(decoded.modify_ts, legacy.modify_ts); - assert!(decoded.owner_key.is_empty()); - assert_eq!(decoded.strips.len(), 1); - assert!(decoded.strips[0].placement_assessment.is_none()); - assert!(!decoded.strips[0].placement_repair_required); -} #[test] fn attributed_chunk_persistence_round_trips_owner_key() { @@ -111,6 +29,7 @@ fn attributed_chunk_persistence_round_trips_owner_key() { }], ..Chunk::default() }; - let bytes = bincode::serialize(&chunk).unwrap(); + let bytes = encode_chunk_for_tests(&chunk).unwrap(); + assert_eq!(&bytes[..8], b"CCHK\x02\x00\x00\x00"); assert_eq!(decode_chunk_for_tests(&bytes).unwrap(), chunk); } diff --git a/app/crowdb-chunkdb/tests/readiness_probe_test.rs b/app/crowdb-chunkdb/tests/readiness_probe_test.rs new file mode 100644 index 000000000..eb9430b19 --- /dev/null +++ b/app/crowdb-chunkdb/tests/readiness_probe_test.rs @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::{sync::Arc, time::Duration}; + +use crowdb_chunkdb::{ + range_guard::RangeGuard, + routing::{BindingCache, BindingTable}, + storage::ChunkStore, +}; +use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbClientError, ChunkdbRpcTransport}; +use crowdb_kv::cluster::kv_server::KvServer; +use crowdb_kv_client::{ChunkSlotMapClient, ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_protocol::{ + chunk_domain::ChunkDomain, + chunk_id::{ChunkIdParts, CHUNK_TYPE_S3}, + chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, + chunkdb::rpc::{Chunk, ChunkState}, + common::ChunkId, +}; + +#[path = "common/runtime_context.rs"] +#[allow(dead_code)] +mod runtime_context; +#[path = "common/slot_groups.rs"] +mod slot_groups; + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn readiness_reads_both_domains_without_scanning_or_mutating_the_catalog() { + let test = slot_groups::TestGroups::start().await; + let layout = ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + }; + ChunkSlotMapClient::new(Arc::clone(&test.kv)) + .initialize_service(&layout.service_map().unwrap()) + .await + .unwrap(); + let bindings = BindingCache::new(); + bindings + .replace(BindingTable::new(layout.storage_map().unwrap())) + .unwrap(); + let guard = Arc::new(RangeGuard::new()); + guard.install(&layout.service_map().unwrap(), 1).unwrap(); + let store = ChunkStore::new(Arc::clone(&test.kv), bindings.clone()); + let (context, stop) = runtime_context::runtime_context(Arc::clone(&test.kv), bindings, guard); + let system = context.clone().start(ChunkDomain::System).await.unwrap(); + let user = context.start(ChunkDomain::UserData).await.unwrap(); + let service = Arc::new(user.rpc_service.with_system_runtime(Arc::new(system.rpc_service))); + let server = Arc::new(crowdb_rpc_ffi::RpcServer::new(None)); + server.listen("127.0.0.1", 0).unwrap(); + service.register_handlers(&server); + server.start(); + stop.send(true).unwrap(); + for worker in system.handles.into_iter().chain(user.handles) { + tokio::time::timeout(Duration::from_secs(5), worker) + .await + .unwrap() + .unwrap(); + } + let control = Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))); + control.seed_leader(0, 0, test.server.listen_addr().unwrap().to_string()); + let registry = ServiceRegistryClient::from_shared(Arc::clone(&control)); + registry + .heartbeat_chunkdb(1, &format!("127.0.0.1:{}", server.port())) + .await + .unwrap(); + let client = ChunkdbClient::new(registry, Arc::new(ChunkdbRpcTransport::new())); + let reference = crowdb_kv_client::RangeBindingClient::from_shared(Arc::clone(&control)); + let _ = control.drain_window(); + reference.refresh().await.unwrap(); + let binding_scans = control.drain_window().scan.len(); + client.refresh_routes().await.unwrap(); + assert_eq!( + control.drain_window().scan.len(), + binding_scans, + "a complete refresh must not rescan the registry beyond its binding snapshot" + ); + + for records in [0, 320] { + seed_catalog(&store, records).await; + let _ = test.kv.drain_window(); + Box::pin(client.observe_readiness()).await.unwrap(); + let work = test.kv.drain_window(); + assert_eq!(work.get.len(), 2, "one point read for each domain"); + assert_eq!(work.scan.len(), 0, "readiness must not enumerate catalog records"); + assert_eq!(work.put.len() + work.batch_write.len() + work.delete.len(), 0); + } + // Storage corruption must remain an unhealthy result, rather than being + // confused with an authoritative absent key in an empty catalog. + let mut key = b"/chunk/".to_vec(); + key.extend_from_slice( + &ChunkIdParts { + high: u64::from(CHUNK_TYPE_S3) << 56, + low: 0, + } + .to_bytes(), + ); + test.kv + .put(0, 1, &key, b"corrupt chunk record", None) + .await + .unwrap(); + assert!(matches!( + Box::pin(client.observe_readiness()).await, + Err(ChunkdbClientError::Internal(_)) + )); + server.stop(); +} + +async fn seed_catalog(store: &ChunkStore, count: u64) { + for low in 1..=count { + let id = ChunkId { + high: u64::from(CHUNK_TYPE_S3) << 56, + low, + }; + let mut task = slot_groups::finalize(id); + task.eligible_at_ms = u64::MAX; + store + .create_chunk_with_finalize_task( + &Chunk { + id: Some(id), + modify_ts: 1, + chunk_type: i32::from(CHUNK_TYPE_S3), + state: ChunkState::Sealed as i32, + ..Default::default() + }, + &task, + ) + .await + .unwrap(); + } +} diff --git a/app/crowdb-chunkdb/tests/startup_recovery_test.rs b/app/crowdb-chunkdb/tests/startup_recovery_test.rs new file mode 100644 index 000000000..9259a0f98 --- /dev/null +++ b/app/crowdb-chunkdb/tests/startup_recovery_test.rs @@ -0,0 +1,254 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunkdb::{ + lifecycle::LifecycleError, + range_guard::RangeGuard, + runtime::{DomainRuntime, RuntimeContext}, +}; +use crowdb_protocol::{ + chunk_domain::ChunkDomain, + chunk_slot::{ChunkSlotBootstrap, ChunkStorageGroup}, + chunkdb::rpc::{Chunk, ChunkState}, + common::ChunkId, +}; +use std::{sync::Arc, time::Duration}; + +#[path = "common/cluster.rs"] +mod cluster; +#[path = "common/runtime_context.rs"] +#[allow(dead_code)] +mod runtime_context; +#[path = "common/slot_groups.rs"] +#[allow(dead_code)] +mod slot_groups; +use cluster::{seed_hardware, seeded_dg_ids, ChunkdbHarness, DiskdbServer, KvCluster, INSTANCE_ID}; + +struct TestRecovery { + cluster: KvCluster, + diskdb: DiskdbServer, + harness: ChunkdbHarness, + context: RuntimeContext, + stop: tokio::sync::watch::Sender, + id: ChunkId, + persisted: Chunk, +} + +impl TestRecovery { + async fn start() -> Self { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + let mut persisted = harness + .handler + .allocate_chunk( + None, + 1, + 1, + crowdb_protocol::chunkdb::rpc::StripType::Mirror, + 0, + 0, + 3, + crowdb_protocol::chunkdb::rpc::ChunkType::S3, + 0, + 0, + ) + .await + .unwrap(); + let id = persisted.id.unwrap(); + persisted.state = ChunkState::Deleted as i32; + persisted.modify_ts += 1; + harness.store.put_chunk(&persisted).await.unwrap(); + let guard = Arc::new(RangeGuard::new()); + guard + .install( + &ChunkSlotBootstrap { + service_instances: vec![1], + storage_groups: vec![ChunkStorageGroup { + store_id: 0, + group_id: 1, + }], + } + .service_map() + .unwrap(), + 1, + ) + .unwrap(); + let bindings = crowdb_chunkdb::routing::BindingCache::new(); + bindings + .replace(crowdb_chunkdb::routing::default_binding_table(0, 1)) + .unwrap(); + let (context, stop) = runtime_context::runtime_context(cluster.make_crowdb_client(), bindings, guard); + context + .pool + .update_disk_id_lookup(&harness.topology.snapshot().disk_groups()); + cluster + .make_service_registry_client() + .unregister("diskdb", INSTANCE_ID) + .await + .unwrap(); + Self { + cluster, + diskdb, + harness, + context, + stop, + id, + persisted, + } + } + + async fn assert_waiting( + &self, + start: &mut tokio::task::JoinHandle>, + ) { + assert!( + tokio::time::timeout(Duration::from_millis(1_100), start) + .await + .is_err(), + "missing owner must keep the same startup pending, without admitting requests or exiting" + ); + assert_eq!( + self.harness.store.get_chunk(&self.id).await.unwrap(), + self.persisted, + "recovery must retain the durable free intent until its owner confirms completion" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn startup_waits_for_owner_then_completes_the_persisted_free() { + let test = TestRecovery::start().await; + let context = test.context.clone(); + let mut start = tokio::spawn(context.start(ChunkDomain::UserData)); + test.assert_waiting(&mut start).await; + test.cluster + .make_service_registry_client() + .register_diskdb(INSTANCE_ID, &test.diskdb.rpc_endpoint, &seeded_dg_ids(), &[]) + .await + .unwrap(); + let runtime = tokio::time::timeout(Duration::from_secs(5), start) + .await + .unwrap() + .unwrap() + .unwrap(); + let recovered = test.harness.store.get_chunk(&test.id).await.unwrap(); + assert!(recovered.strips.is_empty()); + assert_eq!(recovered.capacity, 0); + let kv = test.cluster.make_ddb_kv_client(); + for strip in &test.persisted.strips { + let crowdb_protocol::chunkdb::rpc::Strip::MirrorStrip(mirror) = strip.strip.as_ref().unwrap() else { + panic!("expected mirror") + }; + for segment in &mirror.segments { + let records = kv + .read_zone_records((0, 1), &segment.disk_id.unwrap(), segment.zone_index) + .await + .unwrap(); + assert!( + records + .free + .iter() + .any(|record| record.key.unit_offset == segment.unit_offset + && record.value.pre_allocation_ts == segment.allocation_ts + && record.value.previous_owner == segment.owner_chunk + && record.value.unit_count == segment.unit_count), + "recovery must persist the exact incarnation-qualified free fact" + ); + } + } + test.stop.send(true).unwrap(); + for worker in runtime.handles { + tokio::time::timeout(Duration::from_secs(5), worker) + .await + .unwrap() + .unwrap(); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stopping_pending_startup_preserves_intents_without_admitting_requests() { + let test = TestRecovery::start().await; + let context = test.context.clone(); + let mut start = tokio::spawn(context.start(ChunkDomain::UserData)); + test.assert_waiting(&mut start).await; + test.stop.send(true).unwrap(); + assert!(matches!( + tokio::time::timeout(Duration::from_secs(1), start) + .await + .unwrap() + .unwrap(), + Err(LifecycleError::InvalidRequest(_)) + )); + assert_eq!( + test.harness.store.get_chunk(&test.id).await.unwrap(), + test.persisted + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn malformed_durable_segment_is_terminal_instead_of_waiting_for_an_owner() { + let mut test = TestRecovery::start().await; + let crowdb_protocol::chunkdb::rpc::Strip::MirrorStrip(mirror) = + test.persisted.strips[0].strip.as_mut().unwrap() + else { + panic!("expected mirror") + }; + mirror.segments[0].disk_id = None; + test.persisted.modify_ts += 1; + test.harness.store.put_chunk(&test.persisted).await.unwrap(); + assert!(matches!( + tokio::time::timeout(Duration::from_secs(1), test.context.start(ChunkDomain::UserData)) + .await + .unwrap(), + Err(LifecycleError::Cleanup(_)) + )); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn startup_retains_retired_segment_intents_until_the_owner_returns() { + let mut test = TestRecovery::start().await; + let crowdb_protocol::chunkdb::rpc::Strip::MirrorStrip(mirror) = + test.persisted.strips[0].strip.as_ref().unwrap() + else { + panic!("expected mirror") + }; + test.persisted.cleanup_intents = vec![crowdb_protocol::chunkdb::rpc::StripCleanupIntent { + operation_id: Some(ChunkId { high: 31, low: 41 }), + retired_segments: mirror.segments.clone(), + not_before_ms: 0, + }]; + test.persisted.strips.clear(); + test.persisted.state = ChunkState::Sealed as i32; + test.persisted.modify_ts += 1; + test.harness.store.put_chunk(&test.persisted).await.unwrap(); + let context = test.context.clone(); + let mut start = tokio::spawn(context.start(ChunkDomain::UserData)); + test.assert_waiting(&mut start).await; + test.cluster + .make_service_registry_client() + .register_diskdb(INSTANCE_ID, &test.diskdb.rpc_endpoint, &seeded_dg_ids(), &[]) + .await + .unwrap(); + let runtime = tokio::time::timeout(Duration::from_secs(5), start) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(test + .harness + .store + .get_chunk(&test.id) + .await + .unwrap() + .cleanup_intents + .is_empty()); + test.stop.send(true).unwrap(); + for worker in runtime.handles { + tokio::time::timeout(Duration::from_secs(5), worker) + .await + .unwrap() + .unwrap(); + } +} diff --git a/app/crowdb-chunkdb/tests/stream_seal_test.rs b/app/crowdb-chunkdb/tests/stream_seal_test.rs new file mode 100644 index 000000000..e3b4fdec1 --- /dev/null +++ b/app/crowdb-chunkdb/tests/stream_seal_test.rs @@ -0,0 +1,126 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +mod common; + +use common::cluster::{seed_hardware, ChunkdbHarness, DiskdbServer, KvCluster}; +use crowdb_chunkdb::lifecycle::LifecycleError; +use crowdb_protocol::chunk_stream::StreamName; +use crowdb_protocol::chunkdb::rpc::{ChunkState, ChunkType, StripType}; + +#[tokio::test] +async fn stream_and_wal_seal_cannot_shorten_a_persisted_byte_cursor() { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + for purpose in [ChunkType::Stream, ChunkType::Wal] { + let chunk = harness + .handler + .allocate_chunk_owned( + None, + 4, + 1, + StripType::Mirror, + 0, + 0, + 2, + purpose, + 9, + 60_000, + StreamName { + high: 7, + low: purpose as u64, + } + .chunk_owner_key(), + ) + .await + .unwrap(); + let id = chunk.id.unwrap(); + let advanced = harness + .handler + .advance_chunk_write(&id, 9, chunk.modify_ts, 17, None, 60_000) + .await + .unwrap(); + let error = harness.handler.seal_chunk_bytes(&id, 16).await.unwrap_err(); + assert!( + matches!(error, LifecycleError::InvalidRequest(_)), + "{purpose:?}: {error}" + ); + assert_eq!( + harness.handler.query_chunk(&id).await.unwrap(), + advanced, + "rejected shortening must not mutate state, version, strips or cleanup" + ); + let sealed = harness.handler.seal_chunk_bytes(&id, 17).await.unwrap(); + assert_eq!(sealed.state, ChunkState::Sealed as i32); + assert_eq!(sealed.acknowledged_cursor, 17); + assert!(sealed.modify_ts > advanced.modify_ts); + } +} + +#[tokio::test] +async fn stream_seal_can_publish_a_newer_synchronized_byte_boundary() { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + let chunk = harness + .handler + .allocate_chunk_owned( + None, + 4, + 1, + StripType::Mirror, + 0, + 0, + 2, + ChunkType::Stream, + 9, + 60_000, + StreamName { high: 71, low: 73 }.chunk_owner_key(), + ) + .await + .unwrap(); + let id = chunk.id.unwrap(); + assert_eq!(chunk.acknowledged_cursor, 0); + let sealed = harness.handler.seal_chunk_bytes(&id, 73).await.unwrap(); + assert_eq!(sealed.acknowledged_cursor, 73); + assert_eq!(sealed.state, ChunkState::Sealed as i32); + assert_eq!(sealed.sealed_length, 1); +} + +#[tokio::test] +async fn exact_empty_seal_records_zero_bytes_and_zero_physical_capacity() { + let cluster = KvCluster::start_pinned().await; + seed_hardware(&cluster.make_hardware_client()).await; + let _diskdb = DiskdbServer::start(&cluster).await; + let harness = ChunkdbHarness::start(&cluster).await; + let chunk = harness + .handler + .allocate_chunk_owned( + None, + 4, + 1, + StripType::Mirror, + 0, + 0, + 2, + ChunkType::Stream, + 9, + 60_000, + StreamName { high: 79, low: 83 }.chunk_owner_key(), + ) + .await + .unwrap(); + let sealed = harness + .handler + .seal_chunk_bytes(&chunk.id.unwrap(), 0) + .await + .unwrap(); + assert_eq!(sealed.state, ChunkState::Sealed as i32); + assert_eq!(sealed.acknowledged_cursor, 0); + assert_eq!(sealed.sealed_length, 0); + assert_eq!(sealed.capacity, 0); + assert!(sealed.strips.is_empty()); +} diff --git a/app/crowdb-cli/Cargo.toml b/app/crowdb-cli/Cargo.toml index 1f634dd04..4b5348784 100644 --- a/app/crowdb-cli/Cargo.toml +++ b/app/crowdb-cli/Cargo.toml @@ -15,6 +15,8 @@ name = "crowdb-cli" path = "src/main.rs" [dependencies] +futures = "0.3" +uuid = { version = "1", features = ["v4"] } crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } crowdb-common = { path = "../../lib/crowdb-common/rust" } @@ -53,8 +55,8 @@ name = "cluster_cli_test" path = "tests/cluster_cli_test.rs" [[test]] -name = "launch_registry_cli_test" -path = "tests/launch_registry_cli_test.rs" +name = "cluster_target_test" +path = "tests/cluster_target_test.rs" [[test]] name = "direct_cli_test" @@ -77,5 +79,13 @@ name = "s3_cli_test" path = "tests/s3_cli_test.rs" [[test]] -name = "container_cluster_test" -path = "tests/container_cluster_test.rs" +name = "cluster_status_test" +path = "tests/cluster_status_test.rs" + +[[test]] +name = "cluster_lifecycle_test" +path = "tests/cluster_lifecycle_test.rs" + +[[test]] +name = "cluster_workloads_test" +path = "tests/cluster_workloads_test.rs" diff --git a/app/crowdb-cli/src/commands.rs b/app/crowdb-cli/src/commands.rs index f167646e1..0dfcc48aa 100644 --- a/app/crowdb-cli/src/commands.rs +++ b/app/crowdb-cli/src/commands.rs @@ -1,27 +1,23 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Command module dispatch for the four-domain CLI (R126). +//! Command module dispatch for cluster operations and attached workloads. //! //! Each subcommand module defines a `*Verb` enum (clap) and a `run_*` //! async function that builds an [`OpContext`] and delegates to the //! corresponding `ops::*` function. pub(crate) mod bench; -pub(crate) mod chunk; pub(crate) mod cluster; -pub(crate) mod container_cluster; pub(crate) mod kv; -pub(crate) mod launch; pub(crate) mod port_alloc; pub(crate) mod s3; pub(crate) use bench::{run_bench_verb, BenchVerb}; -pub(crate) use chunk::{run_chunk_diskdb_verb, run_chunk_stub_verb, ChunkDiskdbVerb, ChunkStubVerb}; pub(crate) use cluster::{run_cluster_verb, ClusterVerb}; pub(crate) use kv::{ - run_group_verb, run_kv_data_verb, run_kv_server_verb, run_replica_verb, run_store_verb, GroupVerb, - KvDataVerb, KvServerVerb, ReplicaVerb, StoreVerb, + run_group_verb, run_kv_data_verb, run_replica_verb, run_store_verb, GroupVerb, KvDataVerb, ReplicaVerb, + StoreVerb, }; pub(crate) use port_alloc::{run as run_port_alloc, PortAllocArgs}; pub(crate) use s3::{run_s3_verb, S3Verb}; @@ -32,30 +28,35 @@ use crowdb_console_shared::ops::OpContext; use crate::Cli; -/// Build a Group 0 context for hardware operations without reading the old -/// local console state. The CLI endpoint is a discovery seed, while the -/// optional launch registry is validated only as local process policy. +/// Build a Group 0 context from the verified application target. pub(crate) async fn authority_context(cli: &Cli) -> Result { - if let Some(path) = &cli.registry { - crowdb_console_shared::config::web::LaunchRegistry::load(path).map_err(|error| { - eprintln!("error: load launch registry: {error}"); - ExitCode::from(2) - })?; - } - let mgmt_url = format!("http://{}:{}", cli.system_ip, cli.system_port); - let group0_endpoint = match crowdb_console_shared::clients::http::ServerClient::new(&mgmt_url) { - Ok(client) => client - .topology() - .await - .ok() - .and_then(|stores| stores.into_iter().find(|store| store.store_id == 0)) - .and_then(|store| store.listen_addr), - Err(_) => None, - } - .unwrap_or_else(|| format!("{}:{}", cli.system_ip, cli.system_port)); + let (management, rpc) = group0_connection(cli).await?; Ok(OpContext::new( - group0_endpoint, - vec![mgmt_url], + rpc, + management, crowdb_console_shared::ConsoleConfig::default(), )) } + +pub(crate) async fn group0_connection(cli: &Cli) -> Result<(Vec, String), ExitCode> { + let target = cli.target.as_ref().ok_or_else(|| { + eprintln!("error: select --cluster or --management-endpoint before a cluster operation or workload"); + ExitCode::from(2) + })?; + target.require_native_transport().await.map_err(|error| { + eprintln!("error: cluster native transport: {error}"); + ExitCode::from(2) + })?; + let native = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + target.identity.group0_management_endpoints.clone(), + )); + native.seed_leader(0, 0, target.identity.group0_rpc_endpoints[0].clone()); + target.verify_native_authority(&native).await.map_err(|error| { + eprintln!("error: cluster native identity: {error}"); + ExitCode::from(2) + })?; + Ok(( + target.identity.group0_management_endpoints.clone(), + target.identity.group0_rpc_endpoints[0].clone(), + )) +} diff --git a/app/crowdb-cli/src/commands/bench.rs b/app/crowdb-cli/src/commands/bench.rs index 7d62f3793..2919f81da 100644 --- a/app/crowdb-cli/src/commands/bench.rs +++ b/app/crowdb-cli/src/commands/bench.rs @@ -1,10 +1,10 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! `bench` command — KV, RPC, disk, chunk, and chunk-IO workloads. +//! `bench` command — KV, disk, chunk, chunk-IO, and S3 workloads. //! //! Module index only; workload implementations live in `bench/`. Each -//! sub-topic has its own sub-module: `kv`, `rpc`, `disk`, `chunk`, `io`. +//! sub-topic has its own sub-module: `kv`, `disk`, `chunk`, `io`, `s3`. //! Shared helpers (`loader`, `metrics`, `result`, `verb`) live directly //! under `bench/`. @@ -15,7 +15,6 @@ pub mod kv; pub mod loader; pub mod metrics; pub mod result; -pub mod rpc; pub mod s3; pub mod verb; diff --git a/app/crowdb-cli/src/commands/bench/chunk.rs b/app/crowdb-cli/src/commands/bench/chunk.rs index 39320d8cf..a562fcc31 100644 --- a/app/crowdb-cli/src/commands/bench/chunk.rs +++ b/app/crowdb-cli/src/commands/bench/chunk.rs @@ -236,9 +236,21 @@ async fn append( strip_size: 1, strip_count: 1, strip_type: strip_type as i32, - data_num: args.data_num, - code_num: args.code_num, - copy_count: args.copy_count, + data_num: if matches!(args.strip_type, ChunkdbStripMode::Ec) { + args.data_num + } else { + 0 + }, + code_num: if matches!(args.strip_type, ChunkdbStripMode::Ec) { + args.code_num + } else { + 0 + }, + copy_count: if matches!(args.strip_type, ChunkdbStripMode::Mirror) { + args.copy_count + } else { + 0 + }, }) .await .map(|_| ()) @@ -256,9 +268,21 @@ async fn allocate(client: &ChunkdbClient, args: &ChunkdbArgs) -> crowdb_chunkdb_ write_granularity: args.write_granularity_kb, strip_count: args.strip_count, strip_type: strip_type as i32, - data_num: args.data_num, - code_num: args.code_num, - copy_count: args.copy_count, + data_num: if matches!(args.strip_type, ChunkdbStripMode::Ec) { + args.data_num + } else { + 0 + }, + code_num: if matches!(args.strip_type, ChunkdbStripMode::Ec) { + args.code_num + } else { + 0 + }, + copy_count: if matches!(args.strip_type, ChunkdbStripMode::Mirror) { + args.copy_count + } else { + 0 + }, chunk_type: ChunkType::S3 as i32, writer_epoch: 0, writer_lease_ms: 0, @@ -307,7 +331,6 @@ async fn seal(client: &ChunkdbClient, id: ChunkId) -> crowdb_chunkdb_client::Res client .seal_chunk(SealChunkRequest { chunk_id: Some(id), - seal_length: 0, seal_bytes: 0, }) .await diff --git a/app/crowdb-cli/src/commands/bench/disk/db.rs b/app/crowdb-cli/src/commands/bench/disk/db.rs index 21b359468..da0ca92aa 100644 --- a/app/crowdb-cli/src/commands/bench/disk/db.rs +++ b/app/crowdb-cli/src/commands/bench/disk/db.rs @@ -32,6 +32,7 @@ struct TaskResult { latency_ns: Vec, } +#[derive(Clone)] struct TaskContext { mixed: bool, client: Arc, @@ -83,10 +84,19 @@ pub async fn run(cli: &Cli, verb: DiskdbBenchVerb) -> ExitCode { eprintln!("no owned disk-groups discovered"); return ExitCode::FAILURE; } - if let Err(error) = validate_topology(&client, &groups).await { + if let Err(error) = validate_topology(&client, &groups, args.unit_size_bytes).await { eprintln!("invalid diskdb benchmark topology: {error}"); return ExitCode::FAILURE; } + let mixed = matches!(verb, DiskdbBenchVerb::Mix(_)); + if mixed { + // Both endpoints of the space comparison must include prior frees. + // The final compaction also consumes frees predating this invocation. + if let Err(error) = compact_all(&client, &groups).await { + eprintln!("diskdb baseline compaction failed: {error}"); + return ExitCode::FAILURE; + } + } let baseline = match busy_bytes(&client, &groups).await { Ok(value) => value, Err(error) => { @@ -94,7 +104,6 @@ pub async fn run(cli: &Cli, verb: DiskdbBenchVerb) -> ExitCode { return ExitCode::FAILURE; } }; - let mixed = matches!(verb, DiskdbBenchVerb::Mix(_)); let mut metrics = BenchMetrics::new(&cli.log_dir, args.metrics_interval); metrics.start(); let (sender, receiver) = crossbeam_channel::unbounded::(); @@ -104,36 +113,24 @@ pub async fn run(cli: &Cli, verb: DiskdbBenchVerb) -> ExitCode { let sequence = Arc::new(AtomicU64::new(1)); let deadline = Instant::now() + Duration::from_secs(args.duration_secs); let started = Instant::now(); - let mut handles = Vec::with_capacity(args.concurrency); - for task_id in 0..args.concurrency { - let client = Arc::clone(&client); - let groups = Arc::clone(&groups); - let args = args.clone(); - let sender = sender.clone(); - let receiver = receiver.clone(); - let stop = Arc::clone(&stop); - let sequence = Arc::clone(&sequence); - let compacting = Arc::clone(&compacting); - let in_flight = Arc::clone(&in_flight); - let recorder = Arc::clone(&metrics.recorder); - handles.push(tokio::spawn(run_task( - task_id, - TaskContext { - mixed, - client, - groups, - args, - sender, - receiver, - stop, - compacting, - in_flight, - sequence, - recorder, - deadline, - }, - ))); - } + let context = TaskContext { + mixed, + client: Arc::clone(&client), + groups: Arc::clone(&groups), + args: args.clone(), + sender: sender.clone(), + receiver: receiver.clone(), + stop, + compacting, + in_flight, + sequence, + recorder: Arc::clone(&metrics.recorder), + deadline, + }; + let handles = (0..args.concurrency) + .map(|task_id| tokio::spawn(run_task(task_id, context.clone()))) + .collect(); + drop(context); drop(sender); let mut total = collect_results(handles).await; let elapsed = started.elapsed(); @@ -216,8 +213,7 @@ fn report(total: &mut TaskResult, input: &ReportInput<'_>) -> ExitCode { .unwrap_or(0); let stop_reason = if total.exhausted { "exhausted" } else { "deadline" }; println!( - "diskdb bench mode={:?} workload={} stop={} elapsed={:.3}s ops={} ops_per_sec={} allocated={} freed={} live={} errors={} avg_us={} p50_us={} p99_us={} busy_delta={} expected_delta={}", - input.args.mode, + "diskdb bench workload={} stop={} elapsed={:.3}s ops={} ops_per_sec={} allocated={} freed={} live={} errors={} avg_us={} p50_us={} p99_us={} baseline_busy={} final_busy={} busy_delta={} expected_delta={}", if input.mixed { "mix-70-30" } else { "allocate" }, stop_reason, elapsed.as_secs_f64(), @@ -234,6 +230,8 @@ fn report(total: &mut TaskResult, input: &ReportInput<'_>) -> ExitCode { }, percentile(&total.latency_ns, 50) / 1_000, percentile(&total.latency_ns, 99) / 1_000, + input.baseline, + input.final_busy, actual_delta, expected_delta, ); @@ -390,23 +388,31 @@ async fn busy_bytes(client: &DiskdbClient, groups: &[u64]) -> Result Result<(), DiskdbClientError> { - if groups.len() != 3 { - return Err(DiskdbClientError::Rpc(format!( - "expected 3 disk-groups, found {}", - groups.len() - ))); - } +async fn validate_topology( + client: &DiskdbClient, + groups: &[u64], + unit_size: u64, +) -> Result<(), DiskdbClientError> { for group in groups { let response = client.query_disk_group(*group).await?; - let disks = response - .disk_groups + if response.disk_groups.len() != 1 || response.disk_groups[0].disk_group_id != *group { + return Err(DiskdbClientError::Rpc(format!( + "disk-group {group} did not return its current geometry" + ))); + } + let current = &response.disk_groups[0]; + if current.disks.is_empty() || current.allocatable_disk_count == 0 || current.free_bytes == 0 { + return Err(DiskdbClientError::Rpc(format!( + "disk-group {group} has no allocatable storage" + ))); + } + if current + .disks .iter() - .map(|entry| entry.disks.len()) - .sum::(); - if disks != 4 { + .any(|disk| u64::from(disk.unit_size_bytes) != unit_size) + { return Err(DiskdbClientError::Rpc(format!( - "disk-group {group} must contain 4 disks, found {disks}" + "disk-group {group} unit size differs from requested accounting unit {unit_size}" ))); } } diff --git a/app/crowdb-cli/src/commands/bench/io.rs b/app/crowdb-cli/src/commands/bench/io.rs index 17e52d2fb..6e2897d01 100644 --- a/app/crowdb-cli/src/commands/bench/io.rs +++ b/app/crowdb-cli/src/commands/bench/io.rs @@ -35,7 +35,7 @@ async fn connect( diskio_connections_per_endpoint: usize, diskio_rpc_workers: u32, ) -> Result { - let seeds = vec![format!("http://{}:{}", cli.system_ip, cli.system_port)]; + let (seeds, _) = crate::commands::group0_connection(cli).await?; ChunkIoClient::connect(ChunkIoClientConfig { management_seeds: seeds, diskio_connections_per_endpoint, @@ -54,6 +54,7 @@ fn large_policy( block_size: usize, data_num: usize, code_num: usize, + mirror_copies: Option, ) -> LargeWritePolicy { LargeWritePolicy { ec_scheme: EcScheme::new(data_num, code_num), @@ -61,6 +62,7 @@ fn large_policy( max_chunk_size: object_chunk_size, read_buffer_size: block_size, max_cached_buffer: block_size.saturating_mul(data_num), + large_mirror_copies: mirror_copies, ..ChunkClientConfig::new(crowdb_protocol::chunkdb::rpc::ChunkType::S3) }), } @@ -91,7 +93,13 @@ async fn run_large_write(cli: &Cli, args: ChunkioArgs) -> ExitCode { Ok(client) => client, Err(code) => return code, }; - let mut policy = large_policy(args.chunk_size, args.block_size, args.data_num, args.code_num); + let mut policy = large_policy( + args.chunk_size, + args.block_size, + args.data_num, + args.code_num, + args.mirror_copies, + ); Arc::make_mut(&mut policy.client).prefetch_strips_per_chunk = args.prefetch_strips_per_chunk; let mut metrics = BenchMetrics::new(&cli.log_dir, args.metrics_interval); let client = client.with_metrics(&metrics.chunk_io); @@ -213,7 +221,13 @@ async fn run_read(cli: &Cli, args: ChunkioReadArgs, workload: ReadBenchmarkWorkl mixed_large_percent: args.mixed_large_percent, seed: args.seed, workload, - large_policy: large_policy(args.chunk_size, args.block_size, args.data_num, args.code_num), + large_policy: large_policy( + args.chunk_size, + args.block_size, + args.data_num, + args.code_num, + args.mirror_copies, + ), }, ) .await; diff --git a/app/crowdb-cli/src/commands/bench/kv/client.rs b/app/crowdb-cli/src/commands/bench/kv/client.rs index ce6fe8dfc..d8c5031e2 100644 --- a/app/crowdb-cli/src/commands/bench/kv/client.rs +++ b/app/crowdb-cli/src/commands/bench/kv/client.rs @@ -38,7 +38,7 @@ impl Default for KvClientTunables { } } -/// Build a `CrowdbKvClient` from `--system-*` with the given +/// Build a `CrowdbKvClient` from the resolved cluster target with the given /// `read_endpoint_policy`. /// /// # Errors @@ -48,22 +48,7 @@ pub(crate) async fn build_kv_client( read_endpoint_policy: ReadEndpointPolicy, tunables: &KvClientTunables, ) -> Result { - let seed = format!("http://{}:{}", cli.system_ip, cli.system_port); - let server = crowdb_console_shared::clients::http::ServerClient::new(&seed).map_err(|error| { - eprintln!("bench kv client: {error}"); - ExitCode::from(2) - })?; - let effective_g0 = server - .topology() - .await - .ok() - .and_then(|stores| stores.into_iter().find(|store| store.store_id == 0)) - .and_then(|store| store.listen_addr) - .ok_or_else(|| { - eprintln!("bench kv client: Group 0 endpoint unavailable at {seed}"); - ExitCode::from(2) - })?; - let seeds = vec![seed]; + let (seeds, effective_g0) = crate::commands::group0_connection(cli).await?; let mut client_config = ClientConfig::new(seeds); client_config.read_endpoint_policy = read_endpoint_policy; diff --git a/app/crowdb-cli/src/commands/bench/kv/prepare.rs b/app/crowdb-cli/src/commands/bench/kv/prepare.rs index f7fbbe4cf..b0ddecfed 100644 --- a/app/crowdb-cli/src/commands/bench/kv/prepare.rs +++ b/app/crowdb-cli/src/commands/bench/kv/prepare.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! `bench kv prepare` — pre-populate `--keys` keys into store 0 / group 0. +//! `bench kv prepare` writes only the requested dataset into the selected group. //! //! Concurrent bulk-put loop with `--concurrency` tasks. No latency //! histogram — this is a setup step, not a measurement. Reports total @@ -32,24 +32,6 @@ pub async fn run(cli: &Cli, args: PrepareArgs) -> ExitCode { let value_size = args.value_size.max(1); - // Warm up: wait for the leader to be ready before bulk-putting. - // local-deploy may return before the leader is fully stable. - let warm_key = b"k__warmup__"; - let warm_val = vec![0u8; value_size]; - for _ in 0..50 { - if client - .put(store_id, group_id, warm_key, &warm_val, None) - .await - .is_ok() - { - // Extra settle time: the leader may be elected but not yet - // processing writes at full capacity. - tokio::time::sleep(Duration::from_millis(200)).await; - break; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - let next_id = Arc::new(AtomicU64::new(0)); let ok_count = Arc::new(AtomicU64::new(0)); let err_count = Arc::new(AtomicU64::new(0)); diff --git a/app/crowdb-cli/src/commands/bench/kv/read.rs b/app/crowdb-cli/src/commands/bench/kv/read.rs index 4d7073300..169987c36 100644 --- a/app/crowdb-cli/src/commands/bench/kv/read.rs +++ b/app/crowdb-cli/src/commands/bench/kv/read.rs @@ -101,8 +101,7 @@ pub async fn run(cli: &Cli, args: ReadArgs) -> ExitCode { }; let json = serde_json::to_value(&result).unwrap_or_default(); tracing::info!(report = %json, "bench_report"); - println!("{result}"); - ExitCode::SUCCESS + result.finish(&cli.log_dir) } #[allow(clippy::too_many_arguments)] diff --git a/app/crowdb-cli/src/commands/bench/kv/scan.rs b/app/crowdb-cli/src/commands/bench/kv/scan.rs index cd84dc086..3c43826f9 100644 --- a/app/crowdb-cli/src/commands/bench/kv/scan.rs +++ b/app/crowdb-cli/src/commands/bench/kv/scan.rs @@ -165,8 +165,7 @@ pub async fn run(cli: &Cli, args: ScanArgs) -> ExitCode { }; let json = serde_json::to_value(&result).unwrap_or_default(); tracing::info!(report = %json, "bench_report"); - println!("{result}"); - ExitCode::SUCCESS + result.finish(&cli.log_dir) } /// Parse a `--value-size-mix` string like `64:70,1024:20,16384:10` diff --git a/app/crowdb-cli/src/commands/bench/kv/write.rs b/app/crowdb-cli/src/commands/bench/kv/write.rs index 0c38e51ed..58ff06a39 100644 --- a/app/crowdb-cli/src/commands/bench/kv/write.rs +++ b/app/crowdb-cli/src/commands/bench/kv/write.rs @@ -164,8 +164,7 @@ pub async fn run(cli: &Cli, args: WriteArgs) -> ExitCode { server_metrics, }; metrics.write_report(&result); - println!("{result}"); - ExitCode::SUCCESS + result.finish(&cli.log_dir) } /// Build a deterministic value for key `id`: byte `i` = `(id + i) % 256`. diff --git a/app/crowdb-cli/src/commands/bench/result.rs b/app/crowdb-cli/src/commands/bench/result.rs index f3ca0736b..97cb5fd73 100644 --- a/app/crowdb-cli/src/commands/bench/result.rs +++ b/app/crowdb-cli/src/commands/bench/result.rs @@ -31,6 +31,25 @@ pub struct BenchResult { pub server_metrics: Option, } +impl BenchResult { + /// Keep the console report and a structured latest-run artifact for measurement tools. + #[must_use] + pub fn finish(&self, directory: &std::path::Path) -> std::process::ExitCode { + println!("{self}"); + let path = directory.join("report.json"); + let result = (|| -> Result<(), Box> { + let file = std::fs::File::create(&path)?; + serde_json::to_writer_pretty(file, self)?; + Ok(()) + })(); + if let Err(error) = result { + eprintln!("error: retain benchmark report {}: {error}", path.display()); + return std::process::ExitCode::from(2); + } + std::process::ExitCode::SUCCESS + } +} + #[derive(Debug, Default, Serialize)] pub struct BenchOps { #[serde(skip_serializing_if = "Option::is_none")] diff --git a/app/crowdb-cli/src/commands/bench/s3.rs b/app/crowdb-cli/src/commands/bench/s3.rs index 05026ae6f..c5f4cdc75 100644 --- a/app/crowdb-cli/src/commands/bench/s3.rs +++ b/app/crowdb-cli/src/commands/bench/s3.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Thin CLI adapter for the shared memory-backed S3 benchmark. +//! Thin CLI adapter for an attached, authenticated S3 workload. use std::process::ExitCode; use std::time::Duration; @@ -11,7 +11,7 @@ use crowdb_console_shared::ops::s3_bench::{self, MixWeights, S3BenchConfig, S3Be use super::verb::{S3Args, S3BenchVerb}; use crate::Cli; -pub async fn run(_cli: &Cli, verb: S3BenchVerb) -> ExitCode { +pub async fn run(cli: &Cli, verb: S3BenchVerb) -> ExitCode { let (workload, args) = match verb { S3BenchVerb::Write(args) => (S3BenchWorkload::Write, args), S3BenchVerb::Read(args) => (S3BenchWorkload::Read, args), @@ -19,7 +19,7 @@ pub async fn run(_cli: &Cli, verb: S3BenchVerb) -> ExitCode { S3BenchVerb::List(args) => (S3BenchWorkload::List, args), S3BenchVerb::Mix(args) => (S3BenchWorkload::Mix, args), }; - match execute(workload, args).await { + match execute(cli, workload, args).await { Ok(()) => ExitCode::SUCCESS, Err(error) => { eprintln!("error: {error}"); @@ -28,23 +28,30 @@ pub async fn run(_cli: &Cli, verb: S3BenchVerb) -> ExitCode { } } -async fn execute(workload: S3BenchWorkload, args: S3Args) -> crowdb_console_shared::error::Result<()> { +async fn execute( + cli: &Cli, + workload: S3BenchWorkload, + args: S3Args, +) -> crowdb_console_shared::error::Result<()> { + let client = crate::commands::s3::client(cli).await?; let mix = MixWeights::parse(&args.ratio)?; let output = args.output.clone(); - let result = s3_bench::run(S3BenchConfig { - work_dir: args.work_dir, - workload, - object_size: args.object_size, - dataset_objects: args.dataset_objects, - concurrency: args.concurrency, - operations: args.operations, - duration: Duration::from_secs(args.duration_secs), - warmup_operations: args.warmup_operations, - seed: args.seed, - memory_budget_bytes: args.memory_budget_bytes, - list_limit: args.list_limit, - mix, - }) + let result = s3_bench::run( + S3BenchConfig { + workload, + object_size: args.object_size, + dataset_objects: args.dataset_objects, + concurrency: args.concurrency, + operations: args.operations, + duration: Duration::from_secs(args.duration_secs), + warmup_operations: args.warmup_operations, + seed: args.seed, + max_dataset_bytes: args.max_dataset_bytes, + list_limit: args.list_limit, + mix, + }, + client, + ) .await?; let json = serde_json::to_vec_pretty(&result) .map_err(|error| crowdb_console_shared::error::Error::Config(error.to_string()))?; @@ -61,12 +68,12 @@ async fn execute(workload: S3BenchWorkload, args: S3Args) -> crowdb_console_shar result.operations_per_second ); println!( - " object_size: {} dataset_objects: {} peak_resident: {} / {} bytes", - result.object_size, result.dataset_objects, result.peak_resident_bytes, result.memory_budget_bytes + " object_size: {} dataset_objects: {} declared_dataset_bound: {} bytes", + result.object_size, result.dataset_objects, result.max_dataset_bytes ); println!( - " backing: kv={} wal={} diskio={} chunk-kv={}", - result.backing.kv, result.backing.wal, result.backing.diskio, result.backing.chunk_kv + " invocation bucket: {}; server resource usage is not observed", + result.bucket ); if result.total_errors > 0 { println!( diff --git a/app/crowdb-cli/src/commands/bench/verb.rs b/app/crowdb-cli/src/commands/bench/verb.rs index 3de57d14b..6c193de46 100644 --- a/app/crowdb-cli/src/commands/bench/verb.rs +++ b/app/crowdb-cli/src/commands/bench/verb.rs @@ -10,14 +10,12 @@ use clap::{Subcommand, ValueEnum}; use crate::Cli; -/// Top-level `bench` verb: either an RPC echo workload or a KV workload. +/// Workloads attached to a selected application cluster. #[derive(Subcommand, Debug)] pub enum BenchVerb { - /// KV-layer workload (prepare / read / write / scan / clean). + /// KV-layer workload (prepare / read / write / scan). #[command(subcommand)] Kv(KvBenchVerb), - /// Raw crowdb-rpc echo throughput benchmark against a fb-server. - Rpc(RpcArgs), /// Distributed disk-block allocation benchmark. #[command(subcommand)] Diskdb(DiskdbBenchVerb), @@ -27,7 +25,7 @@ pub enum BenchVerb { /// End-to-end chunk data IO benchmark. #[command(subcommand)] Chunkio(ChunkioBenchVerb), - /// S3 HTTP workloads over an invocation-owned memory cluster. + /// Authenticated S3 HTTP workloads on the selected cluster. #[command(subcommand)] S3(S3BenchVerb), } @@ -43,9 +41,6 @@ pub enum S3BenchVerb { #[derive(clap::Args, Debug, Clone)] pub struct S3Args { - /// Empty directory used for the invocation-owned memory cluster. - #[arg(long)] - pub work_dir: std::path::PathBuf, #[arg(long, default_value_t = 4 * 1024)] pub object_size: usize, #[arg(long, default_value_t = 128)] @@ -62,7 +57,7 @@ pub struct S3Args { #[arg(long, default_value_t = 1)] pub seed: u64, #[arg(long, default_value_t = 2 * 1024 * 1024 * 1024)] - pub memory_budget_bytes: u64, + pub max_dataset_bytes: u64, #[arg(long, default_value_t = 100)] pub list_limit: usize, #[arg(long, default_value = "w20R70RR5L5")] @@ -112,6 +107,9 @@ pub struct ChunkioArgs { pub data_num: usize, #[arg(long, default_value_t = 1)] pub code_num: usize, + /// Use large mirror strips instead of EC, with the selected copy count. + #[arg(long, value_parser = clap::value_parser!(u32).range(1..=5))] + pub mirror_copies: Option, #[arg(long, default_value_t = 1)] pub seed: u8, #[arg(long, default_value_t = 1)] @@ -199,6 +197,9 @@ pub struct ChunkioReadArgs { pub data_num: usize, #[arg(long, default_value_t = 1)] pub code_num: usize, + /// Use large mirror strips instead of EC, with the selected copy count. + #[arg(long, value_parser = clap::value_parser!(u32).range(1..=5))] + pub mirror_copies: Option, #[arg(long, default_value_t = 1)] pub seed: u8, #[arg(long, default_value_t = 1)] @@ -281,9 +282,6 @@ pub struct DiskdbArgs { /// Expected unit size, used for space verification. #[arg(long, default_value_t = 1_048_576)] pub unit_size_bytes: u64, - /// Storage mode configured by cluster initialization. - #[arg(long, value_enum, default_value_t = DiskdbBenchMode::Mem)] - pub mode: DiskdbBenchMode, /// Deterministic workload seed. #[arg(long, default_value_t = 1)] pub seed: u64, @@ -292,12 +290,6 @@ pub struct DiskdbArgs { pub metrics_interval: u64, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] -pub enum DiskdbBenchMode { - Mem, - Block, -} - #[derive(Subcommand, Debug)] pub enum KvBenchVerb { /// Pre-populate `--keys` keys into store 0 / group 0. @@ -310,40 +302,6 @@ pub enum KvBenchVerb { Scan(ScanArgs), } -#[derive(clap::Args, Debug)] -pub struct RpcArgs { - /// Run duration in seconds. - #[arg(long, default_value_t = 10)] - pub duration_secs: u64, - /// Number of loader coroutines / tasks. - #[arg(long, default_value_t = 1)] - pub loader_num: usize, - /// Number of client connections to the fb-server. - #[arg(long, default_value_t = 1)] - pub connections: usize, - /// Echo payload size in bytes. - #[arg(long, default_value_t = 128)] - pub value_size: usize, - /// Client transport I/O engines (must match the fb-server). - #[arg(long, default_value_t = 1)] - pub io_engines: u32, - /// Client transport I/O worker threads (must match the fb-server). - #[arg(long, default_value_t = 1)] - pub io_workers: u32, - /// Concurrency model: `coroutine` (C++ coroutines) or `tokio`. - #[arg(long, value_enum, default_value_t = BenchMode::Coroutine)] - pub mode: BenchMode, - /// fb-server port to connect to. - #[arg(long)] - pub server_port: u16, - /// Enable TCP Nagle (coalesce small frames per writev). - #[arg(long, default_value_t = false)] - pub enable_nagle: bool, - /// Metrics flush interval in seconds. 0 disables the metrics log. - #[arg(long, default_value_t = 1)] - pub metrics_interval: u64, -} - #[derive(clap::Args, Debug)] pub struct PrepareArgs { /// Number of keys to write into the target store/group. @@ -482,12 +440,6 @@ pub struct ScanArgs { pub group: u64, } -#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] -pub enum BenchMode { - Coroutine, - Tokio, -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] pub enum BenchReadMode { Linearizable, @@ -514,7 +466,6 @@ pub enum BenchScanDirection { pub async fn run_bench_verb(cli: &Cli, verb: BenchVerb) -> ExitCode { match verb { - BenchVerb::Rpc(args) => super::rpc::run(cli, args).await, BenchVerb::Diskdb(verb) => super::disk::db::run(cli, verb).await, BenchVerb::Chunkdb(verb) => super::chunk::run(cli, verb).await, BenchVerb::Chunkio(verb) => super::io::run(cli, verb).await, diff --git a/app/crowdb-cli/src/commands/chunk.rs b/app/crowdb-cli/src/commands/chunk.rs deleted file mode 100644 index 84d7bcd6c..000000000 --- a/app/crowdb-cli/src/commands/chunk.rs +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! `chunk` domain — chunk-disk allocation and chunk stub service -//! management. - -pub mod diskdb; -pub mod stub; - -pub(crate) use diskdb::{run_chunk_diskdb_verb, ChunkDiskdbVerb}; -pub(crate) use stub::{run_chunk_stub_verb, ChunkStubVerb}; diff --git a/app/crowdb-cli/src/commands/chunk/diskdb.rs b/app/crowdb-cli/src/commands/chunk/diskdb.rs deleted file mode 100644 index 737697fe4..000000000 --- a/app/crowdb-cli/src/commands/chunk/diskdb.rs +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! `chunk diskdb` command handlers — diskdb lifecycle + maintenance. - -use std::process::ExitCode; - -use clap::Subcommand; - -use crowdb_console_shared::ops::chunk; - -use crate::commands::authority_context; -use crate::commands::launch::{self, LaunchVerb}; -use crate::Cli; - -#[derive(Subcommand, Debug)] -pub enum ChunkDiskdbVerb { - Deploy { - #[arg(short = 'n', long)] - node: u64, - }, - Start { - #[arg(short = 'n', long)] - node: u64, - }, - Restart { - #[arg(short = 'n', long)] - node: u64, - }, - Stop { - #[arg(short = 'n', long)] - node: u64, - }, - Delete { - #[arg(short = 'n', long)] - node: u64, - }, - /// List living diskdb instances discovered from the group-0 - /// service registry. Pass `--endpoint` to bypass discovery and - /// query a specific diskdb directly. - List { - /// Explicit diskdb RPC endpoint (e.g. `127.0.0.1:11000`). - /// When set, discovery is bypassed. - #[arg(long)] - endpoint: Option, - }, - Usage, - ScanStatus, - Scan, - Recalc, - Compact, - Rebuild, -} - -pub async fn run_chunk_diskdb_verb(cli: &Cli, verb: ChunkDiskdbVerb) -> ExitCode { - match verb { - ChunkDiskdbVerb::List { endpoint } => run_list(cli, endpoint.as_deref()).await, - ChunkDiskdbVerb::Deploy { node } | ChunkDiskdbVerb::Start { node } => { - launch::run( - cli, - LaunchVerb::Start { - node, - service: "diskdb".into(), - }, - ) - .await - } - ChunkDiskdbVerb::Restart { node } => { - launch::run( - cli, - LaunchVerb::Restart { - node, - service: "diskdb".into(), - }, - ) - .await - } - ChunkDiskdbVerb::Stop { node } => { - launch::run( - cli, - LaunchVerb::Stop { - node, - service: "diskdb".into(), - }, - ) - .await - } - other => { - eprintln!("chunk diskdb {other:?} — not yet implemented (Phase 3)"); - ExitCode::from(1) - } - } -} - -async fn run_list(cli: &Cli, explicit_endpoint: Option<&str>) -> ExitCode { - let ctx = match authority_context(cli).await { - Ok(ctx) => ctx, - Err(code) => return code, - }; - - // If an explicit endpoint is given, bypass discovery and just - // print it (the caller asked for that specific instance). - if let Some(ep) = explicit_endpoint { - println!("diskdb endpoint (explicit): {ep}"); - return ExitCode::SUCCESS; - } - - // Discover all living diskdb instances from group-0. - let instances = match chunk::discover_all_diskdb_endpoints(&ctx).await { - Ok(v) => v, - Err(e) => { - eprintln!("error: discover diskdb instances: {e}"); - return ExitCode::from(1); - } - }; - - if instances.is_empty() { - eprintln!("no living diskdb instances — register one with crowdb-web or start crowdb-diskdb"); - return ExitCode::from(1); - } - - println!("living diskdb instances ({}):", instances.len()); - for (id, ep) in &instances { - println!(" instance {id}: {ep}"); - } - - ExitCode::SUCCESS -} diff --git a/app/crowdb-cli/src/commands/chunk/stub.rs b/app/crowdb-cli/src/commands/chunk/stub.rs deleted file mode 100644 index a7900561b..000000000 --- a/app/crowdb-cli/src/commands/chunk/stub.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! `chunk chunkdb/diskio` + data-plane stub command handlers. - -use std::process::ExitCode; - -use clap::Subcommand; - -use crate::commands::launch::{self, LaunchVerb}; -use crate::Cli; - -#[derive(Subcommand, Debug)] -pub enum ChunkStubVerb { - #[command(subcommand)] - Chunkdb(ServiceVerb), - #[command(subcommand)] - Diskio(ServiceVerb), - Allocate, - Free, - Write, - Read, - Gc, -} - -#[derive(Subcommand, Debug)] -pub enum ServiceVerb { - Deploy { - #[arg(short = 'n', long)] - node: u64, - }, - Start { - #[arg(short = 'n', long)] - node: u64, - }, - Restart { - #[arg(short = 'n', long)] - node: u64, - }, - Stop { - #[arg(short = 'n', long)] - node: u64, - }, - List, -} - -pub async fn run_chunk_stub_verb(cli: &Cli, verb: ChunkStubVerb) -> ExitCode { - match verb { - ChunkStubVerb::Chunkdb(verb) => run_service(cli, "chunkdb", verb).await, - ChunkStubVerb::Diskio(verb) => run_service(cli, "diskio", verb).await, - other => { - eprintln!("chunk {other:?} — not yet implemented (Phase 3)"); - ExitCode::from(1) - } - } -} - -async fn run_service(cli: &Cli, service: &str, verb: ServiceVerb) -> ExitCode { - if matches!(verb, ServiceVerb::List) { - return run_list(cli, service).await; - } - let service = service.into(); - let verb = match verb { - ServiceVerb::Deploy { node } | ServiceVerb::Start { node } => LaunchVerb::Start { node, service }, - ServiceVerb::Restart { node } => LaunchVerb::Restart { node, service }, - ServiceVerb::Stop { node } => LaunchVerb::Stop { node, service }, - ServiceVerb::List => unreachable!(), - }; - launch::run(cli, verb).await -} - -async fn run_list(cli: &Cli, service: &str) -> ExitCode { - let ctx = match crate::commands::authority_context(cli).await { - Ok(ctx) => ctx, - Err(code) => return code, - }; - match ctx.sysmd().read_service_instances(service).await { - Ok(instances) => { - println!("living {service} instances ({}):", instances.len()); - for (id, instance) in instances { - println!(" instance {id}: {}", instance.rpc_endpoint); - } - ExitCode::SUCCESS - } - Err(error) => { - eprintln!("error: read {service} registration: {error}"); - ExitCode::from(2) - } - } -} diff --git a/app/crowdb-cli/src/commands/cluster.rs b/app/crowdb-cli/src/commands/cluster.rs index 3b4e63fa2..26c842af6 100644 --- a/app/crowdb-cli/src/commands/cluster.rs +++ b/app/crowdb-cli/src/commands/cluster.rs @@ -1,10 +1,12 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! `cluster` domain — cluster-level ops: init, destroy, clean, status, -//! topology, plus hardware subcommands (rack/node/disk-group/disk). +//! Common OCI cluster lifecycle, observation and hardware metadata. pub mod hardware; +mod local; +mod status; +pub(crate) mod target; pub(crate) use hardware::{ run_disk_group_verb, run_disk_verb, run_node_verb, run_rack_verb, DiskGroupVerb, DiskVerb, NodeVerb, @@ -18,183 +20,31 @@ use clap::Subcommand; use crate::commands::authority_context; use crate::Cli; -async fn local_deploy_context( - cli: &Cli, - existing_cluster: bool, -) -> Result { - if !existing_cluster { - let mgmt = format!("http://{}:{}", cli.system_ip, cli.system_port); - let rpc_hint = format!("{}:{}", cli.system_ip, cli.system_port); - return Ok(crowdb_console_shared::ops::OpContext::new( - rpc_hint, - vec![mgmt], - crowdb_console_shared::ConsoleConfig::default(), - )); - } - let ctx = authority_context(cli).await?; - { - let racks = crowdb_console_shared::ops::hardware::list_racks_from_group0(&ctx) - .await - .map_err(|error| { - eprintln!("error: read Group 0 racks: {error}"); - ExitCode::from(2) - })?; - let nodes = crowdb_console_shared::ops::hardware::list_nodes_from_group0(&ctx, None) - .await - .map_err(|error| { - eprintln!("error: read Group 0 nodes: {error}"); - ExitCode::from(2) - })?; - let mut servers = Vec::with_capacity(nodes.len()); - for node in &nodes { - let url = ctx.live_node_mgmt_url(node.id).await.map_err(|error| { - eprintln!("error: resolve live KV node {}: {error}", node.id); - ExitCode::from(2) - })?; - let mut server = crowdb_console_shared::config::ServerEntry::new(node.id.to_string(), url); - server.node_id = Some(node.id); - servers.push(server); - } - let mut config = ctx.config_mut(); - config.racks = racks; - config.nodes = nodes; - config.servers = servers; - } - Ok(ctx) -} - #[derive(Subcommand, Debug)] pub enum ClusterVerb { - /// Initialize the cluster by bootstrapping group 0 on the listed nodes. - Init { - #[arg(short = 'n', long, value_delimiter = ',')] - nodes: Vec, - /// Versioned bootstrap topology input for the first registry-mode init. - #[arg(long, value_name = "PATH")] - bootstrap_file: Option, - }, - /// Deploy a local N-node KV cluster on 127.0.0.1 (forks - /// `crowdb-kv-server` on each node, bootstraps group 0). - /// With `-t rpc`, deploys a standalone crowdb-rpc-fb-server (echo) - /// for RPC regression bench. - #[command(name = "local-deploy")] - LocalDeploy { - #[arg(short = 'n', long, default_value_t = 3)] - nodes: usize, - /// Service type: `kv`, `diskdb`, `chunkdb`, `combined`, or `rpc`. - #[arg(short = 't', long, default_value = "kv")] - service_type: String, - /// [rpc] Listen port. 0 = auto-allocate. - #[arg(long, default_value_t = 0)] - rpc_port: u16, - /// [rpc] Independent epoll instances (round-robin). - #[arg(long, default_value_t = 1)] - io_engines: u32, - /// [rpc] Total I/O worker threads. - #[arg(long, default_value_t = 1)] - io_workers: u32, - /// [rpc] Enable Nagle's algorithm (disable `TCP_NODELAY`). - #[arg(long, default_value_t = false)] - enable_nagle: bool, - /// [kv] `--rpc-workers` for the spawned server. 0 = server default (2). - #[arg(long, default_value_t = 0)] - rpc_workers: u32, - /// [diskio] RPC I/O worker threads. 0 = server default (4). - #[arg(long, default_value_t = 0)] - diskio_rpc_workers: u32, - /// [diskio] Backend for auto-discovered local disks (null|mem). - #[arg(long, default_value = "null", value_parser = ["null", "mem"])] - diskio_dummy_disk_type: String, - /// [kv] `--peer-pool-size` for the spawned server. 0 = server default (2). - #[arg(long, default_value_t = 0)] - peer_pool_size: usize, - /// [kv] `--max-inflight` for the spawned server. 0 = server default. - #[arg(long, default_value_t = 0)] - max_inflight: usize, - /// [kv] `--coalesce-max-keys` for the spawned server. 0 = server default (32). - #[arg(long, default_value_t = 0)] - coalesce_max_keys: usize, - /// [kv] Enable `--event-write` on the spawned server. - #[arg(long, default_value_t = false)] - event_write: bool, - /// [kv] `--send-queue-capacity` for the spawned server. 0 = server default (4096). - #[arg(long, default_value_t = 0)] - send_queue_capacity: u32, - /// [kv] `--metrics-interval` for the spawned server (seconds). 0 = server default (5). - #[arg(long, default_value_t = 0)] - metrics_interval: u64, - /// [kv] `--kv-backend` for the spawned server (file|block|mem-block). - /// Empty = server default (block). - #[arg(long, default_value = "")] - kv_backend: String, - /// [kv] `--wal-backend` for the spawned server (file|mem-block|block-device). - /// Empty = server default. - #[arg(long, default_value = "")] - wal_backend: String, - /// [kv] Enable `--no-fsync` on the spawned server. - #[arg(long, default_value_t = false)] - no_fsync: bool, - /// [diskdb] Disk-groups provisioned per node. - #[arg(long, default_value_t = 1)] - disk_groups_per_node: usize, - /// [diskdb] Disks provisioned per disk-group. - #[arg(long, default_value_t = 4)] - disks_per_group: usize, - /// [diskdb] Logical capacity of each disk. - #[arg(long, default_value_t = 1_099_511_627_776_u64)] - disk_capacity_bytes: u64, - /// [diskdb] Logical zone size of each disk. - #[arg(long, default_value_t = 274_877_906_944_u64)] - disk_zone_size_bytes: u64, - /// [diskdb] Allocation unit size. - #[arg(long, default_value_t = 1_048_576_u32)] - disk_unit_size_bytes: u32, - /// [diskdb] Existing store-0 KV groups used for allocation records. - #[arg(long, value_delimiter = ',', default_value = "1")] - data_groups: Vec, - /// [diskdb] KV client connections kept per endpoint. 0 = server default (1). - #[arg(long, default_value_t = 0)] - kv_connections: usize, - /// [diskdb] KV client crowdb-rpc I/O workers. 0 = server default (2). - #[arg(long, default_value_t = 0)] - kv_client_rpc_workers: u32, - /// [diskdb] Coalesce concurrently queued free requests. - #[arg(long, default_value_t = false)] - free_batch: bool, - /// [diskdb] Maximum free records per coalesced KV proposal. 0 = default (256). - #[arg(long, default_value_t = 0)] - free_flush_max_batch: u32, - /// [chunkdb] `DiskDB` client connections kept per endpoint. 0 = server default (1). - #[arg(long, default_value_t = 0)] - diskdb_connections: usize, - /// [chunkdb] `DiskDB` client crowdb-rpc I/O workers. 0 = server default (2). - #[arg(long, default_value_t = 0)] - diskdb_client_rpc_workers: u32, - /// [chunkdb] Number of instances to deploy. - #[arg(long, default_value_t = 3)] - chunkdb_instances: usize, - /// [chunkdb] Permit unsafe EC placement fallback. - #[arg(long, default_value_t = false)] - allow_unsafe_ec: bool, + /// Start or resume a named OCI cluster; one node is the default. + Start(local::StartArgs), + /// Stop the selected cluster's owned containers, preserving its data. + Stop, + /// Observe current application identity, initialization and readiness. + Status { + #[arg(long)] + json: bool, + /// Observation budget in seconds. + #[arg(long, default_value_t = 10, value_parser = clap::value_parser!(u64).range(1..))] + timeout: u64, }, - /// Tear down the entire cluster (all groups, stores, servers, sysdata). - Destroy, - /// Wipe user data on every node + wait for re-election. Preserves - /// group-0 sysdata + topology — servers stay running. Use --store/--group - /// to target a non-system group (recommended for benchmarks). - Clean { - /// Target store ID (default 0 = system store). - #[arg(long, default_value_t = 0)] - store: u64, - /// Target group ID (default 0 = system group; use 1+ for bench groups). + /// Check whole-cluster readiness; exit 1 when the contract is unmet. + Check { + #[arg(long)] + json: bool, + /// Maximum wait in seconds; zero observes once. #[arg(long, default_value_t = 0)] - group: u64, - /// Restart `DiskDB`, `DiskIO`, and `ChunkDB` after the KV wipe. - #[arg(long, default_value_t = false)] - restart_services: bool, + wait: u64, + /// Maximum duration of one observation request in seconds. + #[arg(long, default_value_t = 10, value_parser = clap::value_parser!(u64).range(1..))] + timeout: u64, }, - /// Show cluster status (list all stores from group-0 sysdata). - Status, /// Show the topology view from a node's `/topology` endpoint. Topology { #[arg(short = 'n', long)] @@ -223,483 +73,12 @@ pub enum ClusterVerb { }, } -#[allow(clippy::too_many_lines)] pub async fn run_cluster_verb(cli: &Cli, verb: ClusterVerb) -> ExitCode { match verb { - ClusterVerb::Init { - nodes, - bootstrap_file, - } => { - let Some(registry) = &cli.registry else { - eprintln!("error: cluster init requires --registry and a versioned bootstrap file"); - return ExitCode::from(2); - }; - let ctx = match authority_context(cli).await { - Ok(c) => c, - Err(c) => return c, - }; - let node_ids: Vec = match nodes.iter().map(|s| s.parse::()).collect::>() { - Ok(ids) => ids, - Err(e) => { - eprintln!("error: invalid node id: {e}"); - return ExitCode::from(1); - } - }; - let sealed_path = registry.with_extension("bootstrap-intent.toml"); - if let Some(source) = bootstrap_file { - let intent = match crowdb_console_shared::bootstrap_intent::BootstrapIntent::load(&source) { - Ok(intent) => intent, - Err(error) => { - eprintln!("error: load bootstrap file: {error}"); - return ExitCode::from(2); - } - }; - if intent.members() != node_ids.as_slice() { - eprintln!("error: bootstrap file members differ from --nodes"); - return ExitCode::from(1); - } - if let Err(error) = intent.seal(&sealed_path) { - eprintln!("error: seal bootstrap intent: {error}"); - return ExitCode::from(2); - } - } else if !sealed_path.exists() { - eprintln!("error: --bootstrap-file is required for the first init"); - return ExitCode::from(1); - } - let result = - crowdb_console_shared::ops::cluster::init_with_intent(&ctx, &node_ids, &sealed_path).await; - match result { - Ok(summary) => { - println!( - "cluster initialized: store {}, group {}, {} nodes", - summary.store_id, - summary.group_id, - summary.nodes.len() - ); - ExitCode::SUCCESS - } - Err(e) => { - eprintln!("error: cluster init: {e}"); - ExitCode::from(2) - } - } - } - ClusterVerb::LocalDeploy { - nodes, - service_type, - rpc_port, - io_engines, - io_workers, - enable_nagle, - rpc_workers, - diskio_rpc_workers, - diskio_dummy_disk_type, - peer_pool_size, - max_inflight, - coalesce_max_keys, - event_write, - send_queue_capacity, - metrics_interval, - kv_backend, - wal_backend, - no_fsync, - disk_groups_per_node, - disks_per_group, - disk_capacity_bytes, - disk_zone_size_bytes, - disk_unit_size_bytes, - data_groups, - kv_connections, - kv_client_rpc_workers, - free_batch, - free_flush_max_batch, - diskdb_connections, - diskdb_client_rpc_workers, - chunkdb_instances, - allow_unsafe_ec, - } => match service_type.as_str() { - "combined" => { - let ctx = match local_deploy_context(cli, false).await { - Ok(context) => context, - Err(code) => return code, - }; - let workspace = - deploy_workspace(cli).unwrap_or_else(|| std::path::PathBuf::from("cli-deploy")); - let tunables = crowdb_console_shared::ops::cluster::KvDeployTunables { - rpc_workers: nonzero(rpc_workers), - peer_pool_size: nonzero(peer_pool_size), - max_inflight: nonzero(max_inflight), - coalesce_max_keys: nonzero(coalesce_max_keys), - event_write: event_write.then_some(true), - send_queue_capacity: nonzero(send_queue_capacity), - metrics_interval: nonzero(metrics_interval), - kv_backend: (!kv_backend.is_empty()).then_some(kv_backend), - wal_backend: (!wal_backend.is_empty()).then_some(wal_backend), - no_fsync: no_fsync.then_some(true), - }; - let disk = crowdb_console_shared::ops::cluster::LocalDiskdbDeployConfig { - keepalive_interval_secs: None, - disk_groups_per_node, - disks_per_group, - capacity_bytes: disk_capacity_bytes, - zone_size_bytes: disk_zone_size_bytes, - unit_size_bytes: disk_unit_size_bytes, - data_groups, - rpc_workers: nonzero(rpc_workers), - kv_connections: nonzero(kv_connections), - kv_client_rpc_workers: nonzero(kv_client_rpc_workers), - free_batch_enabled: free_batch.then_some(true), - free_flush_max_batch: nonzero(free_flush_max_batch), - }; - let chunk = crowdb_console_shared::ops::cluster::LocalChunkdbDeployConfig { - instance_count: chunkdb_instances, - storage_groups: vec![1, 2, 3], - dynamic_ownership: false, - allow_unsafe_ec, - rpc_workers: nonzero(rpc_workers), - diskio_rpc_workers: nonzero(diskio_rpc_workers), - kv_connections: nonzero(kv_connections), - kv_client_rpc_workers: nonzero(kv_client_rpc_workers), - diskdb_connections: nonzero(diskdb_connections), - diskdb_client_rpc_workers: nonzero(diskdb_client_rpc_workers), - metrics_interval: nonzero(metrics_interval), - }; - match crowdb_console_shared::ops::cluster::local_deploy_combined( - &ctx, - &workspace, - Some(&tunables), - &disk, - &chunk, - &diskio_dummy_disk_type, - ) - .await - { - Ok(summary) => { - println!( - "local-deploy combined: {} KV nodes, {} racks, {} DiskDB, {} ChunkDB, {} DiskIO", - summary.kv_nodes, - summary.racks, - summary.diskdb_instances, - summary.chunkdb_instances, - summary.diskio_instances - ); - if let Some(seed) = ctx.config().servers.first() { - println!("Group 0 management seed: {}", seed.url); - } - ExitCode::SUCCESS - } - Err(error) => { - eprintln!("error: local-deploy combined: {error}"); - ExitCode::from(2) - } - } - } - "kv" => { - let ctx = match local_deploy_context(cli, false).await { - Ok(c) => c, - Err(c) => return c, - }; - let tunables = crowdb_console_shared::ops::cluster::KvDeployTunables { - rpc_workers: nonzero(rpc_workers), - peer_pool_size: nonzero(peer_pool_size), - max_inflight: nonzero(max_inflight), - coalesce_max_keys: nonzero(coalesce_max_keys), - event_write: if event_write { Some(true) } else { None }, - send_queue_capacity: nonzero(send_queue_capacity), - metrics_interval: nonzero(metrics_interval), - kv_backend: if kv_backend.is_empty() { - None - } else { - Some(kv_backend) - }, - wal_backend: if wal_backend.is_empty() { - None - } else { - Some(wal_backend) - }, - no_fsync: if no_fsync { Some(true) } else { None }, - }; - let workspace = deploy_workspace(cli); - match crowdb_console_shared::ops::cluster::local_deploy( - &ctx, - nodes, - workspace.as_deref(), - Some(&tunables), - ) - .await - { - Ok(summary) => { - println!( - "local-deploy complete: {} nodes (rack {}, nodes [{}]), group 0 bootstrapped", - summary.node_count, - summary.rack_id, - summary - .node_ids - .iter() - .map(std::string::ToString::to_string) - .collect::>() - .join(", ") - ); - if let Some(seed) = ctx.config().servers.first() { - println!("Group 0 management seed: {}", seed.url); - } - ExitCode::SUCCESS - } - Err(e) => { - eprintln!("error: local-deploy: {e}"); - ExitCode::from(2) - } - } - } - "rpc" => { - let ctx = match local_deploy_context(cli, false).await { - Ok(c) => c, - Err(c) => return c, - }; - let rpc_cfg = crowdb_console_shared::ops::cluster::RpcDeployConfig { - port: rpc_port, - io_engines, - io_workers, - enable_nagle, - ..Default::default() - }; - let workspace = deploy_workspace(cli); - match crowdb_console_shared::ops::cluster::local_deploy_rpc( - &ctx, - &rpc_cfg, - workspace.as_deref(), - ) - .await - { - Ok(summary) => { - println!( - "local-deploy rpc: port={}, pid={}, io_engines={}, io_workers={}, nagle={}", - summary.port, summary.pid, summary.io_engines, summary.io_workers, summary.nagle - ); - ExitCode::SUCCESS - } - Err(e) => { - eprintln!("error: local-deploy rpc: {e}"); - ExitCode::from(2) - } - } - } - "diskdb" => { - let ctx = match local_deploy_context(cli, true).await { - Ok(c) => c, - Err(c) => return c, - }; - let workspace = - deploy_workspace(cli).unwrap_or_else(|| std::path::PathBuf::from("cli-deploy")); - let config = crowdb_console_shared::ops::cluster::LocalDiskdbDeployConfig { - keepalive_interval_secs: None, - disk_groups_per_node, - disks_per_group, - capacity_bytes: disk_capacity_bytes, - zone_size_bytes: disk_zone_size_bytes, - unit_size_bytes: disk_unit_size_bytes, - data_groups, - rpc_workers: nonzero(rpc_workers), - kv_connections: nonzero(kv_connections), - kv_client_rpc_workers: nonzero(kv_client_rpc_workers), - free_batch_enabled: free_batch.then_some(true), - free_flush_max_batch: nonzero(free_flush_max_batch), - }; - match crowdb_console_shared::ops::cluster::local_deploy_diskdb(&ctx, &workspace, &config) - .await - { - Ok(summary) => { - println!( - "local-deploy diskdb: {} instances, {} disk-groups, {} disks, data-groups {:?}", - summary.instance_count, - summary.disk_group_count, - summary.disk_count, - summary.data_groups - ); - ExitCode::SUCCESS - } - Err(error) => { - eprintln!("error: local-deploy diskdb: {error}"); - ExitCode::from(2) - } - } - } - "chunkdb" => { - let ctx = match local_deploy_context(cli, true).await { - Ok(context) => context, - Err(code) => return code, - }; - let workspace = - deploy_workspace(cli).unwrap_or_else(|| std::path::PathBuf::from("cli-deploy")); - let chunk = crowdb_console_shared::ops::cluster::LocalChunkdbDeployConfig { - instance_count: chunkdb_instances, - storage_groups: vec![1, 2, 3], - dynamic_ownership: false, - allow_unsafe_ec, - rpc_workers: nonzero(rpc_workers), - diskio_rpc_workers: nonzero(diskio_rpc_workers), - kv_connections: nonzero(kv_connections), - kv_client_rpc_workers: nonzero(kv_client_rpc_workers), - diskdb_connections: nonzero(diskdb_connections), - diskdb_client_rpc_workers: nonzero(diskdb_client_rpc_workers), - metrics_interval: nonzero(metrics_interval), - }; - match crowdb_console_shared::ops::cluster::local_deploy_chunkdb(&ctx, &workspace, &chunk) - .await - { - Ok(summary) => { - println!("local-deploy chunkdb: {} instances", summary.instance_count); - ExitCode::SUCCESS - } - Err(error) => { - eprintln!("error: local-deploy chunkdb: {error}"); - ExitCode::from(2) - } - } - } - other => { - eprintln!( - "error: local-deploy: unsupported service type `{other}` (expected kv, diskdb, chunkdb, combined, or rpc)" - ); - ExitCode::from(1) - } - }, - ClusterVerb::Destroy => { - let Some(path) = &cli.registry else { - eprintln!("error: cluster destroy requires --registry to stop local processes"); - return ExitCode::from(2); - }; - let registry = match crowdb_console_shared::config::web::LaunchRegistry::load(path) { - Ok(registry) => registry, - Err(error) => { - eprintln!("error: load launch registry: {error}"); - return ExitCode::from(2); - } - }; - let runtime = match crowdb_console_shared::launch::LaunchRuntime::for_registry(path) { - Ok(runtime) => runtime, - Err(error) => { - eprintln!("error: launch runtime: {error}"); - return ExitCode::from(2); - } - }; - let ctx = match authority_context(cli).await { - Ok(c) => c, - Err(c) => return c, - }; - match crowdb_console_shared::ops::cluster::destroy(&ctx).await { - Ok(()) => { - for launch in ®istry.launches { - if let Err(error) = runtime.stop(launch).await { - eprintln!( - "error: stop {} on node {}: {error}", - launch.service_id, launch.node_id - ); - return ExitCode::from(2); - } - } - println!("cluster destroy complete"); - ExitCode::SUCCESS - } - Err(e) => { - eprintln!("error: cluster destroy: {e}"); - ExitCode::from(2) - } - } - } - ClusterVerb::Clean { - store, - group, - restart_services, - } => { - let restart = if restart_services { - let Some(path) = &cli.registry else { - eprintln!("error: --restart-services requires --registry"); - return ExitCode::from(2); - }; - let registry = match crowdb_console_shared::config::web::LaunchRegistry::load(path) { - Ok(registry) => registry, - Err(error) => { - eprintln!("error: load launch registry: {error}"); - return ExitCode::from(2); - } - }; - let runtime = match crowdb_console_shared::launch::LaunchRuntime::for_registry(path) { - Ok(runtime) => runtime, - Err(error) => { - eprintln!("error: launch runtime: {error}"); - return ExitCode::from(2); - } - }; - Some((registry, runtime)) - } else { - None - }; - let ctx = match authority_context(cli).await { - Ok(c) => c, - Err(c) => return c, - }; - match crowdb_console_shared::ops::cluster::clean(&ctx, store, group).await { - Ok(mut result) => { - if let Some((registry, runtime)) = restart { - for kind in ["diskio", "diskdb", "chunkdb"] { - for launch in registry - .launches - .iter() - .filter(|launch| launch.service_id == kind) - { - if let Err(error) = runtime.restart(launch).await { - eprintln!("error: restart {kind} on node {}: {error}", launch.node_id); - return ExitCode::from(2); - } - result.restarted_services += 1; - } - } - } - println!( - "cluster clean: wiped {} nodes, restarted {} services, leader = {}", - result.wiped_nodes, result.restarted_services, result.new_leader - ); - ExitCode::SUCCESS - } - Err(e) => { - eprintln!("error: cluster clean: {e}"); - ExitCode::from(2) - } - } - } - ClusterVerb::Status => { - let ctx = match authority_context(cli).await { - Ok(c) => c, - Err(c) => return c, - }; - match crowdb_console_shared::ops::cluster::status(&ctx).await { - Ok(stores) => { - if stores.is_empty() { - println!("(no stores)"); - } else { - println!("{:<12} {:<12} NODES", "STORE", "REPLICAS"); - for s in &stores { - println!( - "{:<12} {:<12} {}", - s.store_id, - s.node_ids.len(), - s.node_ids - .iter() - .map(std::string::ToString::to_string) - .collect::>() - .join(",") - ); - } - } - ExitCode::SUCCESS - } - Err(e) => { - eprintln!("error: cluster status: {e}"); - ExitCode::from(2) - } - } - } + ClusterVerb::Start(args) => local::start(cli, &args).await, + ClusterVerb::Stop => local::stop(cli).await, + ClusterVerb::Status { json, timeout } => status::run(cli, json, false, 0, timeout).await, + ClusterVerb::Check { json, wait, timeout } => status::run(cli, json, true, wait, timeout).await, ClusterVerb::Topology { node } => { let ctx = match authority_context(cli).await { Ok(c) => c, @@ -738,18 +117,3 @@ pub async fn run_cluster_verb(cli: &Cli, verb: ClusterVerb) -> ExitCode { ClusterVerb::Disk { verb } => run_disk_verb(cli, verb).await, } } - -/// Convert 0 to `None` (use server default), pass through nonzero values. -fn nonzero(v: T) -> Option { - (v != T::default()).then_some(v) -} - -/// Resolve a command-owned deployment workspace when one is configured. -/// Ordinary CLI commands leave `log_dir` empty and use the library default. -fn deploy_workspace(cli: &Cli) -> Option { - if cli.log_dir.as_os_str().is_empty() { - None - } else { - Some(cli.log_dir.join("deploy")) - } -} diff --git a/app/crowdb-cli/src/commands/cluster/local.rs b/app/crowdb-cli/src/commands/cluster/local.rs new file mode 100644 index 000000000..b16b55417 --- /dev/null +++ b/app/crowdb-cli/src/commands/cluster/local.rs @@ -0,0 +1,237 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Runtime effects use host ownership receipts; readiness uses application evidence. + +mod docker; +mod host; +mod launch; +mod state; + +use crate::Cli; +use clap::Args; +use crowdb_console_shared::{ + deployment::target::{ClusterTargetBinding, RuntimeOwnership, RuntimeSelection}, + error::Result, +}; +use crowdb_protocol::mgmt::cluster::ClusterPhase; +use docker::{owned, strings, Docker}; +use state::{invalid, Launch}; +use std::process::ExitCode; + +#[derive(Args, Debug)] +pub struct StartArgs { + #[command(flatten)] + host: host::HostArgs, + /// Initial voter count, used only when accepting a new launch. + #[arg(long, value_parser = clap::value_parser!(u8).range(1..=3))] + nodes: Option, + /// Already loaded CROWDB node image; start never builds or pulls. + #[arg(long)] + image: Option, + /// First localhost console port; each node reserves four ports, ten apart. + #[arg(long)] + port_base: Option, + #[arg(long)] + cpus: Option, + #[arg(long)] + memory_mib: Option, + /// Application readiness wait in seconds; zero observes once. + #[arg(long, default_value_t = 180)] + wait: u64, + #[arg(long)] + json: bool, +} + +fn selection(cli: &Cli) -> Result<&str> { + if !cli.management_endpoints.is_empty() { + return Err(invalid( + "runtime start/stop requires a local cluster name, not management endpoints", + )); + } + let name = cli.cluster.as_deref().unwrap_or("local"); + if !state::name_valid(name) { + return Err(invalid("invalid cluster name")); + } + Ok(name) +} + +fn binding_path(name: &str) -> std::path::PathBuf { + crowdb_protocol::port::namespace::runtime_root() + .join("persistent/clusters") + .join(name) + .join("target.json") +} + +fn binding(record: &Launch) -> Result> { + let Some(binding) = load_target(&record.cluster)? else { + return Ok(None); + }; + let expected = ownership(record); + if binding.version != 1 + || binding.name != record.cluster + || binding.runtime.as_ref() != Some(&expected) + || binding.management_endpoints != record.origins() + || uuid::Uuid::parse_str(&binding.cluster_id) + .map_or(true, |id| id.is_nil() || id.to_string() != binding.cluster_id) + { + return Err(invalid( + "cluster target binding differs from accepted runtime ownership", + )); + } + Ok(Some(binding)) +} + +pub(super) fn load_target(name: &str) -> Result> { + if !state::name_valid(name) { + return Err(invalid("invalid cluster name")); + } + let Some(value) = state::read_json(&binding_path(name))? else { + return Ok(None); + }; + let binding: ClusterTargetBinding = + serde_json::from_value(value).map_err(|_| invalid("invalid selected cluster target binding"))?; + if binding.version != 1 + || binding.name != name + || uuid::Uuid::parse_str(&binding.cluster_id) + .map_or(true, |id| id.is_nil() || id.to_string() != binding.cluster_id) + { + return Err(invalid("selected cluster target binding has an invalid identity")); + } + crowdb_console_shared::deployment::target::management_origins(&binding.management_endpoints) + .map_err(|error| invalid(&error.to_string()))?; + Ok(Some(binding)) +} + +pub(super) fn pending_origins(name: &str) -> Result> { + if let Some(record) = host::load(name)? { + if Launch::load(name, &state::directory())?.is_some() { + return Err(invalid("conflicting host and bridge ownership receipts")); + } + return Ok(record.origins); + } + Launch::load(name, &state::directory())? + .map(|record| record.origins()) + .ok_or_else(|| { + invalid("selected cluster has no readable target binding or runtime ownership receipt") + }) +} + +fn ownership(record: &Launch) -> RuntimeOwnership { + RuntimeOwnership { + selection: RuntimeSelection { + runtime: "docker".into(), + endpoint: record.runtime.endpoint.clone(), + context: record.runtime.context.clone(), + }, + owner: record.owner.clone(), + } +} + +pub(super) async fn start(cli: &Cli, args: &StartArgs) -> ExitCode { + match start_inner(cli, args).await { + Ok(ready) => { + if ready { + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } + } + Err(error) => failure(error), + } +} + +async fn start_inner(cli: &Cli, args: &StartArgs) -> Result { + let name = selection(cli)?; + if host::selected(name, args)? { + return host::start(args, name).await; + } + if args.nodes.is_some_and(|nodes| !matches!(nodes, 1 | 3)) { + return Err(invalid("initial voter count must be one or three")); + } + // Existing target ownership must be checked before any runtime mutation. + if Launch::load(name, &state::directory())?.is_none() && state::read_json(&binding_path(name))?.is_some() + { + return Err(invalid( + "selected application binding has no local runtime ownership receipt", + )); + } + let record = launch::accepted(&Docker, name, args).await?; + let previous = binding(&record)?; + launch::run(&Docker, &record).await?; + let report = super::status::observe_origins(&record.origins(), previous.as_ref(), true, args.wait, 10) + .await + .map_err(|error| invalid(&error))?; + if report.observation.cluster != record.cluster || report.observation.initial_voters != record.nodes { + return Err(invalid("runtime endpoints report a different deployment policy")); + } + if let Some(cluster_id) = &report.observation.cluster_id { + let target = ClusterTargetBinding { + version: 1, + name: record.cluster.clone(), + cluster_id: cluster_id.clone(), + management_endpoints: record.origins(), + runtime: Some(ownership(&record)), + }; + state::seal_json( + &binding_path(name), + &serde_json::to_value(target).map_err(|_| invalid("invalid application target"))?, + )?; + } + if args.json { + println!( + "{}", + serde_json::to_string_pretty(&report).map_err(|_| invalid("invalid status output"))? + ); + } else { + println!( + "cluster {}: {:?} (UUID {})", + record.cluster, + report.phase, + report.observation.cluster_id.as_deref().unwrap_or("unbound") + ); + for fact in &report.observation.dependencies { + println!(" {}: {:?}: {}", fact.id, fact.state, fact.detail); + } + for reason in &report.reasons { + println!(" {}: {}: {}", reason.dependency, reason.code, reason.detail); + } + } + Ok(report.phase == ClusterPhase::Ready) +} + +pub(super) async fn stop(cli: &Cli) -> ExitCode { + match stop_inner(cli).await { + Ok(()) => ExitCode::SUCCESS, + Err(error) => failure(error), + } +} + +async fn stop_inner(cli: &Cli) -> Result<()> { + let name = selection(cli)?; + if let Some(record) = host::load(name)? { + return host::stop(name, record).await; + } + let record = Launch::load(name, &state::directory())? + .ok_or_else(|| invalid("selected cluster has no local runtime ownership receipt"))?; + binding(&record)?; + Docker.check(&record).await?; + for item in Docker.objects("container").await? { + if owned(&item, &record, "container") && item["State"]["Running"] == true { + let identity = item["Id"] + .as_str() + .ok_or_else(|| invalid("container ID absent"))?; + let current = Docker.inspect("container", identity).await?; + docker::require_owned(¤t, &record, "container")?; + Docker.check(&record).await?; + Docker.call(&strings(&["stop", identity])).await?; + } + } + println!("cluster {name}: stopped; retained data and ownership"); + Ok(()) +} + +fn failure(error: impl std::fmt::Display) -> ExitCode { + eprintln!("error: cluster lifecycle: {error}"); + ExitCode::from(2) +} diff --git a/app/crowdb-cli/src/commands/cluster/local/docker.rs b/app/crowdb-cli/src/commands/cluster/local/docker.rs new file mode 100644 index 000000000..40964c805 --- /dev/null +++ b/app/crowdb-cli/src/commands/cluster/local/docker.rs @@ -0,0 +1,156 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Exact local Docker ownership; names alone never authorize an effect. + +use super::state::{invalid, DockerIdentity, Launch}; +use crowdb_console_shared::error::Result; +use serde_json::Value; +use std::time::Duration; + +pub(super) const CLUSTER: &str = "org.crowdb.cluster"; +pub(super) const OWNER: &str = "org.crowdb.cluster.owner"; +pub(super) struct Docker; + +impl Docker { + pub async fn call(&self, args: &[String]) -> Result { + let program = std::env::var_os("CROWDB_CONTAINER_RUNTIME").unwrap_or_else(|| "docker".into()); + let output = tokio::time::timeout( + Duration::from_secs(60), + tokio::process::Command::new(program) + .args(args) + .kill_on_drop(true) + .output(), + ) + .await + .map_err(|_| invalid("Docker command exceeded its 60-second bound"))??; + if !output.status.success() { + return Err(invalid(&format!( + "Docker {}: {}", + args.first().map_or("operation", String::as_str), + String::from_utf8_lossy(&output.stderr) + ))); + } + if output.stdout.len() > 16 * 1024 * 1024 { + return Err(invalid("Docker observation exceeds its size bound")); + } + String::from_utf8(output.stdout) + .map(|output| output.trim().into()) + .map_err(|_| invalid("invalid Docker output")) + } + pub async fn identity(&self) -> Result { + let context = self.call(&strings(&["context", "show"])).await?; + let details: Vec = + serde_json::from_str(&self.call(&strings(&["context", "inspect", &context])).await?) + .map_err(|_| invalid("invalid Docker context observation"))?; + let endpoint = std::env::var("DOCKER_HOST") + .ok() + .or_else(|| { + details + .first()? + .get("Endpoints")? + .get("docker")? + .get("Host")? + .as_str() + .map(String::from) + }) + .ok_or_else(|| invalid("Docker context has no endpoint"))?; + let daemon = self.call(&strings(&["info", "--format", "{{.ID}}"])).await?; + Ok(DockerIdentity { + daemon, + context, + endpoint, + }) + } + pub async fn check(&self, record: &Launch) -> Result<()> { + if self.identity().await? != record.runtime { + return Err(invalid( + "Docker endpoint/context differs from accepted launch inputs", + )); + } + Ok(()) + } + pub async fn objects(&self, kind: &str) -> Result> { + let args = if kind == "container" { + strings(&["ps", "-aq"]) + } else { + strings(&[kind, "ls", "-q"]) + }; + let ids = self.call(&args).await?; + if ids.is_empty() { + return Ok(Vec::new()); + } + let mut args = strings(&[kind, "inspect"]); + args.extend(ids.lines().map(String::from)); + if args.len() > 4098 { + return Err(invalid("Docker resource inventory exceeds bound")); + } + serde_json::from_str(&self.call(&args).await?) + .map_err(|_| invalid("invalid Docker resource observation")) + } + pub async fn inspect(&self, kind: &str, identity: &str) -> Result { + let mut values: Vec = + serde_json::from_str(&self.call(&strings(&[kind, "inspect", identity])).await?) + .map_err(|_| invalid("invalid Docker resource observation"))?; + if values.len() != 1 { + return Err(invalid("Docker resource observation is not unique")); + } + Ok(values.remove(0)) + } + pub async fn ensure(&self, record: &Launch, kind: &str, name: &str) -> Result { + if let Some(item) = self + .objects(kind) + .await? + .into_iter() + .find(|item| object_name(item) == name) + { + require_owned(&item, record, kind)?; + return Ok(item); + } + self.check(record).await?; + let mut args = strings(&[kind, "create"]); + args.extend(labels(record)); + args.push(name.into()); + self.call(&args).await?; + let item = self.inspect(kind, name).await?; + require_owned(&item, record, kind)?; + Ok(item) + } +} + +pub(super) fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| (*value).into()).collect() +} +pub(super) fn labels(record: &Launch) -> Vec { + vec![ + "--label".into(), + format!("{CLUSTER}={}", record.cluster), + "--label".into(), + format!("{OWNER}={}", record.owner), + ] +} +pub(super) fn object_name(item: &Value) -> &str { + item["Name"].as_str().unwrap_or_default().trim_start_matches('/') +} +pub(super) fn owned(item: &Value, record: &Launch, kind: &str) -> bool { + let labels = if kind == "container" { + &item["Config"]["Labels"] + } else { + &item["Labels"] + }; + labels[CLUSTER] == record.cluster + && labels[OWNER] == record.owner + && (if kind == "network" { + object_name(item) == record.resource_name(kind, 0) + } else { + (1..=record.nodes).any(|index| object_name(item) == record.resource_name(kind, index)) + }) +} +pub(super) fn require_owned(item: &Value, record: &Launch, kind: &str) -> Result<()> { + if !owned(item, record, kind) { + return Err(invalid( + "Docker resource ownership differs from accepted cluster inputs", + )); + } + Ok(()) +} diff --git a/app/crowdb-cli/src/commands/cluster/local/host.rs b/app/crowdb-cli/src/commands/cluster/local/host.rs new file mode 100644 index 000000000..b1025a621 --- /dev/null +++ b/app/crowdb-cli/src/commands/cluster/local/host.rs @@ -0,0 +1,200 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One production node per host, with runtime ownership separate from application identity. + +mod args; +mod helper; +use super::{ + binding_path, load_target, + state::{self, invalid, Launch}, + StartArgs, +}; +pub(super) use args::HostArgs; +use args::{Network, Runtime}; +use crowdb_console_shared::{ + deployment::target::{ClusterTargetBinding, RuntimeOwnership, RuntimeSelection}, + error::Result, +}; +use crowdb_protocol::mgmt::cluster::ClusterPhase; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::path::PathBuf; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RuntimeIdentity { + selection: RuntimeSelection, + daemon: String, + snapshotter: String, + binary: String, +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct HostLaunch { + version: u8, + cluster: String, + owner: String, + name: String, + runtime: RuntimeIdentity, + runtime_image: String, + image_id: String, + options: Value, + disks: Value, + file_hashes: Value, + pub(super) origins: Vec, +} + +fn path(name: &str) -> Result { + state::path( + name, + &crowdb_protocol::port::namespace::runtime_root().join("persistent/oci-hosts"), + ) +} + +pub(super) fn load(name: &str) -> Result> { + state::read_json(&path(name)?)? + .map(|value| { + let record: HostLaunch = + serde_json::from_value(value).map_err(|_| invalid("invalid host launch receipt"))?; + record.validate(name)?; + Ok(record) + }) + .transpose() +} + +impl HostLaunch { + fn validate(&self, name: &str) -> Result<()> { + if self.version != 1 + || self.cluster != name + || self.name != format!("crowdb-{name}-host") + || uuid::Uuid::parse_str(&self.owner) + .map_or(true, |id| id.is_nil() || id.to_string() != self.owner) + || !matches!(self.runtime.selection.runtime.as_str(), "docker" | "containerd") + || self.runtime.selection.endpoint.is_empty() + || self.runtime.daemon.is_empty() + || self.options["network"] != "host" + || !matches!(self.options["nodes"].as_u64(), Some(1 | 3)) + || self.origins.len() != 1 + { + return Err(invalid("invalid host runtime identity or deployment policy")); + } + crowdb_console_shared::deployment::target::management_origins(&self.origins) + .map_err(|error| invalid(&error.to_string()))?; + Ok(()) + } + fn ownership(&self) -> RuntimeOwnership { + RuntimeOwnership { + selection: self.runtime.selection.clone(), + owner: self.owner.clone(), + } + } + fn check_binding(&self) -> Result> { + let target = load_target(&self.cluster)?; + if target.as_ref().is_some_and(|binding| { + binding.runtime.as_ref() != Some(&self.ownership()) + || binding.management_endpoints != self.origins + }) { + return Err(invalid("cluster target differs from accepted host ownership")); + } + Ok(target) + } + fn value(&self) -> Result { + serde_json::to_value(self).map_err(|_| invalid("invalid host launch receipt")) + } +} + +pub(super) fn selected(name: &str, args: &StartArgs) -> Result { + let previous = load(name)?; + if previous.is_some() || args.host.network == Some(Network::Host) { + if Launch::load(name, &state::directory())?.is_some() { + return Err(invalid("cluster already has a Docker bridge ownership receipt")); + } + if args.port_base.is_some() { + return Err(invalid( + "host networking uses port 9090; --port-base is for a local bridge", + )); + } + return Ok(true); + } + if args.host.runtime == Some(Runtime::Containerd) || args.host.has_host_inputs() { + return Err(invalid( + "host/containerd inputs require --network host; local bridge uses Docker", + )); + } + Ok(false) +} + +pub(super) async fn start(args: &StartArgs, name: &str) -> Result { + let previous = load(name)?; + if previous.is_none() && state::read_json(&binding_path(name))?.is_some() { + return Err(invalid( + "selected application binding has no host runtime ownership receipt", + )); + } + if let Some(record) = &previous { + record.check_binding()?; + } + let mut options = serde_json::to_value(&args.host).map_err(|_| invalid("invalid host options"))?; + for (key, value) in [ + ("nodes", json!(args.nodes)), + ("image", json!(args.image)), + ("cpus", json!(args.cpus)), + ("memory_mib", json!(args.memory_mib)), + ] { + options[key] = value; + } + let value=helper::invoke(json!({"operation":"prepare","cluster":name,"options":options,"record":previous.map(|record|record.value()).transpose()?})).await?; + let record: HostLaunch = + serde_json::from_value(value.clone()).map_err(|_| invalid("invalid accepted host receipt"))?; + record.validate(name)?; + state::seal_json(&path(name)?, &value)?; + let target = record.check_binding()?; + helper::invoke(json!({"operation":"run","cluster":name,"options":{},"record":value})).await?; + let report = super::super::status::observe_origins(&record.origins, target.as_ref(), true, args.wait, 10) + .await + .map_err(|error| invalid(&error))?; + if report.observation.cluster != name + || Some(u64::from(report.observation.initial_voters)) != record.options["nodes"].as_u64() + { + return Err(invalid("host endpoints report a different deployment policy")); + } + if let Some(cluster_id) = &report.observation.cluster_id { + let binding = ClusterTargetBinding { + version: 1, + name: name.into(), + cluster_id: cluster_id.clone(), + management_endpoints: record.origins.clone(), + runtime: Some(record.ownership()), + }; + state::seal_json( + &binding_path(name), + &serde_json::to_value(binding).map_err(|_| invalid("invalid host target"))?, + )?; + } + if args.json { + println!( + "{}", + serde_json::to_string_pretty(&report).map_err(|_| invalid("invalid status output"))? + ); + } else { + println!( + "cluster {}: {:?} (UUID {})", + name, + report.phase, + report.observation.cluster_id.as_deref().unwrap_or("unbound") + ); + } + Ok(report.phase == ClusterPhase::Ready) +} + +pub(super) async fn stop(name: &str, record: HostLaunch) -> Result<()> { + if Launch::load(name, &state::directory())?.is_some() { + return Err(invalid("conflicting host and bridge ownership receipts")); + } + record.check_binding()?; + helper::invoke(json!({"operation":"stop","cluster":name,"record":record.value()?})).await?; + println!("cluster {name}: stopped; retained data and ownership"); + Ok(()) +} diff --git a/app/crowdb-cli/src/commands/cluster/local/host/args.rs b/app/crowdb-cli/src/commands/cluster/local/host/args.rs new file mode 100644 index 000000000..e369022ab --- /dev/null +++ b/app/crowdb-cli/src/commands/cluster/local/host/args.rs @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use clap::{Args, ValueEnum}; +use serde::Serialize; +use std::{net::IpAddr, path::PathBuf}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(super) enum Runtime { + Docker, + Containerd, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum, Serialize)] +#[serde(rename_all = "kebab-case")] +pub(super) enum Network { + Bridge, + Host, +} + +#[derive(Args, Debug, Serialize)] +pub(crate) struct HostArgs { + #[arg(long, value_enum)] + pub(super) runtime: Option, + #[arg(long, value_enum)] + pub(super) network: Option, + /// Local Unix runtime socket; accepted with the daemon identity. + #[arg(long)] + runtime_endpoint: Option, + /// Containerd snapshotter; host mode only. + #[arg(long, value_parser = ["overlayfs", "native"])] + snapshotter: Option, + #[arg(long)] + data_root: Option, + #[arg(long)] + physical_host_id: Option, + #[arg(long)] + interface: Option, + /// This host's management IP on the selected interface; port 9090. + #[arg(long)] + management_address: Option, + #[arg(long)] + password_file: Option, + /// Explicit production node policy, including supplied media and public URIs. + #[arg(long)] + node_policy_file: Option, + #[arg(long)] + device: Vec, + #[arg(long)] + disk_file: Vec, + #[arg(long)] + seed: Vec, +} + +impl HostArgs { + pub(super) fn has_host_inputs(&self) -> bool { + self.runtime_endpoint.is_some() + || self.snapshotter.is_some() + || self.data_root.is_some() + || self.physical_host_id.is_some() + || self.interface.is_some() + || self.management_address.is_some() + || self.password_file.is_some() + || self.node_policy_file.is_some() + || !self.device.is_empty() + || !self.disk_file.is_empty() + || !self.seed.is_empty() + } +} diff --git a/app/crowdb-cli/src/commands/cluster/local/host/helper.rs b/app/crowdb-cli/src/commands/cluster/local/host/helper.rs new file mode 100644 index 000000000..eee6a5c5b --- /dev/null +++ b/app/crowdb-cli/src/commands/cluster/local/host/helper.rs @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Embed the same host validators used by direct OCI launchers. + +use super::super::state::{self, invalid}; +use crowdb_console_shared::error::Result; +use serde_json::Value; +use std::{ + fs, + io::Write, + os::unix::fs::DirBuilderExt, + path::PathBuf, + process::{Command, Stdio}, +}; + +macro_rules! host_module { + ($name:literal) => { + ( + $name, + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../container/oci-image/", + $name + )), + ) + }; +} + +const MODULES: &[(&str, &str)] = &[ + host_module!("host_lifecycle.py"), + host_module!("host_inputs.py"), + host_module!("host_runtime.py"), + ( + "runtime.py", + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../container/oci-image/runtime.py" + )), + ), + ( + "run_node.py", + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../container/oci-image/run-node.py" + )), + ), + ( + "storage.py", + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../container/oci-image/storage.py" + )), + ), + ( + "storage_identity.py", + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../container/oci-image/storage_identity.py" + )), + ), + ( + "builder.py", + include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../container/oci-image/builder.py" + )), + ), +]; + +struct Modules(PathBuf); +impl Drop for Modules { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } +} + +fn execute(request: &Value) -> Result { + let parent = crowdb_protocol::port::namespace::runtime_root().join("artifacts/cli/host-runtime"); + state::check_path(&parent)?; + fs::DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&parent)?; + let directory = parent.join(uuid::Uuid::new_v4().to_string()); + fs::DirBuilder::new().mode(0o700).create(&directory)?; + let modules = Modules(directory); + for (name, source) in MODULES { + fs::write(modules.0.join(name), source)?; + } + let mut child = Command::new("python3") + .arg(modules.0.join("host_lifecycle.py")) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| { + invalid(&format!( + "host runtime lifecycle requires Python 3.11 or newer: {error}" + )) + })?; + child + .stdin + .take() + .ok_or_else(|| invalid("host runtime helper has no input"))? + .write_all(&serde_json::to_vec(request).map_err(|_| invalid("invalid host request"))?)?; + let output = child.wait_with_output()?; + if !output.status.success() { + return Err(invalid(&String::from_utf8_lossy(&output.stderr))); + } + serde_json::from_slice(&output.stdout).map_err(|_| invalid("invalid host runtime helper response")) +} + +pub(super) async fn invoke(request: Value) -> Result { + tokio::task::spawn_blocking(move || execute(&request)) + .await + .map_err(|error| invalid(&format!("host runtime helper failed: {error}")))? +} diff --git a/app/crowdb-cli/src/commands/cluster/local/launch.rs b/app/crowdb-cli/src/commands/cluster/local/launch.rs new file mode 100644 index 000000000..664205ef6 --- /dev/null +++ b/app/crowdb-cli/src/commands/cluster/local/launch.rs @@ -0,0 +1,191 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One/three normal OCI nodes; the monitor owns bootstrap and preparation. + +use super::{ + docker::{labels, object_name, require_owned, strings, Docker}, + state::{self, invalid, Launch}, + StartArgs, +}; +use crowdb_console_shared::error::Result; +use serde_json::{json, Value}; +use std::time::Duration; + +pub(super) async fn accepted(docker: &Docker, cluster: &str, args: &StartArgs) -> Result { + if let Some(previous) = Launch::load(cluster, &state::directory())? { + docker.check(&previous).await?; + if args.nodes.is_some_and(|value| value != previous.nodes) + || args.port_base.is_some_and(|value| value != previous.port_base) + || args.image.as_ref().is_some_and(|value| value != &previous.image) + || args + .cpus + .is_some_and(|value| value.to_bits() != previous.cpus.to_bits()) + || args.memory_mib.is_some_and(|value| value != previous.memory_mib) + { + return Err(invalid("launch arguments differ from accepted cluster inputs")); + } + return Ok(previous); + } + let image = args.image.as_deref().unwrap_or("crowdb-node:dev"); + let metadata = docker.inspect("image", image).await.map_err(|error| { + invalid(&format!( + "image inspection failed: {error}; build and load the OCI node image before start" + )) + })?; + if metadata["Os"] != "linux" + || metadata["Architecture"] != "amd64" + || metadata["Config"]["Labels"]["org.opencontainers.image.source"] + != "https://github.com/buzzcrow/crowdb" + { + return Err(invalid("local start requires the Linux amd64 CROWDB node image")); + } + let record = Launch { + version: 1, + cluster: cluster.into(), + owner: uuid::Uuid::new_v4().to_string(), + runtime: docker.identity().await?, + nodes: args.nodes.unwrap_or(1), + port_base: args.port_base.unwrap_or(9090), + image: image.into(), + image_id: metadata["Id"] + .as_str() + .ok_or_else(|| invalid("image has no immutable ID"))? + .into(), + cpus: args.cpus.unwrap_or(1.0), + memory_mib: args.memory_mib.unwrap_or(1536), + }; + record.seal(&state::directory())?; + Ok(record) +} + +pub(super) async fn run(docker: &Docker, record: &Launch) -> Result<()> { + docker.check(record).await?; + docker.inspect("image", &record.image_id).await?; + let network = record.resource_name("network", 0); + let bridge = docker.ensure(record, "network", &network).await?; + if bridge["Driver"] != "bridge" { + return Err(invalid("local cluster requires its owned bridge")); + } + let containers = docker.objects("container").await?; + for index in 1..=record.nodes { + let name = record.resource_name("container", index); + let volume = record.resource_name("volume", index); + docker.ensure(record, "volume", &volume).await?; + if let Some(previous) = containers.iter().find(|item| object_name(item) == name) { + verify(previous, record, index)?; + if previous["State"]["Running"] != true { + docker.check(record).await?; + docker + .call(&strings(&[ + "start", + previous["Id"] + .as_str() + .ok_or_else(|| invalid("container ID absent"))?, + ])) + .await?; + } + } else { + docker.check(record).await?; + let identity = docker.call(&arguments(record, index)).await?; + verify(&docker.inspect("container", &identity).await?, record, index)?; + } + } + Ok(()) +} + +fn arguments(record: &Launch, index: u8) -> Vec { + let name = record.resource_name("container", index); + let network = record.resource_name("network", 0); + let volume = record.resource_name("volume", index); + let mut args = strings(&["run", "-d", "--pull", "never", "--name", &name]); + args.extend(labels(record)); + args.extend(strings(&[ + "--network", + &network, + "--cpus", + &record.cpus.to_string(), + "--memory", + &format!("{}m", record.memory_mib), + "--mount", + &format!("type=volume,source={volume},target=/opt/crowdb/data"), + ])); + let base = record.port_base + u16::from(index - 1) * 10; + for (offset, protocol) in [(0_u16, ""), (1, "S3"), (2, "ICEBERG"), (3, "DATASET")] { + args.extend(strings(&[ + "-p", + &format!("127.0.0.1:{}:{}", base + offset, 9090 + offset), + ])); + if !protocol.is_empty() { + args.extend(strings(&[ + "-e", + &format!("CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{}", base + offset), + ])); + } + } + args.extend(strings(&[ + "-e", + &format!("CROWDB_CLUSTER={}", record.cluster), + "-e", + &format!("CROWDB_INITIAL_VOTERS={}", record.nodes), + ])); + args.push(record.image_id.clone()); + args +} + +pub(super) fn verify(item: &Value, record: &Launch, index: u8) -> Result<()> { + require_owned(item, record, "container")?; + let base = record.port_base + u16::from(index - 1) * 10; + let mut ports = serde_json::Map::new(); + for offset in 0..4 { + ports.insert( + format!("{}/tcp", 9090 + offset), + json!([{"HostIp":"127.0.0.1", "HostPort":(base+offset).to_string()}]), + ); + } + let mounts = item["Mounts"] + .as_array() + .ok_or_else(|| invalid("container mounts absent"))?; + let mounted: Vec<_> = mounts + .iter() + .filter(|mount| mount["Destination"] == "/opt/crowdb/data") + .collect(); + let nano_cpus = u64::try_from(Duration::from_secs_f64(record.cpus).as_nanos()) + .map_err(|_| invalid("invalid CPU boundary"))?; + if item["Image"] != record.image_id + || item["HostConfig"]["NanoCpus"] != nano_cpus + || item["HostConfig"]["Memory"] != record.memory_mib * 1024 * 1024 + || item["HostConfig"]["NetworkMode"] != record.resource_name("network", 0) + || item["HostConfig"]["PortBindings"] != Value::Object(ports) + || mounted.len() != 1 + || mounted[0]["Type"] != "volume" + || mounted[0]["RW"] != true + || mounted[0]["Name"] != record.resource_name("volume", index) + { + return Err(invalid( + "container image, resources, endpoints or volume differ from accepted inputs", + )); + } + let environment = item["Config"]["Env"] + .as_array() + .ok_or_else(|| invalid("container environment absent"))?; + let mut expected = vec![ + format!("CROWDB_CLUSTER={}", record.cluster), + format!("CROWDB_INITIAL_VOTERS={}", record.nodes), + ]; + for (offset, protocol) in [(1, "S3"), (2, "ICEBERG"), (3, "DATASET")] { + expected.push(format!( + "CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{}", + base + offset + )); + } + if expected + .iter() + .any(|expected| !environment.iter().any(|value| value.as_str() == Some(expected))) + { + return Err(invalid( + "container deployment policy differs from accepted inputs", + )); + } + Ok(()) +} diff --git a/app/crowdb-cli/src/commands/cluster/local/state.rs b/app/crowdb-cli/src/commands/cluster/local/state.rs new file mode 100644 index 000000000..694db0da8 --- /dev/null +++ b/app/crowdb-cli/src/commands/cluster/local/state.rs @@ -0,0 +1,197 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Host-owned Docker launch inputs, shared with independent Pixi shortcuts. + +use crowdb_console_shared::error::{Error, Result}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{self, File, OpenOptions}, + io::{Read, Write}, + os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + path::{Path, PathBuf}, +}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct DockerIdentity { + pub daemon: String, + pub context: String, + pub endpoint: String, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct Launch { + pub version: u8, + pub cluster: String, + pub owner: String, + pub runtime: DockerIdentity, + pub nodes: u8, + pub port_base: u16, + pub image: String, + pub image_id: String, + pub cpus: f64, + pub memory_mib: u64, +} + +pub(super) fn directory() -> PathBuf { + crowdb_protocol::port::namespace::runtime_root().join("persistent/oci-clusters") +} + +pub(super) fn name_valid(name: &str) -> bool { + !name.is_empty() + && name.len() <= 48 + && name.as_bytes()[0].is_ascii_alphanumeric() + && name + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.-".contains(&byte)) +} + +pub(super) fn path(name: &str, directory: &Path) -> Result { + if !name_valid(name) { + return Err(invalid("invalid cluster name")); + } + check_path(directory)?; + Ok(directory.join(format!("{name}.json"))) +} + +pub(super) fn check_path(path: &Path) -> Result<()> { + if !path.is_absolute() { + return Err(invalid("cluster state path must be absolute")); + } + for parent in path.ancestors() { + match fs::symlink_metadata(parent) { + Ok(metadata) if metadata.file_type().is_symlink() => { + return Err(invalid("cluster state paths cannot contain symlinks")) + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + } + Ok(()) +} + +pub(super) fn read_json(path: &Path) -> Result> { + check_path(path)?; + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if !metadata.is_file() || metadata.permissions().mode() & 0o077 != 0 || metadata.len() > 1024 * 1024 { + return Err(invalid("cluster receipt must be a bounded private regular file")); + } + let mut bytes = Vec::new(); + OpenOptions::new() + .read(true) + .custom_flags(libc::O_NOFOLLOW) + .open(path)? + .take(1024 * 1024 + 1) + .read_to_end(&mut bytes)?; + if bytes.len() > 1024 * 1024 { + return Err(invalid("cluster receipt exceeds its bound")); + } + serde_json::from_slice(&bytes) + .map(Some) + .map_err(|_| invalid("invalid cluster receipt JSON")) +} + +pub(super) fn seal_json(path: &Path, value: &serde_json::Value) -> Result<()> { + check_path(path)?; + let parent = path + .parent() + .ok_or_else(|| invalid("cluster receipt has no directory"))?; + fs::DirBuilder::new().recursive(true).mode(0o700).create(parent)?; + let temporary = parent.join(format!(".{}.tmp", uuid::Uuid::new_v4())); + let result = (|| { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(&serde_json::to_vec_pretty(value).map_err(|_| invalid("invalid cluster receipt"))?)?; + file.sync_all()?; + match fs::hard_link(&temporary, path) { + Ok(()) => File::open(parent)?.sync_all()?, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + if read_json(path)?.as_ref() != Some(value) { + return Err(invalid("cluster receipt conflicts with accepted inputs")); + } + Ok(()) + })(); + let cleanup = fs::remove_file(temporary); + result?; + cleanup?; + Ok(()) +} + +impl Launch { + pub fn load(name: &str, directory: &Path) -> Result> { + let Some(value) = read_json(&path(name, directory)?)? else { + return Ok(None); + }; + let launch: Self = + serde_json::from_value(value).map_err(|_| invalid("invalid Docker launch receipt fields"))?; + launch.validate()?; + if launch.cluster != name { + return Err(invalid("cluster name differs from its receipt")); + } + Ok(Some(launch)) + } + pub fn seal(&self, directory: &Path) -> Result<()> { + self.validate()?; + seal_json( + &path(&self.cluster, directory)?, + &serde_json::to_value(self).map_err(|_| invalid("invalid Docker launch receipt"))?, + ) + } + pub fn validate(&self) -> Result<()> { + let owner = + uuid::Uuid::parse_str(&self.owner).map_err(|_| invalid("invalid cluster ownership token"))?; + if self.version != 1 + || !name_valid(&self.cluster) + || owner.is_nil() + || owner.to_string() != self.owner + || !matches!(self.nodes, 1 | 3) + || !(1..=65512).contains(&self.port_base) + || !self.cpus.is_finite() + || self.cpus <= 0.0 + || self.cpus > f64::from(u32::MAX) + || self.memory_mib < 512 + || self.memory_mib.checked_mul(1024 * 1024).is_none() + || self.image.is_empty() + || self.image_id.strip_prefix("sha256:").map_or(true, |digest| { + digest.len() != 64 || !digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) + || self.runtime.daemon.is_empty() + || self.runtime.context.is_empty() + || !self.runtime.endpoint.starts_with("unix://") + { + return Err(invalid( + "invalid local Docker launch identity, image or resources", + )); + } + Ok(()) + } + pub fn origins(&self) -> Vec { + (0..self.nodes) + .map(|index| format!("http://127.0.0.1:{}", self.port_base + u16::from(index) * 10)) + .collect() + } + pub fn resource_name(&self, kind: &str, index: u8) -> String { + let base = format!("crowdb-{}", self.cluster); + if kind == "network" { + base + } else { + format!("{base}-{index}{}", if kind == "volume" { "-data" } else { "" }) + } + } +} + +pub(super) fn invalid(message: &str) -> Error { + Error::Config(message.into()) +} diff --git a/app/crowdb-cli/src/commands/cluster/status.rs b/app/crowdb-cli/src/commands/cluster/status.rs new file mode 100644 index 000000000..68989a4f2 --- /dev/null +++ b/app/crowdb-cli/src/commands/cluster/status.rs @@ -0,0 +1,275 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! HTTP-only application checks, usable without a local container runtime. + +use crate::Cli; +use crowdb_console_shared::{deployment::target, ops::cluster::readiness}; +use crowdb_protocol::mgmt::cluster::{ + ClusterPhase, ClusterStatus, DependencyKind, DependencyState, ReadinessReason, +}; +use std::{ + process::ExitCode, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tokio::time::Instant; + +pub(crate) async fn run(cli: &Cli, json: bool, check: bool, wait: u64, timeout: u64) -> ExitCode { + match observe(cli, check, wait, timeout).await { + Ok(report) => { + if json { + match serde_json::to_string_pretty(&report) { + Ok(output) => println!("{output}"), + Err(error) => return failure(error), + } + } else { + println!( + "cluster {}: {:?} (UUID {})", + report.observation.cluster, + report.phase, + report.observation.cluster_id.as_deref().unwrap_or("unbound") + ); + for observation in &report.observation.dependencies { + println!( + " {}: {:?}: {}", + observation.id, observation.state, observation.detail + ); + } + for reason in &report.reasons { + println!(" {}: {}: {}", reason.dependency, reason.code, reason.detail); + } + } + if check && report.phase != ClusterPhase::Ready { + ExitCode::from(1) + } else { + ExitCode::SUCCESS + } + } + Err(error) => failure(error), + } +} + +async fn observe(cli: &Cli, check: bool, wait: u64, timeout: u64) -> Result { + let binding = cli + .cluster + .as_deref() + .map(super::local::load_target) + .transpose() + .map_err(|error| error.to_string())? + .flatten(); + let pending = if binding.is_none() && cli.management_endpoints.is_empty() { + cli.cluster + .as_deref() + .map(super::local::pending_origins) + .transpose() + .map_err(|error| error.to_string())? + .unwrap_or_default() + } else { + Vec::new() + }; + let origins = target::management_origins(if cli.management_endpoints.is_empty() { + binding.as_ref().map_or(pending.as_slice(), |binding| { + binding.management_endpoints.as_slice() + }) + } else { + &cli.management_endpoints + }) + .map_err(|error| error.to_string())?; + let report = observe_origins(&origins, binding.as_ref(), check, wait, timeout).await?; + if cli + .cluster + .as_ref() + .is_some_and(|name| name != &report.observation.cluster) + { + return Err("selected cluster and management endpoints disagree on identity".into()); + } + Ok(report) +} + +pub(super) async fn observe_origins( + origins: &[String], + binding: Option<&target::ClusterTargetBinding>, + check: bool, + wait: u64, + timeout: u64, +) -> Result { + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(|error| error.to_string())?; + let budget = Duration::from_secs(timeout); + let deadline = Instant::now() + .checked_add(Duration::from_secs(if check && wait > 0 { + wait + } else { + timeout + })) + .ok_or("wait exceeds the supported clock range")?; + let mut last = None; + let mut known_identity = binding.map(|binding| binding.cluster_id.clone()); + loop { + let request_deadline = deadline.min( + Instant::now() + .checked_add(budget) + .ok_or("timeout exceeds the supported clock range")?, + ); + last = round( + &client, + origins, + binding, + request_deadline, + last, + &mut known_identity, + ) + .await?; + if last + .as_ref() + .is_some_and(|report| report.phase == ClusterPhase::Ready) + || !check + || wait == 0 + || Instant::now() >= deadline + { + return last.ok_or_else(|| "cluster status observation unavailable within its deadline".into()); + } + tokio::time::sleep_until(deadline.min(Instant::now() + Duration::from_millis(250))).await; + if Instant::now() >= deadline { + return last.ok_or_else(|| "cluster status observation unavailable within its deadline".into()); + } + } +} + +async fn round( + client: &reqwest::Client, + origins: &[String], + binding: Option<&target::ClusterTargetBinding>, + deadline: Instant, + last: Option, + known_identity: &mut Option, +) -> Result, String> { + let replies = + futures::future::join_all(origins.iter().map(|origin| read(client, origin, deadline))).await; + let mut selected: Option = None; + let mut missing = false; + for reply in replies { + let report = match reply { + Ok(report) => report, + Err(ReadError::Unavailable) => { + missing = true; + continue; + } + Err(ReadError::Invalid(reason)) => return Err(reason.into()), + }; + let current = report.observation.cluster_id.as_ref(); + if current.is_some_and(|current| known_identity.as_ref().is_some_and(|known| known != current)) + || binding.is_some_and(|binding| report.observation.cluster != binding.name) + || selected + .as_ref() + .or(last.as_ref()) + .is_some_and(|seen| seen.observation.cluster != report.observation.cluster) + { + return Err("selected cluster and management endpoints disagree on identity".into()); + } + if known_identity.is_none() { + known_identity.clone_from(&report.observation.cluster_id); + } + if selected + .as_ref() + .map_or(true, |seen| rank(report.phase) > rank(seen.phase)) + { + selected = Some(report); + } + } + let selected = selected.or(last); + Ok(if missing { + selected.map(unavailable) + } else { + selected + }) +} + +fn unavailable(mut report: ClusterStatus) -> ClusterStatus { + let now = now_ms(); + for fact in &mut report.observation.dependencies { + if fact.kind == DependencyKind::Authority { + fact.state = DependencyState::Unknown; + fact.observed_at_unix_ms = now; + fact.reason = Some(ReadinessReason { + dependency: fact.id.clone(), + code: "management-origin-unavailable".into(), + detail: "A selected management origin did not provide a current observation".into(), + }); + } + } + readiness::reduce(report.observation, now) +} + +enum ReadError { + Unavailable, + Invalid(&'static str), +} +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |value| u64::try_from(value.as_millis()).unwrap_or(u64::MAX)) +} + +async fn read(client: &reqwest::Client, origin: &str, deadline: Instant) -> Result { + tokio::time::timeout_at(deadline, async { + let mut reply = client + .get(format!("{origin}/api/cluster/status")) + .send() + .await + .map_err(|_| ReadError::Unavailable)? + .error_for_status() + .map_err(|_| ReadError::Unavailable)?; + let mut bytes = Vec::new(); + while let Some(chunk) = reply.chunk().await.map_err(|_| ReadError::Unavailable)? { + if bytes.len() + chunk.len() > 1024 * 1024 { + return Err(ReadError::Invalid("cluster status exceeds its size bound")); + } + bytes.extend_from_slice(&chunk); + } + let report: ClusterStatus = serde_json::from_slice(&bytes) + .map_err(|_| ReadError::Invalid("invalid cluster status report"))?; + let observation = report.observation; + if observation.version != 1 + || !matches!(observation.initial_voters, 1 | 3) + || observation.cluster.is_empty() + || observation.policy.as_ref().map_or(true, |policy| { + policy.validate().is_err() + || policy.cluster != observation.cluster + || policy.initial_voters != observation.initial_voters + }) + || (observation.bound && observation.cluster_id.is_none()) + { + return Err(ReadError::Invalid("invalid cluster status identity or policy")); + } + let report = readiness::reduce_current(observation, deadline).await; + if report + .reasons + .iter() + .any(|reason| reason.code == "invalid-observation") + { + return Err(ReadError::Invalid("invalid cluster status observation")); + } + Ok(report) + }) + .await + .map_err(|_| ReadError::Unavailable)? +} + +fn rank(phase: ClusterPhase) -> u8 { + match phase { + ClusterPhase::Ready => 0, + ClusterPhase::Waiting => 1, + ClusterPhase::Initializing => 2, + ClusterPhase::Degraded => 3, + ClusterPhase::Unavailable => 4, + ClusterPhase::Failed => 5, + } +} +fn failure(error: impl std::fmt::Display) -> ExitCode { + eprintln!("error: cluster observation: {error}"); + ExitCode::from(2) +} diff --git a/app/crowdb-cli/src/commands/cluster/target.rs b/app/crowdb-cli/src/commands/cluster/target.rs new file mode 100644 index 000000000..1c9b05229 --- /dev/null +++ b/app/crowdb-cli/src/commands/cluster/target.rs @@ -0,0 +1,86 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Resolve remote application targets before dispatching logical operations. + +use std::process::ExitCode; + +use crowdb_console_shared::deployment::target::{self, ClusterTargetBinding, ResolvedClusterTarget}; + +use crate::Cli; + +pub(crate) async fn resolve(cli: &Cli) -> Result, ExitCode> { + if matches!( + cli.command, + crate::Domain::Cluster { + verb: super::ClusterVerb::Status { .. } + | super::ClusterVerb::Check { .. } + | super::ClusterVerb::Start(_) + | super::ClusterVerb::Stop + } + ) { + return Ok(None); + } + if cli.cluster.is_none() && cli.management_endpoints.is_empty() { + if supports_attachment(&cli.command) { + return Err(failure( + "select --cluster or --management-endpoint before a cluster operation or workload", + )); + } + return Ok(None); + } + validate_command(&cli.command)?; + let binding = cli.cluster.as_deref().map(load).transpose().map_err(failure)?; + target::resolve(binding.as_ref(), &cli.management_endpoints, None) + .await + .map(Some) + .map_err(failure) +} + +fn validate_command(command: &crate::Domain) -> Result<(), ExitCode> { + if !supports_attachment(command) { + return Err(failure( + "this operation does not support application target attachment", + )); + } + Ok(()) +} + +fn supports_attachment(command: &crate::Domain) -> bool { + use crate::{ + commands::{BenchVerb, ClusterVerb}, + Domain, KvVerb, + }; + matches!( + command, + Domain::Cluster { + verb: ClusterVerb::Status { .. } + | ClusterVerb::Check { .. } + | ClusterVerb::Topology { .. } + | ClusterVerb::Rack { .. } + | ClusterVerb::Node { .. } + | ClusterVerb::DiskGroup { .. } + | ClusterVerb::Disk { .. } + } | Domain::Kv { + verb: KvVerb::Store(_) | KvVerb::Group(_) | KvVerb::Replica(_) | KvVerb::Data(_) + } | Domain::S3 { .. } + | Domain::Bench { + verb: BenchVerb::Kv(_) + | BenchVerb::Diskdb(_) + | BenchVerb::Chunkdb(_) + | BenchVerb::Chunkio(_) + | BenchVerb::S3(_) + } + ) +} + +pub(super) fn load(name: &str) -> Result { + super::local::load_target(name) + .map_err(|error| error.to_string())? + .ok_or_else(|| "selected cluster has no readable target binding".into()) +} + +fn failure(error: impl std::fmt::Display) -> ExitCode { + eprintln!("error: resolve cluster target: {error}"); + ExitCode::from(2) +} diff --git a/app/crowdb-cli/src/commands/container_cluster.rs b/app/crowdb-cli/src/commands/container_cluster.rs deleted file mode 100644 index c41aef51b..000000000 --- a/app/crowdb-cli/src/commands/container_cluster.rs +++ /dev/null @@ -1,289 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Host CLI lifecycle for an owned three-node release-container cluster. - -mod bootstrap; -mod maintenance; -mod runtime; -mod state; - -use clap::Args; -use crowdb_console_shared::error::{Error, Result}; -use serde_json::Value; -use state::{State, FILE, INITIALIZING, ROOT_LABEL}; -use std::path::PathBuf; - -#[derive(Args, Debug)] -#[group(required = true, multiple = true)] -pub(crate) struct Target { - #[arg(long, alias = "data-dir")] - pub root: Option, - #[arg(long, alias = "name")] - pub container: Option, -} - -pub(crate) struct Status { - pub root: PathBuf, - pub container: String, - pub image: String, - pub running: usize, - pub total: usize, - pub endpoint: String, - pub web: String, -} - -pub(crate) async fn resolve(target: &Target, create: bool) -> Result { - if let Some(name) = &target.container { - state::validate_member_name(name)?; - } - let mut observed = None; - if let Some(name) = &target.container { - if let Some(value) = runtime::inspect("container", name).await? { - let path = value["Config"]["Labels"][ROOT_LABEL] - .as_str() - .ok_or_else(|| Error::Config("container has no cluster ownership label".into()))?; - let root = std::fs::canonicalize(path)?; - let state = State::load(&root)?; - let index = (0..3) - .find(|index| state.node_name(*index) == *name) - .ok_or_else(|| Error::Config("container is not a recorded cluster member".into()))?; - runtime::verify(&state, index, &value)?; - observed = Some(root); - } - } - let selected = target - .root - .clone() - .or_else(|| observed.clone()) - .or_else(|| { - target.container.as_ref().map(|name| { - crowdb_protocol::port::namespace::runtime_root() - .join("persistent/container-clusters") - .join(name) - }) - }) - .ok_or_else(|| Error::Config("select a cluster root or member container".into()))?; - if create { - std::fs::create_dir_all(&selected)?; - } - let root = std::fs::canonicalize(selected)?; - if observed.is_some_and(|observed| observed != root) { - return Err(Error::Config( - "root and container select different clusters".into(), - )); - } - Ok(root) -} - -pub(crate) async fn start( - target: &Target, - image: Option<&str>, - node_cpus: Option, - memory_mib: Option, - diskdb_heartbeat_interval_secs: Option, -) -> Result { - let root = resolve(target, true).await?; - let mut state = if root.join(FILE).is_file() { - State::load(&root)? - } else { - if std::fs::read_dir(&root)?.next().transpose()?.is_some() { - return Err(Error::Config( - "refusing a non-empty unrecognized cluster root".into(), - )); - } - let name = target - .container - .clone() - .unwrap_or_else(|| format!("crowdb-cluster-{}", state::random_id())); - let image = image.unwrap_or("crowdb-node:dev"); - let value = runtime::inspect("image", image) - .await? - .ok_or_else(|| Error::Config("formal release image is absent".into()))?; - if value["Config"]["Labels"]["org.crowdb.runtime.sha256"] - .as_str() - .is_none() - || value["Os"] != "linux" - { - return Err(Error::Config( - "cluster needs the formal Linux CROWDB release image".into(), - )); - } - let state = State { - version: 1, - id: state::random_id(), - root: root.clone(), - image: value["Id"] - .as_str() - .ok_or_else(|| Error::Config("image identity is absent".into()))? - .into(), - network: format!("{name}-network"), - name, - phase: "allocated".into(), - node_cpus: node_cpus.unwrap_or(1.0), - memory_mib: memory_mib.unwrap_or(2048), - diskdb_heartbeat_interval_secs: diskdb_heartbeat_interval_secs - .unwrap_or(state::DEFAULT_DISKDB_HEARTBEAT_INTERVAL_SECS), - addresses: Vec::new(), - }; - state.save()?; - state - }; - if let Some(image) = image { - let current = runtime::inspect("image", image) - .await? - .ok_or_else(|| Error::Config("requested release image is absent".into()))?; - if current["Id"] != state.image { - return Err(Error::Config("existing cluster keeps its recorded release image; explicit upgrades require a separate operation".into())); - } - } - validate_requested_settings(&state, node_cpus, memory_mib, diskdb_heartbeat_interval_secs)?; - if target - .container - .as_ref() - .is_some_and(|name| !(0..3).any(|index| state.node_name(index) == *name)) - { - return Err(Error::Config( - "container name does not select the recorded cluster".into(), - )); - } - verify_members(&state).await?; - runtime::ensure_network(&mut state).await?; - runtime::ensure_nodes(&state).await?; - state.phase = "nodes".into(); - state.save()?; - bootstrap::run(&mut state).await?; - state.phase = "ready".into(); - state.save()?; - state.client_record()?; - if root.join(INITIALIZING).exists() { - std::fs::remove_file(root.join(INITIALIZING))?; - } - status(target).await -} - -fn validate_requested_settings( - state: &State, - node_cpus: Option, - memory_mib: Option, - diskdb_heartbeat_interval_secs: Option, -) -> Result<()> { - if node_cpus.is_some_and(|cpus| cpus.to_bits() != state.node_cpus.to_bits()) - || memory_mib.is_some_and(|memory| memory != state.memory_mib) - { - return Err(Error::Config( - "existing cluster keeps its recorded resource limits".into(), - )); - } - if diskdb_heartbeat_interval_secs.is_some_and(|interval| interval != state.diskdb_heartbeat_interval_secs) - { - return Err(Error::Config( - "existing cluster keeps its recorded Diskdb heartbeat configuration".into(), - )); - } - Ok(()) -} - -async fn verify_members(state: &State) -> Result>> { - let mut values = Vec::new(); - if let Some(network) = runtime::inspect("network", &state.network).await? { - runtime::verify_network(state, &network)?; - } - for index in 0..3 { - let value = runtime::inspect("container", &state.node_name(index)).await?; - if let Some(value) = &value { - runtime::verify(state, index, value)?; - } - values.push(value); - } - Ok(values) -} - -pub(crate) async fn status(target: &Target) -> Result { - let root = resolve(target, false).await?; - let state = State::load(&root)?; - let values = verify_members(&state).await?; - let web = state.web().unwrap_or_default(); - Ok(Status { - root, - container: state.name, - image: state.image, - running: values - .iter() - .flatten() - .filter(|v| v["State"]["Running"] == true) - .count(), - total: 3, - endpoint: format!("{web}/api/access/s3"), - web, - }) -} - -pub(crate) async fn stop(target: &Target) -> Result { - let root = resolve(target, false).await?; - let mut state = State::load(&root)?; - let values = verify_members(&state).await?; - for index in (0..3).rev() { - if values[index] - .as_ref() - .is_some_and(|value| value["State"]["Running"] == true) - { - runtime::docker(&[ - "stop".into(), - "--time".into(), - "10".into(), - state.node_name(index), - ]) - .await?; - } - } - state.phase = "stopped".into(); - state.save()?; - status(target).await -} - -pub(crate) async fn restart(target: &Target) -> Result { - stop(target).await?; - start(target, None, None, None, None).await -} - -pub(crate) async fn delete(target: &Target) -> Result { - let previous = stop(target).await?; - let state = State::load(&previous.root)?; - let values = verify_members(&state).await?; - for (index, value) in values.iter().enumerate() { - if value.is_some() { - runtime::docker(&["rm".into(), state.node_name(index)]).await?; - } - } - if let Some(network) = runtime::inspect("network", &state.network).await? { - runtime::verify_network(&state, &network)?; - runtime::docker(&["network".into(), "rm".into(), state.network.clone()]).await?; - } - let nodes = state.root.join("nodes"); - if nodes.exists() { - maintenance::run( - &state, - "delete-data", - &nodes, - "/bin/rm", - &[ - "-rf", - "/cluster/data/node0", - "/cluster/data/node1", - "/cluster/data/node2", - ], - ) - .await?; - std::fs::remove_dir(nodes)?; - } - state::remove_temporary(&state)?; - for file in [FILE, INITIALIZING, "s3-mini-cluster.json", "s3-local-state.toml"] { - let path = state.root.join(file); - if path.exists() { - std::fs::remove_file(path)?; - } - } - std::fs::remove_dir(&state.root)?; - Ok(previous) -} diff --git a/app/crowdb-cli/src/commands/container_cluster/bootstrap.rs b/app/crowdb-cli/src/commands/container_cluster/bootstrap.rs deleted file mode 100644 index f350ff481..000000000 --- a/app/crowdb-cli/src/commands/container_cluster/bootstrap.rs +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::state::State; -use crowdb_console_shared::error::{Error, Result}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; -use crowdb_protocol::mgmt::node::setup::{NodeSetupChoices, SetupStorage}; -use reqwest::{Client, Method}; -use serde_json::{json, Value}; -use std::time::{Duration, Instant}; - -mod storage; - -struct Api { - client: Client, - web: String, - kv: CrowdbKvClient, -} - -impl Api { - async fn request(&self, method: Method, path: &str, body: Option) -> Result { - self.request_at(&self.web, method, path, body).await - } - - async fn request_at( - &self, - origin: &str, - method: Method, - path: &str, - body: Option, - ) -> Result { - let read = method == Method::GET; - let mut request = self.client.request(method, format!("{origin}{path}")); - if read { - request = request.timeout(Duration::from_secs(2)); - } - if let Some(body) = body { - request = request.json(&body); - } - let response = request - .send() - .await - .map_err(|e| Error::Config(format!("container cluster request {path}: {e}")))?; - let status = response.status(); - let text = response - .text() - .await - .map_err(|error| Error::Config(format!("container cluster response {path}: {error:?}")))?; - if !status.is_success() { - return Err(Error::Config(format!( - "container cluster {path}: {status}: {text}" - ))); - } - if text.is_empty() { - Ok(Value::Null) - } else { - serde_json::from_str(&text).map_err(|e| Error::Config(e.to_string())) - } - } - async fn get(&self, path: &str) -> Result { - self.request(Method::GET, path, None).await - } - async fn post(&self, path: &str, body: Value) -> Result { - self.request(Method::POST, path, Some(body)).await - } -} - -pub(super) async fn run(state: &mut State) -> Result<()> { - let api = Api { - client: Client::builder() - .no_proxy() - .timeout(Duration::from_secs(20)) - .build() - .map_err(|e| Error::Config(e.to_string()))?, - web: state.web()?, - kv: CrowdbKvClient::new(ClientConfig::new( - state - .addresses - .iter() - .map(|address| format!("http://{address}:10000")) - .collect(), - )), - }; - wait_web(&api, state).await?; - // This CLI owns fixed node identities and explicitly initializes Group 0. - for address in &state.addresses { - api.request_at( - &format!("http://{address}:9090"), - Method::POST, - "/api/node/auto-join", - Some(json!({"enabled": false})), - ) - .await?; - } - let phase = api.get("/api/node/status").await?; - if phase["phase"] != "active" { - prepare(&api, state).await?; - let phase = api.get("/api/node/status").await?; - if phase["phase"] != "active" { - api.post("/api/cluster/init", json!({"nodes":[1,2,3]})).await?; - } - } - wait_phase(&api, "active").await?; - state.phase = "bound".into(); - state.client_record()?; - state.save()?; - storage::prepare(&api, state).await?; - storage::services(&api, state).await?; - Ok(()) -} - -async fn wait_web(api: &Api, state: &State) -> Result<()> { - for address in &state.addresses { - let deadline = Instant::now() + Duration::from_secs(20); - loop { - if api - .client - .get(format!("http://{address}:9090/healthz")) - .timeout(Duration::from_secs(2)) - .send() - .await - .is_ok_and(|r| r.status().is_success()) - { - break; - } - if Instant::now() >= deadline { - return Err(Error::Config(format!( - "container Web endpoint http://{address}:9090 is not reachable from the host" - ))); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - } - Ok(()) -} - -async fn wait_phase(api: &Api, phase: &str) -> Result<()> { - let deadline = Instant::now() + Duration::from_secs(10); - loop { - let observed = api.get("/api/node/status").await?; - if observed["phase"] == phase { - return Ok(()); - } - if Instant::now() >= deadline { - return Err(Error::Config(format!( - "container cluster phase is not {phase}: {observed}" - ))); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } -} - -async fn prepare(api: &Api, state: &mut State) -> Result<()> { - let racks = api.get("/api/racks").await?; - if !racks - .as_array() - .is_some_and(|racks| racks.iter().any(|rack| rack["id"] == 1)) - { - api.post("/api/racks", json!({"id":1,"name":"container cluster"})) - .await?; - } - let deadline = Instant::now() + Duration::from_secs(10); - let candidates = loop { - let observed = api.get("/api/node/candidates").await?; - if observed["candidates"] - .as_array() - .is_some_and(|entries| entries.len() == 3) - { - break observed; - } - if Instant::now() >= deadline { - return Err(Error::Config("three container nodes were not discovered".into())); - } - tokio::time::sleep(Duration::from_millis(100)).await; - }; - let entries = candidates["candidates"] - .as_array() - .ok_or_else(|| Error::Config("node candidates are invalid".into()))?; - for (index, address) in state.addresses.iter().enumerate() { - let candidate = entries - .iter() - .find(|entry| { - entry["advertisement"]["monitor_endpoints"] - .as_array() - .is_some_and(|endpoints| { - endpoints.iter().any(|endpoint| { - endpoint - .as_str() - .and_then(|s| reqwest::Url::parse(s).ok()) - .is_some_and(|url| url.host_str() == Some(address)) - }) - }) - }) - .ok_or_else(|| Error::Config("container endpoint has no discovered node identity".into()))?; - let discovery = &candidate["advertisement"]["discovery_id"]; - let admissions = api.get("/api/node/admissions").await?; - if let Some(record) = admissions - .as_array() - .and_then(|nodes| nodes.iter().find(|node| node["discovery_id"] == *discovery)) - { - if record["node_id"] != index + 1 { - return Err(Error::Config( - "container node admission changed its fixed identity".into(), - )); - } - continue; - } - let password = std::env::var("CROWDB_SSH_PASSWORD").unwrap_or_else(|_| "crowdb".into()); - // The CLI resumes its own fixed storage and service deployment steps. - let mut setup = NodeSetupChoices { - storage: SetupStorage::Diskless, - ..Default::default() - }; - for service in &mut setup.services { - service.enabled = false; - } - let record = api - .post( - "/api/node/admit", - json!({"discovery_id":discovery,"rack_id":1,"ssh_user":"crowdb", - "ssh_port":2222,"ssh_password":password,"setup":setup}), - ) - .await?; - if record["node_id"] != index + 1 { - return Err(Error::Config( - "container node admission returned a different identity".into(), - )); - } - } - state.phase = "prepared".into(); - state.save() -} diff --git a/app/crowdb-cli/src/commands/container_cluster/bootstrap/storage.rs b/app/crowdb-cli/src/commands/container_cluster/bootstrap/storage.rs deleted file mode 100644 index 27300517f..000000000 --- a/app/crowdb-cli/src/commands/container_cluster/bootstrap/storage.rs +++ /dev/null @@ -1,253 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{json, Api, Duration, Error, Instant, Result, State, Value}; -use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; - -mod files; - -pub(super) async fn prepare(api: &Api, state: &mut State) -> Result<()> { - let groups = api.get("/api/stores/0/groups").await?; - if !array(&groups, "Store groups")? - .iter() - .any(|group| group["group_id"] == 1) - { - api.post( - "/api/stores/0/groups", - json!({"group_id":1,"replica_id":10,"nodes":[1,2,3]}), - ) - .await?; - } - for node in 1..=3 { - let groups = api.get(&format!("/api/nodes/{node}/disk-groups")).await?; - if !array(&groups, "Disk groups")? - .iter() - .any(|group| group["id"] == node) - { - api.post( - &format!("/api/nodes/{node}/disk-groups"), - json!({"id":node,"store_id":0,"group_id":1,"name":"container storage"}), - ) - .await?; - } - if !has_service(api, node, &format!("diskdb-{node}")).await? { - api.post( - &format!("/api/nodes/{node}/diskdb/deploy"), - json!({"rpc_port":7100,"keepalive_interval_secs":state.diskdb_heartbeat_interval_secs}), - ) - .await?; - state.phase = "storage".into(); - state.save()?; - } - let disks = api - .get(&format!("/api/nodes/{node}/disk-groups/{node}/disks")) - .await?; - if disks - .as_array() - .ok_or_else(|| Error::Config("disk inventory is invalid".into()))? - .is_empty() - { - files::prepare(api, state, node).await?; - } - } - wait_http(api, &state.addresses, 7101, "/ready", "Diskdb").await?; - wait_diskdb_ownership(api, state).await -} - -async fn wait_diskdb_ownership(api: &Api, state: &State) -> Result<()> { - let seeds = state - .addresses - .iter() - .map(|address| format!("http://{address}:10000")) - .collect(); - let group = api.get("/api/stores/0/groups/0").await?; - let leader = array(&group["replicas"], "Group-0 replicas")? - .iter() - .find(|replica| replica["role"] == "leader") - .and_then(|replica| replica["node_id"].as_u64()) - .and_then(|node| usize::try_from(node).ok()) - .and_then(|node| node.checked_sub(1)) - .and_then(|index| state.addresses.get(index)) - .ok_or_else(|| Error::Config("Group 0 has no observed cluster leader".into()))?; - let client = CrowdbKvClient::new(ClientConfig::new(seeds)); - client.seed_leader(0, 0, format!("{leader}:10100")); - let registry = ServiceRegistryClient::new(client); - let deadline = Instant::now() + Duration::from_secs(10); - loop { - let remaining = deadline.saturating_duration_since(Instant::now()); - let instances = tokio::time::timeout( - remaining.min(Duration::from_secs(2)), - registry.read_all_diskdb_instances(), - ) - .await; - let last = match instances { - Ok(Ok(instances)) => { - let endpoints: Vec<_> = state - .addresses - .iter() - .map(|address| format!("{address}:7102")) - .collect(); - let missing: Vec<_> = (0..3) - .filter_map(|index| { - let group = u64::try_from(index + 1).expect("three node index"); - (!instances.iter().any(|(_, value)| { - endpoints.contains(&value.rpc_endpoint) - && value - .extra - .as_ref() - .and_then(|extra| extra.diskdb.as_ref()) - .is_some_and(|extra| extra.owned_dg_ids.contains(&group)) - })) - .then_some(group) - }) - .collect(); - if missing.is_empty() { - return Ok(()); - } - format!("disk groups {missing:?} have no live Diskdb endpoint") - } - Ok(Err(error)) => error.to_string(), - Err(error) => error.to_string(), - }; - if Instant::now() >= deadline { - return Err(Error::Config(format!( - "Diskdb ownership is not published: {last}" - ))); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } -} - -fn array<'a>(value: &'a Value, description: &str) -> Result<&'a Vec> { - value - .as_array() - .ok_or_else(|| Error::Config(format!("{description} inventory is invalid"))) -} - -async fn has_service(api: &Api, node: usize, id: &str) -> Result { - // Deployment ownership comes from committed intent, independently of - // whether the monitor has finished restarting the retained process. - let node = u64::try_from(node).map_err(|error| Error::Config(error.to_string()))?; - let intent = crowdb_console_shared::deployment::services::current(&api.kv, node, id) - .await - .map_err(|error| Error::Config(format!("read deployment intent {node}/{id}: {error:?}")))?; - Ok(intent.is_some_and(|intent| intent.action != crowdb_protocol::mgmt::node::NodeServiceAction::Delete)) -} - -pub(super) async fn services(api: &Api, state: &mut State) -> Result<()> { - for (kind, http, rpc) in [ - ("diskio", None, Some(7200)), - ("chunkdb", Some(7300), Some(7301)), - ("chunk-kv", Some(7400), Some(7401)), - ("access-server", Some(9000), None), - ] { - if kind == "chunkdb" { - wait_storage(api).await?; - } - if kind == "access-server" { - wait_chunk_kv(api, state).await?; - } - for node in 1..=3 { - if !has_service(api, node, &format!("{kind}-{node}")).await? { - let defaults = api - .get(&format!("/api/deployment-defaults?node_id={node}")) - .await?; - let mut body = defaults[kind].clone(); - body["kind"] = json!(kind); - body["instance_id"] = json!(node.to_string()); - body["test_single_node"] = json!(false); - if let Some(port) = http { - body["http_port"] = json!(port); - } - if let Some(port) = rpc { - body["rpc_port"] = json!(port); - } - match kind { - "diskio" => body["disk_group_id"] = json!(node), - "chunk-kv" => { - body["metadata_store_id"] = json!(0); - body["bootstrap_group_id"] = json!(1); - } - "access-server" => { - body["s3_port"] = json!(9001); - body["health_port"] = json!(9002); - } - _ => {} - } - api.post(&format!("/api/nodes/{node}/services/deploy"), body) - .await?; - state.phase = format!("deploying-{kind}"); - state.save()?; - } - let address = std::slice::from_ref(&state.addresses[node - 1]); - match kind { - "chunkdb" => wait_http(api, address, 7300, "/ready", "Chunkdb").await?, - "chunk-kv" => wait_http(api, address, 7400, "/ready", "Chunk-KV catalog").await?, - "access-server" => wait_http(api, address, 9002, "/_crowdb/health/ready", "Access").await?, - _ => {} - } - } - } - wait_access(api, state).await -} - -async fn wait_storage(api: &Api) -> Result<()> { - let deadline = Instant::now() + Duration::from_secs(10); - loop { - let state = api.get("/api/chunk-storage-readiness").await?; - if state["ready"] == true { - return Ok(()); - } - if Instant::now() >= deadline { - return Err(Error::Config(format!("DiskIO ownership is not ready: {state}"))); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } -} - -async fn wait_chunk_kv(api: &Api, state: &State) -> Result<()> { - wait_http(api, &state.addresses, 7400, "/ready", "Chunk-KV catalog").await -} - -async fn wait_access(api: &Api, state: &State) -> Result<()> { - wait_http(api, &state.addresses, 9002, "/_crowdb/health/ready", "Access S3").await -} - -async fn wait_http(api: &Api, addresses: &[String], port: u16, path: &str, description: &str) -> Result<()> { - for address in addresses { - let deadline = Instant::now() + Duration::from_secs(10); - let mut last; - loop { - match api - .client - .get(format!("http://{address}:{port}{path}")) - .timeout( - deadline - .saturating_duration_since(Instant::now()) - .min(Duration::from_secs(2)), - ) - .send() - .await - { - Ok(response) if response.status().is_success() => break, - Ok(response) => { - last = format!( - "{}: {}", - response.status(), - response.text().await.map_err(|error| Error::Config(format!( - "{description} readiness response from {address}:{port}{path}: {error:?}" - )))? - ); - } - Err(error) => last = error.to_string(), - } - if Instant::now() >= deadline { - return Err(Error::Config(format!( - "{description} at {address}:{port} is not ready: {last}" - ))); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - } - Ok(()) -} diff --git a/app/crowdb-cli/src/commands/container_cluster/bootstrap/storage/files.rs b/app/crowdb-cli/src/commands/container_cluster/bootstrap/storage/files.rs deleted file mode 100644 index 889a2e0b9..000000000 --- a/app/crowdb-cli/src/commands/container_cluster/bootstrap/storage/files.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Resume monitor-owned sparse file admission before starting `DiskIO`. - -use super::{json, Api, Duration, Error, Instant, Result, State}; - -pub(super) async fn prepare(api: &Api, state: &State, node: usize) -> Result<()> { - // Derive the operation from the durable cluster identity so an interrupted - // start resubmits the same intent rather than creating another medium. - let suffix = u64::from_str_radix(&state.id[20..], 16) - .map_err(|error| Error::Config(error.to_string()))? - ^ u64::try_from(node).map_err(|error| Error::Config(error.to_string()))?; - let operation = format!( - "{}-{}-{}-{}-{suffix:012x}", - &state.id[..8], - &state.id[8..12], - &state.id[12..16], - &state.id[16..20], - ); - let path = format!("/api/nodes/{node}/disk-files"); - let mut observed = api - .post( - &path, - json!({ - "operation_id": operation, - "disk_group_id": node, - "capacity_bytes": 16u64 * 1024 * 1024 * 1024, - "sparse": true, - }), - ) - .await?; - let deadline = Instant::now() + Duration::from_secs(10); - loop { - match observed["phase"].as_str() { - Some("waiting-for-diskio" | "registered") => return Ok(()), - Some("failed") => { - return Err(Error::Config(format!( - "container file disk admission failed: {observed}" - ))); - } - _ => {} - } - if Instant::now() >= deadline { - return Err(Error::Config(format!( - "container file disk admission is not published: {observed}" - ))); - } - tokio::time::sleep(Duration::from_millis(100)).await; - observed = api.get(&format!("{path}/{operation}")).await?; - } -} diff --git a/app/crowdb-cli/src/commands/container_cluster/maintenance.rs b/app/crowdb-cli/src/commands/container_cluster/maintenance.rs deleted file mode 100644 index 9c020d195..000000000 --- a/app/crowdb-cli/src/commands/container_cluster/maintenance.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{ - runtime, - state::{State, LABEL, ROOT_LABEL}, -}; -use crowdb_console_shared::error::{Error, Result}; -use std::path::Path; - -/// Bounded release-image filesystem work, recoverable after client interruption. -pub(super) async fn run( - state: &State, - operation: &str, - data: &Path, - executable: &str, - arguments: &[&str], -) -> Result<()> { - let name = format!("{}-{operation}", state.name); - remove_owned(state, &name, data).await?; - let mut args = vec![ - "run".into(), - "--rm".into(), - "--name".into(), - name.clone(), - "--network".into(), - "none".into(), - "--user".into(), - "0".into(), - "--label".into(), - format!("{LABEL}={}", state.id), - "--label".into(), - format!("{ROOT_LABEL}={}", state.root.display()), - "--cpu-shares".into(), - format!("{:.0}", state.node_cpus * 1024.0), - "--memory".into(), - "64m".into(), - "--pids-limit".into(), - "32".into(), - "--entrypoint".into(), - executable.into(), - "--mount".into(), - format!("type=bind,source={},target=/cluster/data", data.display()), - state.image.clone(), - ]; - args.extend(arguments.iter().map(|argument| (*argument).into())); - let result = runtime::docker(&args).await; - let cleanup = remove_owned(state, &name, data).await; - result?; - cleanup -} - -async fn remove_owned(state: &State, name: &str, data: &Path) -> Result<()> { - if let Some(value) = runtime::inspect("container", name).await? { - let mounted = value["Mounts"].as_array().is_some_and(|mounts| { - mounts.iter().any(|mount| { - mount["Source"].as_str() == data.to_str() && mount["Destination"] == "/cluster/data" - }) - }); - if value["Config"]["Labels"][LABEL] != state.id - || value["Config"]["Labels"][ROOT_LABEL].as_str() != state.root.to_str() - || value["Image"] != state.image - || !mounted - { - return Err(Error::Config( - "refusing an unowned filesystem helper container".into(), - )); - } - runtime::docker(&["rm".into(), "-f".into(), name.into()]).await?; - } - Ok(()) -} diff --git a/app/crowdb-cli/src/commands/container_cluster/runtime.rs b/app/crowdb-cli/src/commands/container_cluster/runtime.rs deleted file mode 100644 index 9ea91af12..000000000 --- a/app/crowdb-cli/src/commands/container_cluster/runtime.rs +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::state::{State, LABEL, ROOT_LABEL}; -use crowdb_console_shared::error::{Error, Result}; -use serde_json::Value; -use std::time::Duration; - -pub(super) async fn docker(args: &[String]) -> Result { - let program = std::env::var_os("CROWDB_CONTAINER_RUNTIME").unwrap_or_else(|| "docker".into()); - let output = tokio::time::timeout( - Duration::from_secs(60), - tokio::process::Command::new(program) - .args(args) - .kill_on_drop(true) - .output(), - ) - .await - .map_err(|_| Error::Config("container runtime command exceeded 60 seconds".into()))??; - if !output.status.success() { - return Err(Error::Config(format!( - "container runtime {}: {}", - args.first().map_or("operation", String::as_str), - String::from_utf8_lossy(&output.stderr) - ))); - } - String::from_utf8(output.stdout).map_err(|e| Error::Config(e.to_string())) -} - -pub(super) async fn inspect(kind: &str, name: &str) -> Result> { - let args = if kind == "container" { - vec!["inspect".into(), name.into()] - } else { - vec![kind.into(), "inspect".into(), name.into()] - }; - match docker(&args).await { - Ok(body) => Ok(serde_json::from_str::>(&body) - .map_err(|e| Error::Config(e.to_string()))? - .into_iter() - .next()), - Err(Error::Config(message)) - if message.contains("No such object") - || message.contains("No such container") - || (message.contains("network ") && message.contains(" not found")) => - { - Ok(None) - } - Err(error) => Err(error), - } -} - -pub(super) fn verify(state: &State, index: usize, value: &Value) -> Result<()> { - let labels = &value["Config"]["Labels"]; - let expected_name = format!("/{}", state.node_name(index)); - let mounted = value["Mounts"].as_array().is_some_and(|mounts| { - mounts.iter().any(|mount| { - mount["Source"].as_str() == state.data(index).to_str() - && mount["Destination"] == "/opt/crowdb/data" - }) - }); - let network = &value["NetworkSettings"]["Networks"][&state.network]; - let address = state - .addresses - .get(index) - .ok_or_else(|| Error::Config("cluster has no fixed node address".into()))?; - let fixed = network["IPAMConfig"]["IPv4Address"] == *address - && (value["State"]["Running"] != true || network["IPAddress"] == *address); - if labels[LABEL] != state.id - || labels[ROOT_LABEL].as_str() != state.root.to_str() - || value["Image"] != state.image - || value["Name"] != expected_name - || !mounted - || !fixed - { - return Err(Error::Config( - "refusing a container with different cluster, image or data ownership".into(), - )); - } - Ok(()) -} - -pub(super) fn verify_network(state: &State, value: &Value) -> Result<()> { - if value["Name"] != state.network - || value["Labels"][LABEL] != state.id - || value["Labels"][ROOT_LABEL].as_str() != state.root.to_str() - { - return Err(Error::Config( - "refusing a network with different cluster ownership".into(), - )); - } - Ok(()) -} - -pub(super) async fn ensure_network(state: &mut State) -> Result<()> { - if let Some(value) = inspect("network", &state.network).await? { - verify_network(state, &value)?; - } else { - docker(&[ - "network".into(), - "create".into(), - "--subnet".into(), - format!( - "10.{}.{}.0/24", - u8::from_str_radix(&state.id[..2], 16).map_err(|e| Error::Config(e.to_string()))?, - u8::from_str_radix(&state.id[2..4], 16).map_err(|e| Error::Config(e.to_string()))? - ), - "--label".into(), - format!("{LABEL}={}", state.id), - "--label".into(), - format!("{ROOT_LABEL}={}", state.root.display()), - state.network.clone(), - ]) - .await?; - } - let network = inspect("network", &state.network) - .await? - .ok_or_else(|| Error::Config("allocated cluster network is absent".into()))?; - verify_network(state, &network)?; - let subnet = network["IPAM"]["Config"][0]["Subnet"] - .as_str() - .and_then(|s| s.split_once('/')) - .ok_or_else(|| Error::Config("Docker bridge needs a reachable IPv4 subnet".into()))?; - let address = subnet - .0 - .parse::() - .map_err(|e| Error::Config(e.to_string()))?; - let prefix = subnet - .1 - .parse::() - .map_err(|e| Error::Config(e.to_string()))?; - if prefix == 0 || prefix > 29 || u32::from(address) & (!u32::MAX >> prefix) != 0 { - return Err(Error::Config( - "cluster bridge needs an aligned IPv4 subnet with three usable node addresses".into(), - )); - } - let base = u32::from(address); - let addresses: Vec<_> = (2..5) - .map(|n| std::net::Ipv4Addr::from(base + n).to_string()) - .collect(); - if !state.addresses.is_empty() && state.addresses != addresses { - return Err(Error::Config( - "cluster network changed fixed node addresses".into(), - )); - } - state.addresses = addresses; - state.save() -} - -pub(super) async fn ensure_nodes(state: &State) -> Result<()> { - let physical = docker(&["info".into(), "--format".into(), "{{.ID}}".into()]).await?; - for index in 0..3 { - let name = state.node_name(index); - if let Some(value) = inspect("container", &name).await? { - verify(state, index, &value)?; - if value["State"]["Running"] != true { - docker(&["start".into(), name]).await?; - } - continue; - } - let data = state.data(index); - std::fs::create_dir_all(&data)?; - super::maintenance::run( - state, - &format!("prepare-{index}"), - &data, - "/bin/chown", - &["-R", "crowdb:crowdb", "/cluster/data"], - ) - .await?; - let mut args = vec![ - "run".into(), - "-d".into(), - "--name".into(), - name, - "--network".into(), - state.network.clone(), - "--ip".into(), - state.addresses[index].clone(), - "--label".into(), - format!("{LABEL}={}", state.id), - "--label".into(), - format!("{ROOT_LABEL}={}", state.root.display()), - "--cpu-shares".into(), - format!("{:.0}", state.node_cpus * 1024.0), - "--memory".into(), - format!("{}m", state.memory_mib), - "--pids-limit".into(), - "512".into(), - "--ulimit".into(), - "core=-1".into(), - "--mount".into(), - format!("type=bind,source={},target=/opt/crowdb/data", data.display()), - "--env".into(), - "CROWDB_STARTUP_MODE=manual".into(), - "--env".into(), - "TOKIO_WORKER_THREADS=1".into(), - "--env".into(), - format!("CROWDB_PHYSICAL_HOST_ID=docker-{}", physical.trim()), - ]; - if std::env::var_os("CROWDB_SSH_PASSWORD").is_some() { - args.extend(["--env".into(), "CROWDB_SSH_PASSWORD".into()]); - } - args.push(state.image.clone()); - docker(&args).await?; - } - Ok(()) -} diff --git a/app/crowdb-cli/src/commands/container_cluster/state.rs b/app/crowdb-cli/src/commands/container_cluster/state.rs deleted file mode 100644 index 6d29a23b2..000000000 --- a/app/crowdb-cli/src/commands/container_cluster/state.rs +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use crowdb_console_shared::error::{Error, Result}; -use rand::RngCore; -use serde::{Deserialize, Serialize}; -use std::fmt::Write as _; -use std::{ - fs, - io::Write, - path::{Path, PathBuf}, -}; - -pub(super) const FILE: &str = "container-cluster.json"; -pub(super) const INITIALIZING: &str = "s3-mini-cluster.initializing.json"; -pub(super) const LABEL: &str = "org.crowdb.cluster.id"; -pub(super) const ROOT_LABEL: &str = "org.crowdb.cluster.root"; -pub(super) const DEFAULT_DISKDB_HEARTBEAT_INTERVAL_SECS: u32 = 10; - -fn default_diskdb_heartbeat_interval_secs() -> u32 { - DEFAULT_DISKDB_HEARTBEAT_INTERVAL_SECS -} - -#[derive(Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub(super) struct State { - pub version: u32, - pub id: String, - pub root: PathBuf, - pub image: String, - pub name: String, - pub network: String, - pub phase: String, - pub node_cpus: f64, - pub memory_mib: u64, - #[serde(default = "default_diskdb_heartbeat_interval_secs")] - pub diskdb_heartbeat_interval_secs: u32, - pub addresses: Vec, -} - -impl State { - pub fn node_name(&self, index: usize) -> String { - if index == 0 { - self.name.clone() - } else { - format!("{}-node{}", self.name, index + 1) - } - } - - pub fn data(&self, index: usize) -> PathBuf { - self.root.join(format!("nodes/node{index}/data")) - } - - pub fn web(&self) -> Result { - self.addresses - .first() - .map(|address| format!("http://{address}:9090")) - .ok_or_else(|| Error::Config("container cluster has no inspected endpoint".into())) - } - - pub fn load(root: &Path) -> Result { - let state: Self = serde_json::from_slice(&fs::read(root.join(FILE))?) - .map_err(|error| Error::Config(error.to_string()))?; - state.validate(root)?; - Ok(state) - } - - pub fn validate(&self, root: &Path) -> Result<()> { - if self.version != 1 - || self.id.len() != 32 - || !self.id.bytes().all(|b| b.is_ascii_hexdigit()) - || self.root != fs::canonicalize(root)? - || self.root.parent().is_none() - || !self.image.starts_with("sha256:") - || self.image.len() != 71 - || !self.image[7..].bytes().all(|b| b.is_ascii_hexdigit()) - || self.network != format!("{}-network", self.name) - || !self.node_cpus.is_finite() - || !(2.0..=262_144.0).contains(&(self.node_cpus * 1024.0).round()) - || self.memory_mib < 512 - || self.diskdb_heartbeat_interval_secs == 0 - || (!self.addresses.is_empty() - && (self.addresses.len() != 3 - || self - .addresses - .iter() - .any(|ip| ip.parse::().is_err()))) - { - return Err(Error::Config( - "container cluster identity, ownership or resources are invalid".into(), - )); - } - for index in 0..3 { - for path in [ - self.root.join("nodes"), - self.root.join(format!("nodes/node{index}")), - self.data(index), - self.data(index).join("disk.img"), - ] { - if fs::symlink_metadata(&path).is_ok_and(|metadata| metadata.file_type().is_symlink()) { - return Err(Error::Config( - "cluster data paths cannot be symbolic links".into(), - )); - } - } - } - validate_name(&self.name) - } - - pub fn save(&self) -> Result<()> { - self.validate(&self.root)?; - atomic( - &self.root.join(FILE), - &self.id, - &serde_json::to_vec_pretty(self).map_err(|e| Error::Config(e.to_string()))?, - )?; - if self.phase != "ready" && self.phase != "stopped" { - atomic( - &self.root.join(INITIALIZING), - &self.id, - &serde_json::to_vec_pretty(self).map_err(|e| Error::Config(e.to_string()))?, - )?; - } - Ok(()) - } - - pub fn client_record(&self) -> Result<()> { - let web = self.web()?; - let record = crowdb_console_shared::ops::s3::MiniClusterRecord { - version: 1, - protected_test: true, - endpoint: format!("{web}/api/access/s3"), - web_endpoint: web, - web_pid: None, - tenant: "default".into(), - storage_profile: crowdb_console_shared::ops::s3::StorageProfile::Persistent, - }; - atomic( - &self.root.join("s3-mini-cluster.json"), - &self.id, - &serde_json::to_vec_pretty(&record).map_err(|e| Error::Config(e.to_string()))?, - )?; - let origins: Vec<_> = self - .addresses - .iter() - .map(|ip| format!("http://{ip}:10000")) - .collect(); - let mut body = format!( - "version = 1\ngroup0_seeds = {}\n", - serde_json::to_string(&origins).map_err(|e| Error::Config(e.to_string()))? - ); - for (index, origin) in origins.iter().enumerate() { - writeln!(body, "\n[[service]]\nid = \"kv-{}\"\nurl = \"{origin}\"\nnode_id = {}\nservice_type = \"paxos-kv\"\nauto_start = false", index + 1, index + 1) - .map_err(|error| Error::Config(error.to_string()))?; - } - atomic(&self.root.join("s3-local-state.toml"), &self.id, body.as_bytes()) - } -} - -pub(super) fn random_id() -> String { - let mut bytes = [0; 16]; - rand::thread_rng().fill_bytes(&mut bytes); - hex::encode(bytes) -} - -pub(super) fn validate_name(name: &str) -> Result<()> { - validate_member_name(name)?; - if name.len() > 48 { - return Err(Error::Validation { - field: "container".into(), - message: "cluster name must contain at most 48 characters".into(), - }); - } - Ok(()) -} - -pub(super) fn validate_member_name(name: &str) -> Result<()> { - if name.is_empty() - || name.len() > 64 - || !name.as_bytes()[0].is_ascii_alphanumeric() - || !name - .bytes() - .all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)) - { - return Err(Error::Validation { field: "container".into(), message: "member container name must start with an alphanumeric character and contain at most 64 alphanumeric, dot, dash or underscore characters".into() }); - } - Ok(()) -} - -pub(super) fn atomic(path: &Path, owner: &str, bytes: &[u8]) -> Result<()> { - use std::os::unix::fs::OpenOptionsExt; - let parent = path - .parent() - .ok_or_else(|| Error::Config("cluster state parent absent".into()))?; - let name = path - .file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| Error::Config("cluster state filename is invalid".into()))?; - let temporary = parent.join(format!("{name}.{owner}.{}.tmp", random_id())); - let result = (|| { - let mut file = fs::OpenOptions::new() - .create_new(true) - .write(true) - .mode(0o600) - .open(&temporary)?; - file.write_all(bytes)?; - file.sync_all()?; - fs::rename(&temporary, path)?; - fs::File::open(parent)?.sync_all() - })(); - if result.is_err() { - let _ = fs::remove_file(&temporary); - } - result.map_err(Error::Io) -} - -pub(super) fn remove_temporary(state: &State) -> Result<()> { - for entry in fs::read_dir(&state.root)? { - let entry = entry?; - let name = entry.file_name(); - let Some(name) = name.to_str() else { - continue; - }; - let owned = [FILE, INITIALIZING, "s3-mini-cluster.json", "s3-local-state.toml"] - .iter() - .any(|file| { - name.strip_prefix(&format!("{file}.{}.", state.id)) - .and_then(|nonce| nonce.strip_suffix(".tmp")) - .is_some_and(|nonce| nonce.len() == 32 && nonce.bytes().all(|b| b.is_ascii_hexdigit())) - }); - if owned && entry.file_type()?.is_file() { - fs::remove_file(entry.path())?; - } - } - Ok(()) -} diff --git a/app/crowdb-cli/src/commands/kv.rs b/app/crowdb-cli/src/commands/kv.rs index 62c942a89..0b41af25c 100644 --- a/app/crowdb-cli/src/commands/kv.rs +++ b/app/crowdb-cli/src/commands/kv.rs @@ -1,16 +1,14 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! `kv` domain — server lifecycle, logical concepts (store/group/replica), +//! `kv` domain — logical concepts (store/group/replica), //! and data-plane operations. Each sub-module owns one slice of the //! domain. pub mod data; pub mod logical; -pub mod server; pub(crate) use data::{run_kv_data_verb, KvDataVerb}; pub(crate) use logical::{ run_group_verb, run_replica_verb, run_store_verb, GroupVerb, ReplicaVerb, StoreVerb, }; -pub(crate) use server::{run_kv_server_verb, KvServerVerb}; diff --git a/app/crowdb-cli/src/commands/kv/server.rs b/app/crowdb-cli/src/commands/kv/server.rs deleted file mode 100644 index 652528c13..000000000 --- a/app/crowdb-cli/src/commands/kv/server.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! `kv server` process controls backed by a launch-only registry. - -use std::process::ExitCode; - -use clap::Subcommand; - -use crate::Cli; - -mod registry; - -#[derive(Subcommand, Debug)] -pub enum KvServerVerb { - Deploy { - #[arg(short = 'n', long)] - node: String, - }, - Start { - #[arg(short = 'n', long)] - node: String, - }, - Restart { - #[arg(short = 'n', long)] - node: String, - }, - Stop { - #[arg(short = 'n', long)] - node: String, - }, - Delete { - #[arg(short = 'n', long)] - node: String, - }, - List, -} - -pub async fn run_kv_server_verb(cli: &Cli, verb: KvServerVerb) -> ExitCode { - let Some(path) = &cli.registry else { - eprintln!("error: kv server controls require --registry"); - return ExitCode::from(2); - }; - registry::run(cli, path, verb).await -} diff --git a/app/crowdb-cli/src/commands/kv/server/registry.rs b/app/crowdb-cli/src/commands/kv/server/registry.rs deleted file mode 100644 index 30b993a03..000000000 --- a/app/crowdb-cli/src/commands/kv/server/registry.rs +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use std::path::Path; -use std::process::ExitCode; - -use crowdb_console_shared::config::web::LaunchRegistry; -use crowdb_console_shared::error::{Error, Result}; -use crowdb_console_shared::launch::LaunchRuntime; - -use super::KvServerVerb; -use crate::Cli; - -pub(super) async fn run(cli: &Cli, path: &Path, verb: KvServerVerb) -> ExitCode { - match execute(cli, path, verb).await { - Ok(()) => ExitCode::SUCCESS, - Err(error) => { - eprintln!("error: {error}"); - ExitCode::from(2) - } - } -} - -async fn execute(cli: &Cli, path: &Path, verb: KvServerVerb) -> Result<()> { - let mut registry = LaunchRegistry::load(path)?; - let runtime = LaunchRuntime::for_registry(path)?; - if matches!(verb, KvServerVerb::List) { - println!("{:<12} {:<26} PID", "NODE", "HOST"); - for launch in registry - .launches - .iter() - .filter(|launch| launch.service_id == "kv") - { - let identity = runtime.status(launch).await?; - println!( - "{:<12} {:<26} {}", - launch.node_id, - launch.host, - identity.map_or_else(|| "stopped".into(), |identity| identity.pid.to_string()) - ); - } - return Ok(()); - } - let node = match &verb { - KvServerVerb::Deploy { node, .. } - | KvServerVerb::Start { node } - | KvServerVerb::Restart { node } - | KvServerVerb::Stop { node } - | KvServerVerb::Delete { node } => node, - KvServerVerb::List => unreachable!(), - } - .parse::() - .map_err(|error| Error::Validation { - field: "node".into(), - message: error.to_string(), - })?; - let launch = registry - .launches - .iter() - .find(|launch| launch.node_id == node && launch.service_id == "kv") - .cloned() - .ok_or_else(|| Error::NotFound { - kind: "configured kv launch".into(), - id: node.to_string(), - })?; - match verb { - KvServerVerb::Deploy { .. } | KvServerVerb::Start { .. } => { - let identity = runtime.start(&launch).await?; - println!("started kv on node {node} (pid {})", identity.pid); - } - KvServerVerb::Restart { .. } => { - let identity = runtime.restart(&launch).await?; - println!("restarted kv on node {node} (pid {})", identity.pid); - } - KvServerVerb::Stop { .. } => { - runtime.stop(&launch).await?; - println!("stopped kv on node {node}"); - } - KvServerVerb::Delete { .. } => { - let ctx = crate::commands::authority_context(cli) - .await - .map_err(|_| Error::Config("cannot initialize authority client".into()))?; - if ctx - .sysmd() - .list_all_replicas() - .await? - .iter() - .any(|replica| replica.node_id == node) - { - return Err(Error::Conflict { - kind: "node with replica membership".into(), - id: node.to_string(), - }); - } - runtime.stop(&launch).await?; - registry - .launches - .retain(|record| record.node_id != node || record.service_id != "kv"); - registry.save(path)?; - println!("removed kv launch for node {node}"); - } - KvServerVerb::List => unreachable!(), - } - Ok(()) -} diff --git a/app/crowdb-cli/src/commands/launch.rs b/app/crowdb-cli/src/commands/launch.rs deleted file mode 100644 index 83ccfd36a..000000000 --- a/app/crowdb-cli/src/commands/launch.rs +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Process controls using launch policy, independent of cluster topology. - -use std::process::ExitCode; - -use clap::Subcommand; -use crowdb_console_shared::config::web::LaunchRegistry; -use crowdb_console_shared::error::{Error, Result}; -use crowdb_console_shared::launch::LaunchRuntime; - -use crate::Cli; - -#[derive(Subcommand, Debug)] -pub enum LaunchVerb { - /// Show configured processes and their current runtime identities. - List, - /// Start a configured process, preserving an already running instance. - Start { - #[arg(long)] - node: u64, - #[arg(long)] - service: String, - }, - Restart { - #[arg(long)] - node: u64, - #[arg(long)] - service: String, - }, - Stop { - #[arg(long)] - node: u64, - #[arg(long)] - service: String, - }, -} - -pub(crate) async fn run(cli: &Cli, verb: LaunchVerb) -> ExitCode { - match execute(cli, verb).await { - Ok(()) => ExitCode::SUCCESS, - Err(error) => { - eprintln!("error: {error}"); - ExitCode::from(2) - } - } -} - -async fn execute(cli: &Cli, verb: LaunchVerb) -> Result<()> { - let path = cli - .registry - .as_ref() - .ok_or_else(|| Error::Config("process controls require --registry".into()))?; - let registry = LaunchRegistry::load(path)?; - let runtime = LaunchRuntime::for_registry(path)?; - if matches!(verb, LaunchVerb::List) { - println!("{:<12} {:<16} {:<26} PID", "NODE", "SERVICE", "HOST"); - for launch in registry.launches { - let identity = runtime.status(&launch).await?; - println!( - "{:<12} {:<16} {:<26} {}", - launch.node_id, - launch.service_id, - launch.host, - identity.map_or_else(|| "stopped".into(), |identity| identity.pid.to_string()) - ); - } - return Ok(()); - } - let (node, service) = match &verb { - LaunchVerb::Start { node, service } - | LaunchVerb::Restart { node, service } - | LaunchVerb::Stop { node, service } => (*node, service.as_str()), - LaunchVerb::List => unreachable!(), - }; - let launch = registry - .launches - .iter() - .find(|launch| launch.node_id == node && launch.service_id == service) - .ok_or_else(|| Error::NotFound { - kind: "configured launch".into(), - id: format!("{node}/{service}"), - })?; - match &verb { - LaunchVerb::Start { .. } => { - let identity = runtime.start(launch).await?; - println!("started {service} on node {node} (pid {})", identity.pid); - } - LaunchVerb::Restart { .. } => { - let identity = runtime.restart(launch).await?; - println!("restarted {service} on node {node} (pid {})", identity.pid); - } - LaunchVerb::Stop { .. } => { - runtime.stop(launch).await?; - println!("stopped {service} on node {node}"); - } - LaunchVerb::List => unreachable!(), - } - Ok(()) -} diff --git a/app/crowdb-cli/src/commands/s3.rs b/app/crowdb-cli/src/commands/s3.rs index 3384cca82..5454e2a12 100644 --- a/app/crowdb-cli/src/commands/s3.rs +++ b/app/crowdb-cli/src/commands/s3.rs @@ -2,23 +2,24 @@ // Licensed under the Apache License, Version 2.0. use std::io::{Read, Write}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::ExitCode; use clap::Subcommand; -use quick_xml::{Reader, Writer}; use rand::RngCore; use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE}; use reqwest::Method; use crate::Cli; +use crowdb_console_shared::ops::s3::S3HttpClient; + +mod output; +mod target; +use output::{format_structured_body, print_body_preview, print_exchange}; +pub(super) use target::client; #[derive(Subcommand, Debug)] pub enum S3Verb { - Cluster { - #[command(subcommand)] - verb: S3ClusterVerb, - }, Bucket { #[command(subcommand)] verb: S3BucketVerb, @@ -29,65 +30,19 @@ pub enum S3Verb { }, } -#[derive(Subcommand, Debug)] -pub enum S3ClusterVerb { - /// Create or resume an owned three-node release-container cluster. - Start { - #[command(flatten)] - target: super::container_cluster::Target, - #[arg(long, env = "CROWDB_CONTAINER_IMAGE")] - image: Option, - /// Relative CPU weight per container (1 CPU = 1024 shares), without a hard quota. - #[arg(long)] - node_cpus: Option, - #[arg(long)] - memory_mib: Option, - /// `DiskDB` discovery and heartbeat cadence, recorded for every restart. - #[arg(long, value_parser = clap::value_parser!(u32).range(1..))] - diskdb_heartbeat_interval_secs: Option, - }, - Status { - #[command(flatten)] - target: super::container_cluster::Target, - }, - /// Stop the cluster containers and preserve their mounted data. - Stop { - #[command(flatten)] - target: super::container_cluster::Target, - }, - Restart { - #[command(flatten)] - target: super::container_cluster::Target, - }, - /// Remove only this cluster's containers, network and data. - Delete { - #[command(flatten)] - target: super::container_cluster::Target, - }, -} - #[derive(Subcommand, Debug)] pub enum S3BucketVerb { #[command(alias = "add")] Put { - #[command(flatten)] - target: super::container_cluster::Target, bucket: String, }, #[command(alias = "remove")] Delete { - #[command(flatten)] - target: super::container_cluster::Target, bucket: String, }, - List { - #[command(flatten)] - target: super::container_cluster::Target, - }, + List {}, #[command(alias = "inspect")] Get { - #[command(flatten)] - target: super::container_cluster::Target, bucket: String, }, } @@ -95,8 +50,6 @@ pub enum S3BucketVerb { #[derive(Subcommand, Debug)] pub enum S3ObjectVerb { Put { - #[command(flatten)] - target: super::container_cluster::Target, bucket: String, key: String, /// Read object content from a file. Reads stdin when no source is set. @@ -110,8 +63,6 @@ pub enum S3ObjectVerb { random_size: Option, }, Get { - #[command(flatten)] - target: super::container_cluster::Target, bucket: String, key: String, #[arg(long)] @@ -121,21 +72,15 @@ pub enum S3ObjectVerb { range: Option<(u64, u64)>, }, Delete { - #[command(flatten)] - target: super::container_cluster::Target, bucket: String, key: String, }, #[command(alias = "inspect")] Head { - #[command(flatten)] - target: super::container_cluster::Target, bucket: String, key: String, }, List { - #[command(flatten)] - target: super::container_cluster::Target, bucket: String, #[arg(long)] prefix: Option, @@ -147,10 +92,12 @@ pub enum S3ObjectVerb { } pub async fn run_s3_verb(cli: &Cli, verb: S3Verb) -> ExitCode { - let result = match verb { - S3Verb::Cluster { verb } => return run_cluster(cli, verb).await, - S3Verb::Bucket { verb } => run_bucket(verb).await, - S3Verb::Object { verb } => run_object(verb).await, + let result = match client(cli).await { + Ok(client) => match verb { + S3Verb::Bucket { verb } => run_bucket(&client, verb).await, + S3Verb::Object { verb } => run_object(&client, verb).await, + }, + Err(error) => Err(error), }; match result { Ok(Some(bytes)) => std::io::stdout().write_all(&bytes).map_or_else( @@ -168,108 +115,57 @@ pub async fn run_s3_verb(cli: &Cli, verb: S3Verb) -> ExitCode { } } -async fn run_cluster(_cli: &Cli, verb: S3ClusterVerb) -> ExitCode { - use super::container_cluster; - let result = match verb { - S3ClusterVerb::Start { - target, - image, - node_cpus, - memory_mib, - diskdb_heartbeat_interval_secs, - } => { - container_cluster::start( - &target, - image.as_deref(), - node_cpus, - memory_mib, - diskdb_heartbeat_interval_secs, - ) - .await - } - S3ClusterVerb::Status { target } => container_cluster::status(&target).await, - S3ClusterVerb::Stop { target } => container_cluster::stop(&target).await, - S3ClusterVerb::Restart { target } => container_cluster::restart(&target).await, - S3ClusterVerb::Delete { target } => container_cluster::delete(&target).await, - }; - match result { - Ok(status) => { - println!("Container cluster: {}", status.container); - println!(" S3 endpoint: {}", status.endpoint); - println!(" Web console: {}", status.web); - println!(" Containers: {}/{}", status.running, status.total); - println!(" Image: {}", status.image); - println!(" Root: {}", status.root.display()); - ExitCode::SUCCESS - } - Err(error) => { - eprintln!("error: {error}"); - ExitCode::from(2) - } - } -} - -async fn run_bucket(verb: S3BucketVerb) -> crowdb_console_shared::error::Result>> { - let (target, method, bucket) = match verb { - S3BucketVerb::Put { target, bucket } => (target, Method::PUT, Some(bucket)), - S3BucketVerb::Delete { target, bucket } => (target, Method::DELETE, Some(bucket)), - S3BucketVerb::List { target } => (target, Method::GET, None), - S3BucketVerb::Get { target, bucket } => (target, Method::GET, Some(bucket)), +async fn run_bucket( + client: &S3HttpClient, + verb: S3BucketVerb, +) -> crowdb_console_shared::error::Result>> { + let (method, bucket) = match verb { + S3BucketVerb::Put { bucket } => (Method::PUT, Some(bucket)), + S3BucketVerb::Delete { bucket } => (Method::DELETE, Some(bucket)), + S3BucketVerb::List {} => (Method::GET, None), + S3BucketVerb::Get { bucket } => (Method::GET, Some(bucket)), }; - let root = super::container_cluster::resolve(&target, false).await?; - let body = send_request(&root, method, bucket.as_deref(), None, &[], None, None).await?; + let body = send_request(client, method, bucket.as_deref(), None, &[], None, None).await?; Ok((!body.is_empty()).then_some(body)) } -async fn run_object(verb: S3ObjectVerb) -> crowdb_console_shared::error::Result>> { - use super::container_cluster::resolve; +async fn run_object( + client: &S3HttpClient, + verb: S3ObjectVerb, +) -> crowdb_console_shared::error::Result>> { match verb { S3ObjectVerb::Put { - target, bucket, key, file, text, random_size, - } => { - let root = resolve(&target, false).await?; - put_object(&root, &bucket, &key, file, text, random_size).await - } + } => put_object(client, &bucket, &key, file, text, random_size).await, S3ObjectVerb::Get { - target, bucket, key, output, range, - } => { - let root = resolve(&target, false).await?; - get_object(&root, &bucket, &key, output, range).await - } - S3ObjectVerb::Delete { target, bucket, key } => { - let root = resolve(&target, false).await?; - send_request(&root, Method::DELETE, Some(&bucket), Some(&key), &[], None, None).await?; + } => get_object(client, &bucket, &key, output, range).await, + S3ObjectVerb::Delete { bucket, key } => { + send_request(client, Method::DELETE, Some(&bucket), Some(&key), &[], None, None).await?; Ok(None) } - S3ObjectVerb::Head { target, bucket, key } => { - let root = resolve(&target, false).await?; - send_request(&root, Method::HEAD, Some(&bucket), Some(&key), &[], None, None).await?; + S3ObjectVerb::Head { bucket, key } => { + send_request(client, Method::HEAD, Some(&bucket), Some(&key), &[], None, None).await?; Ok(None) } S3ObjectVerb::List { - target, bucket, prefix, limit, continuation, - } => { - let root = resolve(&target, false).await?; - list_objects(&root, &bucket, prefix, limit, continuation).await - } + } => list_objects(client, &bucket, prefix, limit, continuation).await, } } async fn put_object( - root: &Path, + client: &S3HttpClient, bucket: &str, key: &str, file: Option, @@ -294,7 +190,7 @@ async fn put_object( let mut headers = HeaderMap::new(); headers.insert(CONTENT_TYPE, HeaderValue::from_static(content_type)); send_request_with_headers( - root, + client, Method::PUT, Some(bucket), Some(key), @@ -308,13 +204,13 @@ async fn put_object( } async fn get_object( - root: &Path, + client: &S3HttpClient, bucket: &str, key: &str, output: Option, range: Option<(u64, u64)>, ) -> crowdb_console_shared::error::Result>> { - let body = send_request(root, Method::GET, Some(bucket), Some(key), &[], None, range).await?; + let body = send_request(client, Method::GET, Some(bucket), Some(key), &[], None, range).await?; if let Some(path) = output { std::fs::write(path, body)?; Ok(None) @@ -323,7 +219,7 @@ async fn get_object( } } -fn parse_range(value: &str) -> Result<(u64, u64), String> { +fn parse_range(value: &str) -> std::result::Result<(u64, u64), String> { let (start, end) = value .split_once('-') .ok_or_else(|| "range must use START-END".to_string())?; @@ -338,7 +234,7 @@ fn parse_range(value: &str) -> Result<(u64, u64), String> { } async fn list_objects( - root: &Path, + client: &S3HttpClient, bucket: &str, prefix: Option, limit: Option, @@ -354,12 +250,12 @@ async fn list_objects( if let Some(value) = continuation { query.push(("continuation-token", value)); } - let body = send_request(root, Method::GET, Some(bucket), None, &query, None, None).await?; + let body = send_request(client, Method::GET, Some(bucket), None, &query, None, None).await?; Ok(Some(body)) } async fn send_request( - root: &Path, + client: &S3HttpClient, method: Method, bucket: Option<&str>, object: Option<&str>, @@ -367,12 +263,22 @@ async fn send_request( body: Option>, range: Option<(u64, u64)>, ) -> crowdb_console_shared::error::Result> { - send_request_with_headers(root, method, bucket, object, query, body, range, HeaderMap::new()).await + send_request_with_headers( + client, + method, + bucket, + object, + query, + body, + range, + HeaderMap::new(), + ) + .await } #[allow(clippy::too_many_arguments)] async fn send_request_with_headers( - root: &Path, + client: &S3HttpClient, method: Method, bucket: Option<&str>, object: Option<&str>, @@ -381,10 +287,9 @@ async fn send_request_with_headers( range: Option<(u64, u64)>, headers: HeaderMap, ) -> crowdb_console_shared::error::Result> { - let exchange = crowdb_console_shared::ops::s3::request_exchange_with_headers( - root, method, bucket, object, query, body, range, headers, - ) - .await?; + let exchange = client + .request_exchange_with_headers(method, bucket, object, query, body, range, headers) + .await?; print_exchange(&exchange); let content_type = exchange.response_headers.get(CONTENT_TYPE).cloned(); if !exchange.status.is_success() { @@ -400,145 +305,3 @@ async fn send_request_with_headers( let (_, body) = exchange.into_result()?; Ok(format_structured_body(content_type.as_ref(), body)) } - -fn print_exchange(exchange: &crowdb_console_shared::ops::s3::S3HttpExchange) { - eprintln!( - "\x1b[1;34mHTTP REQUEST\x1b[0m\n> {} {}", - exchange.request.method, exchange.request.url - ); - print_headers('>', &exchange.request.headers); - match exchange.request.body_bytes { - None => eprintln!("> body: none"), - Some(bytes) => { - if let Some(preview) = &exchange.request.body_preview { - eprintln!("\x1b[35m> body: text ({bytes} bytes)\x1b[0m"); - print_body_preview('>', preview, exchange.request.body_preview_truncated); - } else { - eprintln!("\x1b[35m> body: binary ({bytes} bytes, not shown)\x1b[0m"); - } - } - } - - eprintln!("\x1b[1;34mHTTP RESPONSE\x1b[0m\n< {}", exchange.status); - print_headers('<', &exchange.response_headers); - let body = &exchange.response_body; - if body.is_empty() { - eprintln!("\x1b[35m< body: none\x1b[0m"); - } else { - let kind = body_kind(exchange.response_headers.get(CONTENT_TYPE), body); - match kind { - BodyKind::Json => eprintln!( - "\x1b[35m< body: JSON ({} bytes, formatted on stdout)\x1b[0m", - body.len() - ), - BodyKind::Xml => eprintln!( - "\x1b[35m< body: XML ({} bytes, formatted on stdout)\x1b[0m", - body.len() - ), - BodyKind::Text => eprintln!( - "\x1b[35m< body: text ({} bytes, written to stdout)\x1b[0m", - body.len() - ), - BodyKind::Binary => eprintln!( - "\x1b[35m< body: binary ({} bytes, not shown in trace)\x1b[0m", - body.len() - ), - } - } -} - -fn print_body_preview(prefix: char, body: &[u8], truncated: bool) { - let text = String::from_utf8_lossy(body); - for line in text.lines() { - eprintln!("\x1b[35m{prefix} | {line}\x1b[0m"); - } - if truncated { - eprintln!("\x1b[35m{prefix} | ... preview truncated at 65536 bytes\x1b[0m"); - } -} - -fn print_headers(prefix: char, headers: &HeaderMap) { - if headers.is_empty() { - eprintln!("{prefix} headers: none"); - return; - } - let mut values = headers - .iter() - .map(|(name, value)| { - let value = if matches!(name.as_str(), "authorization" | "cookie" | "set-cookie") { - "".to_string() - } else { - value - .to_str() - .map_or_else(|_| "".to_string(), ToOwned::to_owned) - }; - (name.as_str(), value) - }) - .collect::>(); - values.sort_unstable_by(|left, right| left.0.cmp(right.0)); - for (name, value) in values { - eprintln!("{prefix} {name}: {value}"); - } -} - -#[derive(Clone, Copy)] -enum BodyKind { - Json, - Xml, - Text, - Binary, -} - -fn body_kind(content_type: Option<&reqwest::header::HeaderValue>, body: &[u8]) -> BodyKind { - let media_type = content_type - .and_then(|value| value.to_str().ok()) - .unwrap_or_default() - .to_ascii_lowercase(); - if media_type.contains("json") { - return BodyKind::Json; - } - if media_type.contains("xml") { - return BodyKind::Xml; - } - if media_type.starts_with("text/") { - return BodyKind::Text; - } - match body.iter().find(|byte| !byte.is_ascii_whitespace()) { - Some(b'{' | b'[') => BodyKind::Json, - Some(b'<') => BodyKind::Xml, - _ if std::str::from_utf8(body).is_ok() && !media_type.contains("octet-stream") => BodyKind::Text, - _ => BodyKind::Binary, - } -} - -fn format_structured_body(content_type: Option<&reqwest::header::HeaderValue>, body: Vec) -> Vec { - match body_kind(content_type, &body) { - BodyKind::Json => serde_json::from_slice::(&body) - .ok() - .and_then(|value| serde_json::to_vec_pretty(&value).ok()) - .map_or(body, with_trailing_newline), - BodyKind::Xml => format_xml(&body).map_or(body, with_trailing_newline), - BodyKind::Text | BodyKind::Binary => body, - } -} - -fn format_xml(body: &[u8]) -> Option> { - let mut reader = Reader::from_reader(body); - let mut writer = Writer::new_with_indent(Vec::new(), b' ', 2); - let mut buffer = Vec::new(); - loop { - match reader.read_event_into(&mut buffer).ok()? { - quick_xml::events::Event::Eof => break, - event => writer.write_event(event.into_owned()).ok()?, - } - buffer.clear(); - } - Some(writer.into_inner()) -} - -fn with_trailing_newline(mut body: Vec) -> Vec { - if !body.ends_with(b"\n") { - body.push(b'\n'); - } - body -} diff --git a/app/crowdb-cli/src/commands/s3/output.rs b/app/crowdb-cli/src/commands/s3/output.rs new file mode 100644 index 000000000..c59cd8074 --- /dev/null +++ b/app/crowdb-cli/src/commands/s3/output.rs @@ -0,0 +1,155 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Credential-redacted HTTP diagnostics and structured response formatting. + +use quick_xml::{Reader, Writer}; +use reqwest::header::{HeaderMap, CONTENT_TYPE}; + +pub(super) fn print_exchange(exchange: &crowdb_console_shared::ops::s3::S3HttpExchange) { + eprintln!( + "\x1b[1;34mHTTP REQUEST\x1b[0m\n> {} {}", + exchange.request.method, exchange.request.url + ); + print_headers('>', &exchange.request.headers); + match exchange.request.body_bytes { + None => eprintln!("> body: none"), + Some(bytes) => { + if let Some(preview) = &exchange.request.body_preview { + eprintln!("\x1b[35m> body: text ({bytes} bytes)\x1b[0m"); + print_body_preview('>', preview, exchange.request.body_preview_truncated); + } else { + eprintln!("\x1b[35m> body: binary ({bytes} bytes, not shown)\x1b[0m"); + } + } + } + + eprintln!("\x1b[1;34mHTTP RESPONSE\x1b[0m\n< {}", exchange.status); + print_headers('<', &exchange.response_headers); + let body = &exchange.response_body; + if body.is_empty() { + eprintln!("\x1b[35m< body: none\x1b[0m"); + } else { + let kind = body_kind(exchange.response_headers.get(CONTENT_TYPE), body); + match kind { + BodyKind::Json => eprintln!( + "\x1b[35m< body: JSON ({} bytes, formatted on stdout)\x1b[0m", + body.len() + ), + BodyKind::Xml => eprintln!( + "\x1b[35m< body: XML ({} bytes, formatted on stdout)\x1b[0m", + body.len() + ), + BodyKind::Text => eprintln!( + "\x1b[35m< body: text ({} bytes, written to stdout)\x1b[0m", + body.len() + ), + BodyKind::Binary => eprintln!( + "\x1b[35m< body: binary ({} bytes, not shown in trace)\x1b[0m", + body.len() + ), + } + } +} + +pub(super) fn print_body_preview(prefix: char, body: &[u8], truncated: bool) { + let text = String::from_utf8_lossy(body); + for line in text.lines() { + eprintln!("\x1b[35m{prefix} | {line}\x1b[0m"); + } + if truncated { + eprintln!("\x1b[35m{prefix} | ... preview truncated at 65536 bytes\x1b[0m"); + } +} + +fn print_headers(prefix: char, headers: &HeaderMap) { + if headers.is_empty() { + eprintln!("{prefix} headers: none"); + return; + } + let mut values = headers + .iter() + .map(|(name, value)| { + let value = if matches!( + name.as_str(), + "authorization" | "cookie" | "set-cookie" | "x-amz-security-token" + ) { + "".to_string() + } else { + value + .to_str() + .map_or_else(|_| "".to_string(), ToOwned::to_owned) + }; + (name.as_str(), value) + }) + .collect::>(); + values.sort_unstable_by(|left, right| left.0.cmp(right.0)); + for (name, value) in values { + eprintln!("{prefix} {name}: {value}"); + } +} + +#[derive(Clone, Copy)] +enum BodyKind { + Json, + Xml, + Text, + Binary, +} + +fn body_kind(content_type: Option<&reqwest::header::HeaderValue>, body: &[u8]) -> BodyKind { + let media_type = content_type + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_ascii_lowercase(); + if media_type.contains("json") { + return BodyKind::Json; + } + if media_type.contains("xml") { + return BodyKind::Xml; + } + if media_type.starts_with("text/") { + return BodyKind::Text; + } + match body.iter().find(|byte| !byte.is_ascii_whitespace()) { + Some(b'{' | b'[') => BodyKind::Json, + Some(b'<') => BodyKind::Xml, + _ if std::str::from_utf8(body).is_ok() && !media_type.contains("octet-stream") => BodyKind::Text, + _ => BodyKind::Binary, + } +} + +pub(super) fn format_structured_body( + content_type: Option<&reqwest::header::HeaderValue>, + body: Vec, +) -> Vec { + match body_kind(content_type, &body) { + BodyKind::Json => serde_json::from_slice::(&body) + .ok() + .and_then(|value| serde_json::to_vec_pretty(&value).ok()) + .map_or(body, with_trailing_newline), + BodyKind::Xml => format_xml(&body).map_or(body, with_trailing_newline), + BodyKind::Text | BodyKind::Binary => body, + } +} + +fn format_xml(body: &[u8]) -> Option> { + let mut reader = Reader::from_reader(body); + let mut writer = Writer::new_with_indent(Vec::new(), b' ', 2); + let mut buffer = Vec::new(); + loop { + match reader.read_event_into(&mut buffer).ok()? { + quick_xml::events::Event::Eof => break, + event => writer.write_event(event.into_owned()).ok()?, + } + buffer.clear(); + } + Some(writer.into_inner()) +} + +fn with_trailing_newline(mut body: Vec) -> Vec { + if !body.ends_with(b"\n") { + body.push(b'\n'); + } + body +} diff --git a/app/crowdb-cli/src/commands/s3/target.rs b/app/crowdb-cli/src/commands/s3/target.rs new file mode 100644 index 000000000..c7b294e80 --- /dev/null +++ b/app/crowdb-cli/src/commands/s3/target.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Resolve selected-cluster credentials before any S3 workload request. + +use crate::Cli; +use crowdb_console_shared::{ + error::{Error, Result}, + ops::s3::{S3Credentials, S3HttpClient}, +}; +use serde::Deserialize; +use std::time::Duration; + +#[derive(Deserialize)] +struct AccessCredentials { + s3: ClientCredentials, +} + +#[derive(Deserialize)] +struct ClientCredentials { + endpoint: String, + access_key_id: String, + secret_access_key: String, + region: String, +} + +pub(crate) async fn client(cli: &Cli) -> Result { + let target = cli + .target + .as_ref() + .ok_or_else(|| invalid("select --cluster or --management-endpoint before an S3 operation"))?; + let origin = target + .management_endpoints + .first() + .ok_or_else(|| invalid("selected cluster has no management endpoint"))?; + let http = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(3)) + .build() + .map_err(|_| invalid("cannot construct credential client"))?; + let credentials = tokio::time::timeout(Duration::from_secs(3), async { + let mut reply = http + .get(format!("{origin}/api/access/credentials")) + .send() + .await + .map_err(|_| invalid("selected cluster credentials are unavailable"))? + .error_for_status() + .map_err(|_| invalid("selected cluster rejected credential access"))?; + let mut bytes = Vec::new(); + while let Some(chunk) = reply + .chunk() + .await + .map_err(|_| invalid("cannot read selected cluster credentials"))? + { + if bytes.len() + chunk.len() > 32768 { + return Err(invalid("selected credentials exceed their size bound")); + } + bytes.extend_from_slice(&chunk); + } + serde_json::from_slice::(&bytes) + .map_err(|_| invalid("selected cluster S3 endpoint or credentials are incomplete")) + }) + .await + .map_err(|_| invalid("selected cluster credential observation timed out"))?? + .s3; + S3HttpClient::signed( + credentials.endpoint, + S3Credentials { + access: credentials.access_key_id, + secret: credentials.secret_access_key, + region: credentials.region, + session: None, + }, + ) +} + +fn invalid(message: &str) -> Error { + Error::Config(message.into()) +} diff --git a/app/crowdb-cli/src/main.rs b/app/crowdb-cli/src/main.rs index aa3c066ee..7e777130d 100644 --- a/app/crowdb-cli/src/main.rs +++ b/app/crowdb-cli/src/main.rs @@ -1,13 +1,11 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! `crowdb-cli` CLI entrypoint (R126 restructure). +//! `crowdb-cli` CLI entrypoint. //! -//! Four top-level domains: `cluster`, `kv`, `chunk`, `bench`. The CLI -//! talks directly to the system group via `CrowdbSysmdClient` -//! and to individual `crowdb-kv-server` management APIs — no -//! `crowdb-web` intermediary. The connection target can be any system-group -//! node; leader discovery is automatic. +//! Cluster operations, logical KV operations and attached workloads. The CLI +//! resolves a confirmed application target, then uses Group 0 and native +//! service clients for metadata, data and attached workloads. macro_rules! eprintln { () => { @@ -33,35 +31,26 @@ use std::path::PathBuf; use std::process::ExitCode; use clap::{Parser, Subcommand}; -use crowdb_protocol::KV_SERVER_MGMT_BASE; use commands::{ - run_bench_verb, run_chunk_diskdb_verb, run_chunk_stub_verb, run_cluster_verb, run_group_verb, - run_kv_data_verb, run_kv_server_verb, run_port_alloc, run_replica_verb, run_s3_verb, run_store_verb, - BenchVerb, ChunkDiskdbVerb, ChunkStubVerb, ClusterVerb, GroupVerb, KvDataVerb, KvServerVerb, - PortAllocArgs, ReplicaVerb, S3Verb, StoreVerb, + run_bench_verb, run_cluster_verb, run_group_verb, run_kv_data_verb, run_port_alloc, run_replica_verb, + run_s3_verb, run_store_verb, BenchVerb, ClusterVerb, GroupVerb, KvDataVerb, PortAllocArgs, ReplicaVerb, + S3Verb, StoreVerb, }; #[derive(Parser, Debug)] #[command(name = "crowdb-cli", version, about = "CrowDB cluster console (CLI)")] struct Cli { - /// Versioned bare-metal launch registry; process identity is stored separately. - #[arg(long, global = true, value_name = "PATH")] - registry: Option, + /// Select a saved application cluster binding. + #[arg(long, global = true, env = "CROWDB_CLUSTER")] + cluster: Option, - /// IP address of any system-group node; leader discovery is automatic. - #[arg( - long, - global = true, - alias = "sysmd-ip", - env = "CROWDB_SYSTEM_IP", - default_value = "127.0.0.1" - )] - system_ip: String, + /// Console management origin; repeat to verify multiple cluster members. + #[arg(long = "management-endpoint", global = true)] + management_endpoints: Vec, - /// Management port of any system-group node; leader discovery is automatic. - #[arg(long, global = true, alias = "sysmd-port", env = "CROWDB_SYSTEM_PORT", default_value_t = KV_SERVER_MGMT_BASE, value_parser = clap::value_parser!(u16).range(1..))] - system_port: u16, + #[arg(skip)] + target: Option, /// Latest-run benchmark log directory, computed in `main()` after parse. /// This is not a CLI flag and remains empty for non-benchmark commands. @@ -78,7 +67,6 @@ impl Cli { match &self.command { Domain::Bench { verb } => Some(match verb { BenchVerb::Kv(_) => "bench-kv", - BenchVerb::Rpc(_) => "bench-rpc", BenchVerb::Diskdb(_) => "bench-diskdb", BenchVerb::Chunkdb(_) => "bench-chunkdb", BenchVerb::Chunkio(_) => "bench-chunkio", @@ -91,31 +79,23 @@ impl Cli { #[derive(Subcommand, Debug)] enum Domain { - /// Start, restart, stop, or inspect services from the launch registry. - #[command(subcommand)] - Launch(commands::launch::LaunchVerb), /// Hardware topology + cluster-level ops. #[command(alias = "cls")] Cluster { #[command(subcommand)] verb: ClusterVerb, }, - /// KV layer: server lifecycle + logical concepts + data-plane. + /// Logical KV topology and data operations. Kv { #[command(subcommand)] verb: KvVerb, }, - /// Chunk storage service cluster. - Chunk { - #[command(subcommand)] - verb: ChunkVerb, - }, /// Load injection only. Bench { #[command(subcommand)] verb: BenchVerb, }, - /// Persistent S3 mini-cluster and bucket/object operations. + /// Signed bucket and object operations on the selected cluster. S3 { #[command(subcommand)] verb: S3Verb, @@ -131,8 +111,6 @@ enum Domain { #[derive(Subcommand, Debug)] enum KvVerb { - #[command(subcommand)] - Server(KvServerVerb), #[command(subcommand)] Store(StoreVerb), #[command(subcommand)] @@ -143,14 +121,6 @@ enum KvVerb { Data(KvDataVerb), } -#[derive(Subcommand, Debug)] -enum ChunkVerb { - #[command(subcommand)] - Diskdb(ChunkDiskdbVerb), - #[command(subcommand)] - Stub(ChunkStubVerb), -} - fn main() -> ExitCode { let mut cli = Cli::parse(); print_command(); @@ -253,26 +223,27 @@ fn shell_quote(value: &str) -> String { } async fn dispatch(mut cli: Cli) -> ExitCode { + cli.target = match commands::cluster::target::resolve(&cli).await { + Ok(target) => target, + Err(code) => return code, + }; let command = std::mem::replace( &mut cli.command, Domain::Cluster { - verb: ClusterVerb::Status, + verb: ClusterVerb::Status { + json: false, + timeout: 10, + }, }, ); match command { - Domain::Launch(verb) => commands::launch::run(&cli, verb).await, Domain::Cluster { verb } => run_cluster_verb(&cli, verb).await, Domain::Kv { verb } => match verb { - KvVerb::Server(sv) => run_kv_server_verb(&cli, sv).await, KvVerb::Store(sv) => run_store_verb(&cli, sv).await, KvVerb::Group(gv) => run_group_verb(&cli, gv).await, KvVerb::Replica(rv) => run_replica_verb(&cli, rv).await, KvVerb::Data(dv) => run_kv_data_verb(&cli, dv).await, }, - Domain::Chunk { verb } => match verb { - ChunkVerb::Diskdb(dv) => run_chunk_diskdb_verb(&cli, dv).await, - ChunkVerb::Stub(sv) => run_chunk_stub_verb(&cli, sv).await, - }, Domain::Bench { verb } => run_bench_verb(&cli, verb).await, Domain::S3 { verb } => run_s3_verb(&cli, verb).await, Domain::PortAlloc { args } => run_port_alloc(&args), diff --git a/app/crowdb-cli/tests/chunkdb_bench_test.rs b/app/crowdb-cli/tests/chunkdb_bench_test.rs index aa74214c3..237ccab8d 100644 --- a/app/crowdb-cli/tests/chunkdb_bench_test.rs +++ b/app/crowdb-cli/tests/chunkdb_bench_test.rs @@ -27,19 +27,6 @@ fn allocate_and_mix_expose_placement_and_concurrency_controls() { } } -#[test] -fn combined_deploy_exposes_chunkdb_controls() { - let output = Command::new(crowdb_cli_bin()) - .args(["cluster", "local-deploy", "--help"]) - .output() - .expect("run combined deploy help"); - assert!(output.status.success()); - let stdout = String::from_utf8(output.stdout).expect("help is UTF-8"); - assert!(stdout.contains("combined")); - assert!(stdout.contains("--chunkdb-instances")); - assert!(stdout.contains("--allow-unsafe-ec")); -} - #[test] fn chunkio_write_exposes_large_write_controls() { let output = Command::new(crowdb_cli_bin()) diff --git a/app/crowdb-cli/tests/cluster_cli_test.rs b/app/crowdb-cli/tests/cluster_cli_test.rs index 3eb27ca55..e0cac2801 100644 --- a/app/crowdb-cli/tests/cluster_cli_test.rs +++ b/app/crowdb-cli/tests/cluster_cli_test.rs @@ -23,7 +23,7 @@ fn non_benchmark_command_does_not_create_log_directory() { .current_dir(&root) .env("CROWDB_RUNTIME_ROOT", root.join(".crowdb-runtime")) .env("CROWDB_CLI_STATE", config) - .args(["cluster", "local-deploy", "--service-type", "invalid"]) + .args(["cluster", "status", "--timeout", "1"]) .output() .expect("run crowdb-cli"); assert!(!output.status.success()); @@ -38,25 +38,17 @@ fn benchmark_replaces_its_previous_log_directory() { eprintln!("skipping: crowdb-cli binary not built ({})", cli.display()); return; } - for mode in ["coroutine", "tokio"] { - let root = tempdir(&format!("bench-latest-log-{mode}")); - let log_dir = root.join(".crowdb-runtime/artifacts/cli/bench-rpc"); + for iteration in 0..2 { + let root = tempdir(&format!("bench-latest-log-{iteration}")); + let log_dir = root.join(".crowdb-runtime/artifacts/cli/bench-s3"); fs::create_dir_all(&log_dir).expect("create old benchmark log dir"); fs::write(log_dir.join("old-run.marker"), b"old").expect("write old marker"); let output = Command::new(&cli) .current_dir(&root) .env("CROWDB_RUNTIME_ROOT", root.join(".crowdb-runtime")) - .args([ - "bench", - "rpc", - "--server-port", - "1", - "--metrics-interval", - "0", - "--mode", - mode, - ]) + .env_remove("CROWDB_CLUSTER") + .args(["bench", "s3", "write", "--operations", "1"]) .output() .expect("run benchmark against closed port"); assert!(!output.status.success()); @@ -67,3 +59,20 @@ fn benchmark_replaces_its_previous_log_directory() { let _ = fs::remove_dir_all(root); } } + +#[test] +fn retired_storage_process_commands_are_absent_and_measurements_remain() { + let cli = crowdb_cli_bin(); + for arguments in [vec!["chunk", "--help"], vec!["bench", "rpc", "--help"]] { + let output = Command::new(&cli).args(&arguments).output().unwrap(); + assert_eq!(output.status.code(), Some(2), "{arguments:?}: {output:?}"); + assert!(String::from_utf8_lossy(&output.stderr).contains("unrecognized subcommand")); + } + for family in ["kv", "diskdb", "chunkdb", "chunkio", "s3"] { + let output = Command::new(&cli) + .args(["bench", family, "--help"]) + .output() + .unwrap(); + assert!(output.status.success(), "{family}: {output:?}"); + } +} diff --git a/app/crowdb-cli/tests/cluster_lifecycle_test.rs b/app/crowdb-cli/tests/cluster_lifecycle_test.rs new file mode 100644 index 000000000..27a966722 --- /dev/null +++ b/app/crowdb-cli/tests/cluster_lifecycle_test.rs @@ -0,0 +1,326 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{routing::get, Json, Router}; +use crowdb_protocol::mgmt::{ + cluster::{ClusterObservation, DependencyKind, DependencyObservation, DependencyState}, + node::deployment::NodeDeploymentPolicy, +}; +use serde_json::{json, Value}; +use std::{ + fs, + os::unix::fs::PermissionsExt, + path::{Path, PathBuf}, + process::{Command, Output}, + time::{SystemTime, UNIX_EPOCH}, +}; + +struct TestDocker { + root: crowdb_test_harness::test_dirs::TestDir, + program: PathBuf, +} + +impl TestDocker { + fn new() -> Self { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("cli-lifecycle"); + let program = root.path().join("docker-fixture"); + fs::write(&program, include_str!("common/local_docker.py")).unwrap(); + fs::set_permissions(&program, fs::Permissions::from_mode(0o700)).unwrap(); + let test = Self { root, program }; + test.save(&json!({"runtime":{"daemon":"test-daemon", "context":"default", "endpoint":"unix:///test-docker.sock"}, + "calls":[], "container":{}, "volume":{}, "network":{}})); + test + } + fn save(&self, state: &Value) { + fs::write( + self.root.path().join("docker.json"), + serde_json::to_vec(state).unwrap(), + ) + .unwrap(); + } + fn state(&self) -> Value { + serde_json::from_slice(&fs::read(self.root.path().join("docker.json")).unwrap()).unwrap() + } + fn run(&self, cluster: Option<&str>, args: &[&str], code: i32) -> Output { + let mut command = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")); + command + .env("CROWDB_RUNTIME_ROOT", self.root.path()) + .env("CROWDB_CONTAINER_RUNTIME", &self.program) + .env_remove("DOCKER_HOST") + .env_remove("CROWDB_CLUSTER") + .env_remove("CROWDB_SYSTEM_IP") + .env_remove("CROWDB_SYSTEM_PORT"); + if let Some(cluster) = cluster { + command.args(["--cluster", cluster]); + } + let output = command.arg("cluster").args(args).output().unwrap(); + assert_eq!( + output.status.code(), + Some(code), + "stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + output + } + fn receipt(&self, name: &str) -> PathBuf { + self.root + .path() + .join(format!("persistent/oci-clusters/{name}.json")) + } +} + +fn report(cluster: &str, nodes: u8, ready: bool) -> Value { + let now = u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()).unwrap(); + let policy = NodeDeploymentPolicy { + cluster: cluster.into(), + initial_voters: nodes, + ..NodeDeploymentPolicy::default() + }; + let observation = ClusterObservation { + version: 1, + cluster: cluster.into(), + cluster_id: Some(uuid::Uuid::from_u128(1).to_string()), + bound: true, + initial_voters: nodes, + policy: Some(policy), + observed_at_unix_ms: now, + members: vec![], + operation_id: None, + work_accepted: true, + setup_completed: true, + runtime: None, + dependencies: [ + DependencyKind::Authority, + DependencyKind::Metadata, + DependencyKind::Storage, + DependencyKind::Member, + DependencyKind::Service, + DependencyKind::Protocol, + ] + .into_iter() + .map(|kind| DependencyObservation { + id: format!("{kind:?}"), + kind, + required: true, + state: if ready { + DependencyState::Ready + } else { + DependencyState::Unknown + }, + observed_at_unix_ms: now, + max_age_ms: 60_000, + revision: None, + operation_id: None, + current_operation_id: None, + reason: None, + detail: "current fixture evidence".into(), + }) + .collect(), + }; + serde_json::to_value(crowdb_console_shared::ops::cluster::readiness::reduce( + observation, + now, + )) + .unwrap() +} + +async fn serve(cluster: &str, nodes: u8, ready: bool) -> (u16, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let value = report(cluster, nodes, ready); + let app = Router::new().route( + "/api/cluster/status", + get(move || { + let value = value.clone(); + async move { Json(value) } + }), + ); + ( + port, + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }), + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn default_single_node_start_seals_identity_and_resumes_without_duplicate_resources() { + let test = TestDocker::new(); + let (port, server) = serve("local", 1, true).await; + test.run( + None, + &["start", "--port-base", &port.to_string(), "--wait", "0", "--json"], + 0, + ); + let original = fs::read(test.receipt("local")).unwrap(); + let binding = test.root.path().join("persistent/clusters/local/target.json"); + let target: Value = serde_json::from_slice(&fs::read(binding).unwrap()).unwrap(); + assert_eq!(target["cluster_id"], uuid::Uuid::from_u128(1).to_string()); + assert_eq!(test.state()["container"].as_object().unwrap().len(), 1); + test.run(None, &["stop"], 0); + assert_eq!( + test.state()["container"]["crowdb-local-1"]["State"]["Running"], + false + ); + test.run(None, &["start", "--wait", "0"], 0); + assert_eq!( + test.state()["container"]["crowdb-local-1"]["State"]["Running"], + true + ); + assert_eq!(fs::read(test.receipt("local")).unwrap(), original); + assert_eq!( + test.state()["calls"] + .as_array() + .unwrap() + .iter() + .filter(|args| args[0] == "run") + .count(), + 1 + ); + server.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn three_node_partial_readiness_keeps_resumable_inputs_and_stop_is_scoped() { + let test = TestDocker::new(); + let (port, server) = serve("three", 3, false).await; + let mut partial = test.state(); + partial["fail_run_once"] = json!("crowdb-three-2"); + test.save(&partial); + test.run( + Some("three"), + &[ + "start", + "--nodes", + "3", + "--port-base", + &port.to_string(), + "--wait", + "0", + ], + 2, + ); + assert_eq!(test.state()["container"].as_object().unwrap().len(), 1); + let receipt = fs::read(test.receipt("three")).unwrap(); + let calls = test.state()["calls"].clone(); + test.run(Some("three"), &["status", "--json"], 0); + assert_eq!(test.state()["calls"], calls); + test.run( + Some("three"), + &[ + "start", + "--nodes", + "3", + "--port-base", + &port.to_string(), + "--wait", + "0", + "--json", + ], + 1, + ); + let mut state = test.state(); + assert_eq!(state["container"].as_object().unwrap().len(), 3); + let mut foreign = state["container"]["crowdb-three-1"].clone(); + foreign["Name"] = json!("/crowdb-other-1"); + foreign["Id"] = json!("crowdb-other-1"); + state["container"]["crowdb-other-1"] = foreign; + test.save(&state); + test.run(Some("three"), &["stop"], 0); + let state = test.state(); + for index in 1..=3 { + assert_eq!( + state["container"][format!("crowdb-three-{index}")]["State"]["Running"], + false + ); + } + assert_eq!(state["container"]["crowdb-other-1"]["State"]["Running"], true); + test.run(Some("three"), &["start", "--wait", "0"], 1); + assert_eq!(fs::read(test.receipt("three")).unwrap(), receipt); + assert_eq!( + test.state()["calls"] + .as_array() + .unwrap() + .iter() + .filter(|args| args[0] == "run") + .count(), + 4 + ); + server.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn changed_daemon_or_inputs_fail_without_effects_and_preserve_receipt() { + let test = TestDocker::new(); + let (port, server) = serve("local", 1, true).await; + test.run( + None, + &["start", "--port-base", &port.to_string(), "--wait", "0"], + 0, + ); + let original = fs::read(test.receipt("local")).unwrap(); + let before = test.state()["container"].clone(); + test.run(None, &["start", "--nodes", "3", "--wait", "0"], 2); + let mut state = test.state(); + state["calls"] = json!([]); + state["runtime"]["daemon"] = json!("foreign-daemon"); + test.save(&state); + test.run(None, &["stop"], 2); + test.run(None, &["start", "--wait", "0"], 2); + assert_eq!(test.state()["container"], before); + assert_eq!(fs::read(test.receipt("local")).unwrap(), original); + assert!(test.state()["calls"] + .as_array() + .unwrap() + .iter() + .all(|args| args[0] != "stop" && args[0] != "run" && args[0] != "start")); + server.abort(); +} + +#[test] +fn invalid_topology_and_symlinked_receipts_fail_before_docker() { + let test = TestDocker::new(); + test.run(None, &["start", "--nodes", "2"], 2); + let directory = test.root.path().join("persistent/oci-clusters"); + fs::create_dir_all(&directory).unwrap(); + std::os::unix::fs::symlink(Path::new("/nonexistent"), test.receipt("local")).unwrap(); + test.run(None, &["start", "--wait", "0"], 2); + assert_eq!(test.state()["calls"], json!([])); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn foreign_network_and_changed_container_configuration_are_never_adopted() { + let test = TestDocker::new(); + let (port, server) = serve("local", 1, true).await; + test.run( + None, + &["start", "--port-base", &port.to_string(), "--wait", "0"], + 0, + ); + test.run(None, &["stop"], 0); + let original = test.state(); + for foreign_network in [true, false] { + let mut state = original.clone(); + state["calls"] = json!([]); + if foreign_network { + state["network"]["crowdb-local"]["Labels"]["org.crowdb.cluster.owner"] = json!("foreign"); + } else { + state["container"]["crowdb-local-1"]["HostConfig"]["Memory"] = json!(512); + } + test.save(&state); + test.run(None, &["start", "--wait", "0"], 2); + assert_eq!( + test.state()["container"]["crowdb-local-1"]["State"]["Running"], + false + ); + assert!(test.state()["calls"] + .as_array() + .unwrap() + .iter() + .all(|args| args[0] != "start" + && args[0] != "run" + && args.get(1).and_then(Value::as_str) != Some("create"))); + } + server.abort(); +} diff --git a/app/crowdb-cli/tests/cluster_status_test.rs b/app/crowdb-cli/tests/cluster_status_test.rs new file mode 100644 index 000000000..f7ca0996f --- /dev/null +++ b/app/crowdb-cli/tests/cluster_status_test.rs @@ -0,0 +1,257 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{http::StatusCode, response::IntoResponse, routing::get, Json, Router}; +use crowdb_protocol::mgmt::{ + cluster::{ClusterObservation, ClusterStatus, DependencyKind, DependencyObservation, DependencyState}, + node::deployment::NodeDeploymentPolicy, +}; +use std::{ + process::{Command, Output}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::{SystemTime, UNIX_EPOCH}, +}; + +struct TestStatusServer { + origin: String, + task: tokio::task::JoinHandle<()>, +} +impl Drop for TestStatusServer { + fn drop(&mut self) { + self.task.abort(); + } +} +impl TestStatusServer { + async fn start(cluster_id: String, ready_after: usize, invalid: bool) -> Self { + let identities = if ready_after > 0 && ready_after != usize::MAX { + vec![None, Some(cluster_id)] + } else { + vec![Some(cluster_id)] + }; + Self::with_failures(identities, ready_after, invalid, None, false).await + } + async fn with_failures( + identities: Vec>, + ready_after: usize, + invalid: bool, + unavailable_after: Option, + stalled: bool, + ) -> Self { + Self::with_clock(identities, ready_after, invalid, unavailable_after, stalled, 0).await + } + async fn with_clock( + identities: Vec>, + ready_after: usize, + invalid: bool, + unavailable_after: Option, + stalled: bool, + clock_offset_ms: u64, + ) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let seen = Arc::new(AtomicUsize::new(0)); + let router = Router::new().route( + "/api/cluster/status", + get(move || { + let seen = Arc::clone(&seen); + let identities = identities.clone(); + async move { + let now = + u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()) + .unwrap(); + let count = seen.fetch_add(1, Ordering::Relaxed); + if unavailable_after.is_some_and(|limit| count >= limit) { + return StatusCode::SERVICE_UNAVAILABLE.into_response(); + } + if stalled { + tokio::time::sleep(std::time::Duration::from_secs(60)).await; + } + let ready = count >= ready_after; + let cluster_id = identities[count.min(identities.len() - 1)].clone(); + let bound = cluster_id.is_some(); + let observation = ClusterObservation { + version: if invalid { 2 } else { 1 }, + cluster: NodeDeploymentPolicy::default().cluster, + cluster_id, + bound, + initial_voters: 1, + policy: Some(NodeDeploymentPolicy::default()), + observed_at_unix_ms: now + clock_offset_ms, + members: Vec::new(), + operation_id: None, + work_accepted: true, + setup_completed: true, + runtime: None, + dependencies: [ + DependencyKind::Authority, + DependencyKind::Metadata, + DependencyKind::Storage, + DependencyKind::Member, + DependencyKind::Service, + DependencyKind::Protocol, + ] + .into_iter() + .map(|kind| DependencyObservation { + id: format!("{kind:?}"), + kind, + detail: "Current test evidence".into(), + required: true, + state: if ready || kind == DependencyKind::Authority { + DependencyState::Ready + } else { + DependencyState::Unknown + }, + observed_at_unix_ms: now + clock_offset_ms, + max_age_ms: 3000, + revision: None, + operation_id: None, + current_operation_id: None, + reason: None, + }) + .collect(), + }; + Json(crowdb_console_shared::ops::cluster::readiness::reduce( + observation, + now, + )) + .into_response() + } + }), + ); + let task = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + Self { origin, task } + } +} +fn run(origins: &[&str], args: &[&str]) -> Output { + let mut command = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")); + command + .env_remove("CROWDB_CLUSTER") + .env_remove("CROWDB_SYSTEM_IP") + .env_remove("CROWDB_SYSTEM_PORT") + .env("DOCKER_HOST", "unix:///nonexistent-crowdb-test.sock"); + for origin in origins { + command.args(["--management-endpoint", origin]); + } + command.args(["cluster"]).args(args).output().unwrap() +} +fn report(output: &Output, code: i32) -> ClusterStatus { + assert_eq!( + output.status.code(), + Some(code), + "stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).unwrap() +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn status_returns_valid_unhealthy_json_and_check_returns_one_without_docker() { + let server = TestStatusServer::start(uuid::Uuid::new_v4().to_string(), usize::MAX, false).await; + let status = report(&run(&[&server.origin], &["status", "--json"]), 0); + assert_ne!(status.phase, crowdb_protocol::mgmt::cluster::ClusterPhase::Ready); + report(&run(&[&server.origin], &["check", "--json", "--wait", "1"]), 1); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn check_waits_for_current_application_evidence_then_returns_zero() { + let server = TestStatusServer::start(uuid::Uuid::new_v4().to_string(), 2, false).await; + let status = report(&run(&[&server.origin], &["check", "--json", "--wait", "2"]), 0); + assert_eq!(status.phase, crowdb_protocol::mgmt::cluster::ClusterPhase::Ready); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn invalid_reports_and_mixed_cluster_origins_return_two_and_no_json() { + let invalid = TestStatusServer::start(uuid::Uuid::new_v4().to_string(), 0, true).await; + let output = run(&[&invalid.origin], &["status", "--json"]); + assert_eq!(output.status.code(), Some(2)); + assert!(output.stdout.is_empty()); + let left = TestStatusServer::start(uuid::Uuid::new_v4().to_string(), 0, false).await; + let right = TestStatusServer::start(uuid::Uuid::new_v4().to_string(), 0, false).await; + let output = run(&[&left.origin, &right.origin], &["check", "--json"]); + assert_eq!(output.status.code(), Some(2)); + assert!(output.stdout.is_empty()); + assert!(String::from_utf8_lossy(&output.stderr).contains("disagree on identity")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_lost_endpoint_after_a_valid_observation_remains_an_unmet_readiness_report() { + let server = TestStatusServer::with_failures( + vec![Some(uuid::Uuid::new_v4().to_string())], + usize::MAX, + false, + Some(1), + false, + ) + .await; + let output = run(&[&server.origin], &["check", "--json", "--wait", "1"]); + let report = report(&output, 1); + assert_eq!( + report.phase, + crowdb_protocol::mgmt::cluster::ClusterPhase::Unavailable + ); + assert!(report + .reasons + .iter() + .any(|reason| reason.code == "management-origin-unavailable")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn slow_selected_origins_share_the_caller_deadline_and_never_multiply_it() { + let left = + TestStatusServer::with_failures(vec![Some(uuid::Uuid::new_v4().to_string())], 0, false, None, true) + .await; + let right = + TestStatusServer::with_failures(vec![Some(uuid::Uuid::new_v4().to_string())], 0, false, None, true) + .await; + let started = std::time::Instant::now(); + let output = run( + &[&left.origin, &right.origin], + &["check", "--json", "--wait", "1", "--timeout", "20"], + ); + assert_eq!(output.status.code(), Some(2)); + assert!(output.stdout.is_empty()); + assert!(started.elapsed() < std::time::Duration::from_secs(3)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn accepted_identity_cannot_change_after_an_unbound_observation_gap() { + let server = TestStatusServer::with_failures( + vec![ + Some(uuid::Uuid::new_v4().to_string()), + None, + Some(uuid::Uuid::new_v4().to_string()), + ], + usize::MAX, + false, + None, + false, + ) + .await; + let output = run(&[&server.origin], &["check", "--json", "--wait", "2"]); + assert_eq!(output.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&output.stderr).contains("disagree on identity")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn one_http_observation_can_wait_for_a_small_clock_offset_without_accepting_far_future_data() { + for (offset, code) in [(80, 0), (10_000, 1)] { + let server = TestStatusServer::with_clock( + vec![Some(uuid::Uuid::new_v4().to_string())], + 0, + false, + None, + false, + offset, + ) + .await; + let output = run(&[&server.origin], &["check", "--json", "--wait", "0"]); + let report = report(&output, code); + assert_eq!( + report.phase == crowdb_protocol::mgmt::cluster::ClusterPhase::Ready, + code == 0 + ); + } +} diff --git a/app/crowdb-cli/tests/cluster_target_test.rs b/app/crowdb-cli/tests/cluster_target_test.rs new file mode 100644 index 000000000..049d2f0db --- /dev/null +++ b/app/crowdb-cli/tests/cluster_target_test.rs @@ -0,0 +1,192 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/cluster_target.rs"] +mod common; + +#[path = "common/target_commands.rs"] +mod operations; + +use common::TestTargetServer; +use crowdb_console_shared::deployment::target::ClusterTargetBinding; +use operations::{test_operations, test_rejected_operation}; +use std::{os::unix::fs::PermissionsExt, path::Path, process::Command}; + +fn run(root: &Path, selected: &[&str]) -> String { + let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) + .env("CROWDB_RUNTIME_ROOT", root) + .env_remove("CROWDB_CLUSTER") + .env_remove("CROWDB_SYSTEM_IP") + .env_remove("CROWDB_SYSTEM_PORT") + .args(selected) + .args([ + "kv", + "data", + "put", + "--store", + "0", + "--group", + "1", + "--key", + "target-marker", + "--value", + "untouched", + ]) + .output() + .unwrap(); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + assert!(!output.status.success(), "unexpected success: {stderr}"); + stderr +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cli_mixed_identities_reject_a_workload_before_native_connect() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("cli-target-mixed"); + let left = TestTargetServer::start(&uuid::Uuid::new_v4().to_string()).await; + let right = TestTargetServer::start(&uuid::Uuid::new_v4().to_string()).await; + for args in test_operations() { + let error = test_rejected_operation( + root.path(), + &[ + "--management-endpoint", + &left.origin, + "--management-endpoint", + &right.origin, + ], + &args, + ); + assert!( + error.contains("disagree on cluster identity"), + "{args:?}: {error}" + ); + assert!(!error.contains("native transport"), "{error}"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cli_named_binding_never_adopts_a_replacement_uuid_from_explicit_addresses() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("cli-target-binding"); + let path = root.path().join("persistent/clusters/demo/target.json"); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let original_id = uuid::Uuid::new_v4().to_string(); + let original = TestTargetServer::start(&original_id).await; + let replacement = TestTargetServer::start(&uuid::Uuid::new_v4().to_string()).await; + let bytes = serde_json::to_vec(&ClusterTargetBinding { + version: 1, + name: "demo".into(), + cluster_id: original_id, + management_endpoints: vec![original.origin.clone()], + runtime: None, + }) + .unwrap(); + std::fs::write(&path, &bytes).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + let error = run( + root.path(), + &["--cluster", "demo", "--management-endpoint", &replacement.origin], + ); + assert!(error.contains("disagree on cluster identity"), "{error}"); + assert_eq!(std::fs::read(path).unwrap(), bytes); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cli_http_only_target_fails_native_workload_without_localhost_fallback() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("cli-target-native"); + let server = TestTargetServer::start(&uuid::Uuid::new_v4().to_string()).await; + let error = run(root.path(), &["--management-endpoint", &server.origin]); + assert!(error.contains("native RPC endpoint is unreachable"), "{error}"); + let error = run(root.path(), &["--cluster", "missing"]); + assert!(error.contains("no readable target binding"), "{error}"); +} + +#[test] +fn cli_rejects_removed_system_address_flags() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("cli-target-flags"); + let error = run( + root.path(), + &[ + "--management-endpoint", + "http://127.0.0.1:9090", + "--system-ip", + "127.0.0.1", + ], + ); + assert!(error.contains("unexpected argument '--system-ip'"), "{error}"); +} + +#[test] +fn retained_workloads_require_a_target_before_any_runtime_or_native_effect() { + for args in [ + vec!["bench", "kv", "prepare", "--keys", "1"], + vec!["bench", "diskdb", "allocate"], + vec!["bench", "chunkdb", "allocate"], + vec!["bench", "chunkio", "write"], + vec!["bench", "s3", "write", "--operations", "1"], + ] { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("bench-no-target"); + let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) + .env("CROWDB_RUNTIME_ROOT", root.path()) + .env("CROWDB_CONTAINER_RUNTIME", "/path/not/executed") + .env_remove("CROWDB_CLUSTER") + .env_remove("CROWDB_SYSTEM_IP") + .env_remove("CROWDB_SYSTEM_PORT") + .args(&args) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + let error = String::from_utf8_lossy(&output.stderr); + assert!( + error + .contains("select --cluster or --management-endpoint before a cluster operation or workload"), + "{args:?}: {error}" + ); + assert!(!root.path().join("persistent").exists()); + } +} + +#[test] +fn every_attached_operation_rejects_legacy_address_fallback_before_connecting() { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + for args in test_operations() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("operation-no-target"); + let error = test_rejected_operation(root.path(), &[], &args); + assert!( + error.contains("select --cluster or --management-endpoint"), + "{args:?}: {error}" + ); + } + assert_eq!( + listener.accept().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock + ); +} + +#[test] +fn every_attached_operation_rejects_an_unreachable_management_target() { + let closed = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", closed.local_addr().unwrap()); + drop(closed); + for args in test_operations() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("operation-unreachable"); + let error = test_rejected_operation(root.path(), &["--management-endpoint", &endpoint], &args); + assert!( + error.contains("management endpoint is unavailable"), + "{args:?}: {error}" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn native_operations_reject_unreachable_rpc_without_runtime_fallback() { + let server = TestTargetServer::start(&uuid::Uuid::new_v4().to_string()).await; + for args in test_operations().into_iter().filter(|args| !args.contains(&"s3")) { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("operation-native"); + let error = test_rejected_operation(root.path(), &["--management-endpoint", &server.origin], &args); + assert!( + error.contains("native RPC endpoint is unreachable"), + "{args:?}: {error}" + ); + assert!(!error.contains("load launch registry"), "{error}"); + } +} diff --git a/app/crowdb-cli/tests/cluster_workloads_test.rs b/app/crowdb-cli/tests/cluster_workloads_test.rs new file mode 100644 index 000000000..999e66937 --- /dev/null +++ b/app/crowdb-cli/tests/cluster_workloads_test.rs @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Opt-in process regression against an existing disposable normal OCI cluster. +//! The owning fixture stops `ChunkDB` and its upstream producers before comparing +//! whole-group busy bytes, while retaining KV, `DiskDB` and `DiskIO` processes. + +use crowdb_console_shared::deployment::target; +use crowdb_diskdb_client::{DiskdbClient, DiskdbRpcTransport}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient, ServiceRegistryClient}; +use crowdb_protocol::{ + common::ChunkId, + diskdb::rpc::{AllocateBlocksRequest, FreeBlocksRequest}, +}; +use std::{process::Command, sync::Arc}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires CROWDB_TEST_CLUSTER_MANAGEMENT_ENDPOINT for a disposable normal OCI cluster"] +async fn mixed_disk_workload_normalizes_frees_predating_its_baseline() { + let endpoint = + std::env::var("CROWDB_TEST_CLUSTER_MANAGEMENT_ENDPOINT").expect("explicit disposable target"); + let selected = target::resolve(None, std::slice::from_ref(&endpoint), None) + .await + .unwrap(); + selected.require_native_transport().await.unwrap(); + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + selected.identity.group0_management_endpoints.clone(), + ))); + kv.seed_leader(0, 0, selected.identity.group0_rpc_endpoints[0].clone()); + selected.verify_native_authority(&kv).await.unwrap(); + let client = DiskdbClient::new( + ServiceRegistryClient::from_shared(kv), + Arc::new(DiskdbRpcTransport::new()), + ); + client.refresh_endpoints().await.unwrap(); + let group = client.disk_group_ids()[0]; + let owner = uuid::Uuid::new_v4().as_u64_pair(); + let allocated = client + .allocate_blocks(AllocateBlocksRequest { + disk_group_id: group, + unit_count: 256, + count: 1, + owner_chunk: Some(ChunkId { + high: owner.0, + low: owner.1, + }), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(allocated.segments.len(), 1); + let busy = client.query_disk_group(group).await.unwrap().disk_groups[0].busy_bytes; + let freed = client + .free_blocks(FreeBlocksRequest { + segments: allocated.segments, + }) + .await + .unwrap(); + assert_eq!(freed.freed_count, 1); + assert_eq!( + client.query_disk_group(group).await.unwrap().disk_groups[0].busy_bytes, + busy, + "fixture must retain a pending free before compaction" + ); + let binary = + std::env::var_os("CROWDB_E2E_CLI_BIN").unwrap_or_else(|| env!("CARGO_BIN_EXE_crowdb-cli").into()); + let output = Command::new(binary) + .env_remove("CROWDB_CLUSTER") + .env_remove("CROWDB_SYSTEM_IP") + .env_remove("CROWDB_SYSTEM_PORT") + .args([ + "--management-endpoint", + &endpoint, + "bench", + "diskdb", + "mix", + "--duration-secs", + "1", + "--concurrency", + "1", + "--metrics-interval", + "0", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "stdout={} stderr={}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + let report = String::from_utf8(output.stdout).unwrap(); + let counter = |key: &str| { + report + .split_whitespace() + .find_map(|part| part.strip_prefix(key)) + .unwrap() + .parse::() + .unwrap() + }; + assert!(counter("ops=") > 0); + assert_eq!(counter("errors="), 0); + assert!(counter("expected_delta=") > 0); + assert_eq!(counter("busy_delta="), counter("expected_delta=")); +} diff --git a/app/crowdb-cli/tests/common/cluster_target.rs b/app/crowdb-cli/tests/common/cluster_target.rs new file mode 100644 index 000000000..dad20b520 --- /dev/null +++ b/app/crowdb-cli/tests/common/cluster_target.rs @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{routing::get, Json, Router}; +use crowdb_protocol::mgmt::cluster::ClusterTargetIdentity; + +pub struct TestTargetServer { + pub origin: String, + worker: tokio::task::JoinHandle<()>, +} + +impl TestTargetServer { + pub async fn start(cluster_id: &str) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let closed = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let identity = ClusterTargetIdentity { + version: 1, + cluster_id: cluster_id.into(), + group0_management_endpoints: vec![origin.clone()], + group0_rpc_endpoints: vec![closed.local_addr().unwrap().to_string()], + }; + drop(closed); + let router = Router::new().route( + "/api/cluster/target", + get(move || { + let identity = identity.clone(); + async move { Json(identity) } + }), + ); + let worker = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + Self { origin, worker } + } +} + +impl Drop for TestTargetServer { + fn drop(&mut self) { + self.worker.abort(); + } +} diff --git a/app/crowdb-cli/tests/common/local_docker.py b/app/crowdb-cli/tests/common/local_docker.py new file mode 100644 index 000000000..74681286b --- /dev/null +++ b/app/crowdb-cli/tests/common/local_docker.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Process-boundary Docker fixture; every mutation is persisted for assertions.""" + +import json +import os +from pathlib import Path +import sys + +root = Path(os.environ["CROWDB_RUNTIME_ROOT"]) +path = root / "docker.json" +state = json.loads(path.read_text()) +args = sys.argv[1:] +state["calls"].append(args) +result = "" + +def values(flag): + return [args[index + 1] for index, arg in enumerate(args[:-1]) if arg == flag] + +def one(flag): + return values(flag)[0] + +if args[:2] == ["context", "show"]: + result = state["runtime"]["context"] +elif args[:2] == ["context", "inspect"]: + result = [{"Endpoints": {"docker": {"Host": state["runtime"]["endpoint"]}}}] +elif args[0] == "info": + result = state["runtime"]["daemon"] +elif args[:2] == ["image", "inspect"]: + result = [{"Id": "sha256:" + "a" * 64, "Os": "linux", "Architecture": "amd64", + "Config": {"Labels": {"org.opencontainers.image.source": "https://github.com/buzzcrow/crowdb"}}}] +elif args[:2] == ["ps", "-aq"]: + result = "\n".join(state["container"]) +elif len(args) > 1 and args[1] == "ls": + result = "\n".join(state[args[0]]) +elif len(args) > 1 and args[1] == "inspect": + result = [state[args[0]][name] for name in args[2:]] +elif len(args) > 1 and args[1] == "create": + name = args[-1] + item = {"Id": name, "Name": name, "Labels": dict(value.split("=", 1) for value in values("--label"))} + if args[0] == "network": + item["Driver"] = "bridge" + state[args[0]][name] = item + result = name +elif args[0] == "run": + name = one("--name") + if state.get("fail_run_once") == name: + del state["fail_run_once"] + path.write_text(json.dumps(state)) + print("injected interrupted creation", file=sys.stderr) + sys.exit(1) + volume = dict(part.split("=", 1) for part in one("--mount").split(",")) + ports = {} + for value in values("-p"): + ip, host, container = value.split(":") + ports[container + "/tcp"] = [{"HostIp": ip, "HostPort": host}] + state["container"][name] = { + "Id": name, "Name": "/" + name, "Image": args[-1], "State": {"Running": True}, + "Config": {"Labels": dict(value.split("=", 1) for value in values("--label")), "Env": values("-e")}, + "HostConfig": {"NanoCpus": round(float(one("--cpus")) * 1_000_000_000), + "Memory": int(one("--memory")[:-1]) * 1024 * 1024, + "NetworkMode": one("--network"), "PortBindings": ports}, + "Mounts": [{"Destination": volume["target"], "Type": "volume", "Name": volume["source"], "RW": True}]} + result = name +elif args[0] in ("start", "stop"): + state["container"][args[-1]]["State"]["Running"] = args[0] == "start" + result = args[-1] +else: + raise RuntimeError("unexpected Docker operation: " + repr(args)) +path.write_text(json.dumps(state)) +print(result if isinstance(result, str) else json.dumps(result)) diff --git a/app/crowdb-cli/tests/common/target_commands.rs b/app/crowdb-cli/tests/common/target_commands.rs new file mode 100644 index 000000000..c1efeaa40 --- /dev/null +++ b/app/crowdb-cli/tests/common/target_commands.rs @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::{path::Path, process::Command}; + +pub fn test_operations() -> Vec> { + vec![ + vec!["cluster", "rack", "list"], + vec!["cluster", "node", "list"], + vec!["cluster", "disk-group", "list"], + vec!["cluster", "disk", "list"], + vec!["cluster", "topology", "--node", "1"], + vec!["kv", "store", "list"], + vec!["kv", "group", "list", "--store", "0"], + vec![ + "kv", "replica", "add", "--store", "0", "--group", "1", "--node", "2", + ], + vec![ + "kv", + "data", + "put", + "--store", + "0", + "--group", + "1", + "--key", + "target-marker", + "--value", + "untouched", + ], + vec!["s3", "bucket", "put", "target-marker"], + vec!["bench", "kv", "prepare", "--keys", "1"], + vec!["bench", "diskdb", "allocate"], + vec!["bench", "chunkdb", "allocate"], + vec!["bench", "chunkio", "write"], + vec!["bench", "s3", "write", "--operations", "1"], + ] +} + +pub fn test_rejected_operation(root: &Path, selected: &[&str], args: &[&str]) -> String { + let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) + .env("CROWDB_RUNTIME_ROOT", root) + .env("CROWDB_CONTAINER_RUNTIME", "/path/not/executed") + .env_remove("CROWDB_CLUSTER") + .env_remove("CROWDB_SYSTEM_IP") + .env_remove("CROWDB_SYSTEM_PORT") + .args(selected) + .args(args) + .output() + .unwrap(); + let error = String::from_utf8_lossy(&output.stderr).into_owned(); + assert_eq!(output.status.code(), Some(2), "{args:?}: {error}"); + assert!(!root.join("persistent").exists(), "{args:?}: {error}"); + error +} diff --git a/app/crowdb-cli/tests/container_cluster_test.rs b/app/crowdb-cli/tests/container_cluster_test.rs deleted file mode 100644 index 3f05aadfc..000000000 --- a/app/crowdb-cli/tests/container_cluster_test.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use crowdb_test_harness::test_dirs::TestDir; -use serde_json::json; -use std::os::unix::fs::{symlink, PermissionsExt}; -use std::path::Path; -use std::process::Command; - -fn state(root: &Path) { - std::fs::write( - root.join("container-cluster.json"), - json!({ - "version":1,"id":"0123456789abcdef0123456789abcdef","root":root, - "image":format!("sha256:{}", "a".repeat(64)),"name":"owned-cluster", - "network":"owned-cluster-network","phase":"ready","node_cpus":2.0, - "memory_mib":2048,"addresses":["172.20.0.2","172.20.0.3","172.20.0.4"] - }) - .to_string(), - ) - .unwrap(); -} - -fn runtime(root: &Path, body: &str) -> std::path::PathBuf { - let script = root.join("runtime-stub"); - std::fs::write(&script, format!("#!/bin/sh\n{body}\n")).unwrap(); - std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o700)).unwrap(); - script -} - -#[test] -fn startup_preserves_recorded_heartbeat_configuration_without_runtime_mutation() { - let directory = TestDir::new("cli-container-heartbeat-config").unwrap(); - let root = std::fs::canonicalize(directory.path()).unwrap(); - state(&root); - let before = std::fs::read(root.join("container-cluster.json")).unwrap(); - let script = runtime(&root, "echo 'unexpected runtime mutation' >&2; exit 77"); - let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) - .args(["s3", "cluster", "start", "--root"]) - .arg(&root) - .args(["--diskdb-heartbeat-interval-secs", "3"]) - .env("CROWDB_CONTAINER_RUNTIME", script) - .output() - .unwrap(); - assert!(!output.status.success()); - let error = String::from_utf8_lossy(&output.stderr); - assert!(error.contains("recorded Diskdb heartbeat configuration")); - assert!(!error.contains("unexpected runtime mutation")); - assert_eq!( - std::fs::read(root.join("container-cluster.json")).unwrap(), - before - ); -} - -#[test] -fn lifecycle_rejects_foreign_network_without_mutation() { - let directory = TestDir::new("cli-container-foreign-network").unwrap(); - let root = std::fs::canonicalize(directory.path()).unwrap(); - state(&root); - let script = runtime(&root, "case \"$*\" in\n 'network inspect owned-cluster-network') echo '[{\"Name\":\"owned-cluster-network\",\"Labels\":{\"org.crowdb.cluster.id\":\"foreign\"}}]' ;;\n *) echo 'unexpected runtime mutation' >&2; exit 77 ;;\nesac"); - for action in ["status", "stop", "restart", "delete"] { - let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) - .args(["s3", "cluster", action, "--root"]) - .arg(&root) - .env("CROWDB_CONTAINER_RUNTIME", &script) - .output() - .unwrap(); - assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr).contains("different cluster ownership")); - assert!(root.join("container-cluster.json").exists()); - } -} - -#[test] -fn runtime_failure_is_not_treated_as_missing_network() { - let directory = TestDir::new("cli-container-runtime-error").unwrap(); - let root = std::fs::canonicalize(directory.path()).unwrap(); - state(&root); - let script = runtime(&root, "echo 'runtime executable not found' >&2; exit 42"); - let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) - .args(["s3", "cluster", "status", "--root"]) - .arg(&root) - .env("CROWDB_CONTAINER_RUNTIME", script) - .output() - .unwrap(); - assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr).contains("runtime executable not found")); -} - -#[test] -fn generated_member_name_reaches_runtime_ownership_lookup() { - let directory = TestDir::new("cli-container-member-name").unwrap(); - let script = runtime( - directory.path(), - "echo 'member ownership lookup reached' >&2; exit 42", - ); - let name = format!("{}-node2", "a".repeat(48)); - let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) - .args(["s3", "cluster", "status", "--container", &name]) - .env("CROWDB_CONTAINER_RUNTIME", script) - .output() - .unwrap(); - assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr).contains("member ownership lookup reached")); -} - -#[test] -fn lifecycle_rejects_symlinked_data_before_runtime_operations() { - let directory = TestDir::new("cli-container-data-ownership").unwrap(); - let root = std::fs::canonicalize(directory.path()).unwrap(); - state(&root); - symlink("/", root.join("nodes")).unwrap(); - let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) - .args(["s3", "cluster", "delete", "--root"]) - .arg(&root) - .env("CROWDB_CONTAINER_RUNTIME", "/path/not/executed") - .output() - .unwrap(); - assert!(!output.status.success()); - assert!(String::from_utf8_lossy(&output.stderr).contains("cannot be symbolic links")); - assert!(root.join("container-cluster.json").exists()); -} - -#[test] -fn delete_cleans_owned_partial_writes_and_preserves_foreign_files() { - for foreign in [false, true] { - let directory = TestDir::new("cli-container-partial-state-cleanup").unwrap(); - let root = directory.path().join("cluster"); - std::fs::create_dir(&root).unwrap(); - let root = std::fs::canonicalize(root).unwrap(); - state(&root); - let temporary = root.join(format!( - "s3-mini-cluster.initializing.json.0123456789abcdef0123456789abcdef.{}.tmp", - "a".repeat(32) - )); - std::fs::write(&temporary, "partial write").unwrap(); - let foreign_file = root.join(format!( - "container-cluster.json.ffffffffffffffffffffffffffffffff.{}.tmp", - "b".repeat(32) - )); - if foreign { - std::fs::write(&foreign_file, "operator file").unwrap(); - } - let script = runtime(directory.path(), "echo 'Error: No such object' >&2; exit 1"); - let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) - .args(["s3", "cluster", "delete", "--root"]) - .arg(&root) - .env("CROWDB_CONTAINER_RUNTIME", script) - .output() - .unwrap(); - assert!(!temporary.exists()); - if foreign { - assert!(!output.status.success()); - assert_eq!(std::fs::read_to_string(foreign_file).unwrap(), "operator file"); - } else { - assert!( - output.status.success(), - "{}", - String::from_utf8_lossy(&output.stderr) - ); - assert!(!root.exists()); - } - } -} diff --git a/app/crowdb-cli/tests/direct_cli_test.rs b/app/crowdb-cli/tests/direct_cli_test.rs index 0ffbc58f7..b7559e462 100644 --- a/app/crowdb-cli/tests/direct_cli_test.rs +++ b/app/crowdb-cli/tests/direct_cli_test.rs @@ -1,21 +1,37 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! CLI e2e (R126 direct-to-group-0): `cluster status` and `cluster rack -//! list` route directly through `--system-ip` / `--system-port` against a -//! real `crowdb-kv-server` with group 0 initialized — no `crowdb-web` -//! intermediary. - -mod common; - -use common::direct::crowdb_cli_bin; +use std::process::Command; #[test] -fn clean_rejects_service_restart_without_launch_registry() { - let output = std::process::Command::new(crowdb_cli_bin()) - .args(["cluster", "clean", "--restart-services"]) - .output() - .unwrap(); - assert_eq!(output.status.code(), Some(2)); - assert!(String::from_utf8_lossy(&output.stderr).contains("--registry")); +fn retired_process_commands_and_selectors_fail_before_runtime_access() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("cli-removed-paths"); + for args in [ + vec!["launch", "list"], + vec!["kv", "server", "list"], + vec!["cluster", "init"], + vec!["cluster", "local-deploy"], + vec!["cluster", "destroy"], + vec!["cluster", "clean"], + vec!["chunk", "diskdb", "deploy"], + vec!["bench", "rpc"], + vec!["s3", "cluster", "start"], + vec!["--registry", "missing", "cluster", "status"], + vec!["--system-ip", "127.0.0.1", "cluster", "status"], + vec!["--system-port", "7000", "cluster", "status"], + ] { + let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) + .env("CROWDB_RUNTIME_ROOT", root.path()) + .env("CROWDB_CONTAINER_RUNTIME", "/must/not/execute") + .args(&args) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2), "{args:?}: {output:?}"); + let error = String::from_utf8_lossy(&output.stderr); + assert!( + error.contains("unrecognized subcommand") || error.contains("unexpected argument"), + "{error}" + ); + } + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); } diff --git a/app/crowdb-cli/tests/diskdb_bench_test.rs b/app/crowdb-cli/tests/diskdb_bench_test.rs index 7f71a1a46..09f798274 100644 --- a/app/crowdb-cli/tests/diskdb_bench_test.rs +++ b/app/crowdb-cli/tests/diskdb_bench_test.rs @@ -10,7 +10,7 @@ use std::process::Command; use common::direct::crowdb_cli_bin; #[test] -fn allocate_and_mix_commands_expose_supported_modes() { +fn allocate_and_mix_commands_expose_workload_controls_without_deployment_modes() { for workload in ["allocate", "mix"] { let output = Command::new(crowdb_cli_bin()) .args(["bench", "diskdb", workload, "--help"]) @@ -18,8 +18,7 @@ fn allocate_and_mix_commands_expose_supported_modes() { .expect("run crowdb-cli benchmark help"); assert!(output.status.success(), "{workload} help must succeed"); let stdout = String::from_utf8(output.stdout).expect("help is UTF-8"); - assert!(stdout.contains("--mode ")); - assert!(stdout.contains("possible values: mem, block")); + assert!(!stdout.contains("--mode")); assert!(stdout.contains("--duration-secs")); assert!(stdout.contains("--concurrency")); assert!(stdout.contains("--diskdb-connections")); diff --git a/app/crowdb-cli/tests/launch_registry_cli_test.rs b/app/crowdb-cli/tests/launch_registry_cli_test.rs deleted file mode 100644 index 2f75ec431..000000000 --- a/app/crowdb-cli/tests/launch_registry_cli_test.rs +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -#![cfg(target_os = "linux")] - -use std::os::unix::fs::PermissionsExt; -use std::path::Path; -use std::process::Command; -use std::time::Duration; - -use crowdb_console_shared::config::web::{LaunchRecord, LaunchRegistry}; -use crowdb_console_shared::launch::LaunchRuntime; -use crowdb_console_shared::lifecycle; -use crowdb_test_harness::test_dirs::tempdir_in_test_data; - -struct TestProcesses(Vec); -impl Drop for TestProcesses { - fn drop(&mut self) { - for pid in &self.0 { - if lifecycle::process_is_alive(*pid) { - let _ = lifecycle::stop_pid_with_timeout(*pid, Duration::from_secs(2)); - } - } - } -} - -fn run_command(registry: &Path, port: u16, args: &[&str]) -> String { - let output = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) - .arg("--registry") - .arg(registry) - .arg("--system-port") - .arg(port.to_string()) - .env("CROWDB_CLI_STATE", registry.with_file_name("invalid-legacy.toml")) - .args(args) - .output() - .unwrap(); - assert!( - output.status.success(), - "CLI failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).unwrap() -} - -fn record(directory: &Path, service: &str) -> LaunchRecord { - let binary = directory.join("service"); - std::fs::write(&binary, "#!/bin/sh\nexec sleep 60\n").unwrap(); - std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o700)).unwrap(); - let config = directory.join("service.toml"); - std::fs::write(&config, "").unwrap(); - LaunchRecord { - node_id: 701, - service_id: service.into(), - host: "localhost".into(), - ssh_credential_ref: None, - ssh_user: None, - ssh_port: 22, - binary_path: binary, - service_config_path: config, - workspace: directory.to_owned(), - auto_start: false, - args: Vec::new(), - readiness_url: None, - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn chunk_commands_and_generic_launch_controls_share_process_identity() { - let dir = tempdir_in_test_data("cli-chunk-launch"); - let records: Vec<_> = ["diskdb", "chunkdb", "diskio"] - .iter() - .map(|service| record(dir.path(), service)) - .collect(); - let path = dir.path().join("launches.toml"); - LaunchRegistry { - version: 1, - launches: records.clone(), - } - .save(&path) - .unwrap(); - let runtime = LaunchRuntime::for_registry(&path).unwrap(); - let mut guard = TestProcesses(Vec::new()); - for record in &records { - let mut args = vec!["chunk"]; - if record.service_id != "diskdb" { - args.push("stub"); - } - args.extend([record.service_id.as_str(), "deploy", "--node", "701"]); - run_command(&path, 9, &args); - let first = runtime.status(record).await.unwrap().unwrap(); - guard.0.push(first.pid); - run_command( - &path, - 9, - &[ - "launch", - "start", - "--node", - "701", - "--service", - &record.service_id, - ], - ); - assert_eq!(runtime.status(record).await.unwrap(), Some(first)); - assert!(run_command(&path, 9, &["launch", "list"]).contains(&first.pid.to_string())); - run_command( - &path, - 9, - &[ - "launch", - "restart", - "--node", - "701", - "--service", - &record.service_id, - ], - ); - let next = runtime.status(record).await.unwrap().unwrap(); - guard.0.push(next.pid); - assert_ne!(first, next); - let action = args.len() - 3; - args[action] = "stop"; - run_command(&path, 9, &args); - assert!(runtime.status(record).await.unwrap().is_none()); - } - assert_eq!(LaunchRegistry::load(&path).unwrap().launches, records); -} diff --git a/app/crowdb-cli/tests/s3_cli_test.rs b/app/crowdb-cli/tests/s3_cli_test.rs index 985096d92..fb5fdacb6 100644 --- a/app/crowdb-cli/tests/s3_cli_test.rs +++ b/app/crowdb-cli/tests/s3_cli_test.rs @@ -3,12 +3,20 @@ use std::io::{Read, Write}; use std::net::TcpListener; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::process::Command; use std::sync::mpsc; +#[path = "common/cluster_target.rs"] +mod common; + fn cli() -> Command { - Command::new(env!("CARGO_BIN_EXE_crowdb-cli")) + let mut command = Command::new(env!("CARGO_BIN_EXE_crowdb-cli")); + command + .env_remove("CROWDB_CLUSTER") + .env_remove("CROWDB_SYSTEM_IP") + .env_remove("CROWDB_SYSTEM_PORT"); + command } fn tempdir(tag: &str) -> PathBuf { @@ -21,33 +29,40 @@ fn tempdir(tag: &str) -> PathBuf { path } -fn write_cluster_record(root: &Path, endpoint: &str) { - std::fs::create_dir_all(root).expect("create cluster root"); - let record = serde_json::json!({ - "version": 1, - "endpoint": endpoint, - "tenant": "local", - "storage_profile": "persistent" - }); - std::fs::write( - root.join("s3-mini-cluster.json"), - serde_json::to_vec_pretty(&record).expect("record json"), - ) - .expect("write record"); - std::fs::write( - root.join("s3-local-state.toml"), - "version = 1\ngroup0_seeds = ['http://127.0.0.1:10000']\n[[service]]\nid = 'kv-1'\nurl = 'http://127.0.0.1:10000'\nnode_id = 1\n", - ) - .expect("write launch-only state"); -} - fn mock_http_once(response_content_type: &str, response_body: &[u8]) -> (String, mpsc::Receiver>) { let listener = TcpListener::bind("127.0.0.1:0").expect("bind mock HTTP server"); let endpoint = format!("http://{}", listener.local_addr().expect("mock address")); let body = response_body.to_vec(); let content_type = response_content_type.to_string(); let (sender, receiver) = mpsc::channel(); + let s3_endpoint = endpoint.clone(); std::thread::spawn(move || { + let (mut identity_stream, _) = listener.accept().expect("accept target observation"); + let mut request = Vec::new(); + let mut buffer = [0; 4096]; + while !request.windows(4).any(|part| part == b"\r\n\r\n") { + let size = identity_stream.read(&mut buffer).unwrap(); + assert_ne!(size, 0); + request.extend_from_slice(&buffer[..size]); + } + assert!(request.starts_with(b"GET /api/cluster/target HTTP/1.1")); + let identity = serde_json::json!({"version":1, "cluster_id":uuid::Uuid::from_u128(1).to_string(), + "group0_management_endpoints":["http://127.0.0.1:10000"], "group0_rpc_endpoints":["127.0.0.1:10100"]}).to_string(); + write!(identity_stream, "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", identity.len(), identity).unwrap(); + drop(identity_stream); + let (mut credential_stream, _) = listener.accept().expect("accept credential observation"); + let mut request = Vec::new(); + while !request.windows(4).any(|part| part == b"\r\n\r\n") { + let size = credential_stream.read(&mut buffer).unwrap(); + assert_ne!(size, 0); + request.extend_from_slice(&buffer[..size]); + } + assert!(request.starts_with(b"GET /api/access/credentials HTTP/1.1")); + let credentials = serde_json::json!({"s3":{"endpoint":s3_endpoint, "region":"us-east-1", + "access_key_id":"test-access", "secret_access_key":"test-secret-never-print"}}) + .to_string(); + write!(credential_stream, "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", credentials.len(), credentials).unwrap(); + drop(credential_stream); let (mut stream, _) = listener.accept().expect("accept request"); let mut request = Vec::new(); let mut buffer = [0; 4096]; @@ -88,7 +103,7 @@ fn mock_http_once(response_content_type: &str, response_body: &[u8]) -> (String, } #[test] -fn s3_benchmark_exposes_all_memory_workloads() { +fn s3_benchmark_exposes_attached_workloads_without_deployment_flags() { for workload in ["write", "read", "range-read", "list", "mix"] { let output = cli() .args(["bench", "s3", workload, "--help"]) @@ -100,25 +115,16 @@ fn s3_benchmark_exposes_all_memory_workloads() { String::from_utf8_lossy(&output.stderr) ); let stdout = String::from_utf8_lossy(&output.stdout); - assert!(stdout.contains("--memory-budget-bytes")); + assert!(stdout.contains("--max-dataset-bytes")); assert!(stdout.contains("--operations")); + assert!(!stdout.contains("--work-dir")); } } #[test] fn object_range_rejects_regression_before_cluster_access() { let output = cli() - .args([ - "s3", - "object", - "get", - "--data-dir", - "/path/that/is/not/opened", - "bucket", - "key", - "--range", - "9-3", - ]) + .args(["s3", "object", "get", "bucket", "key", "--range", "9-3"]) .output() .expect("run invalid range"); assert!(!output.status.success()); @@ -130,9 +136,17 @@ fn help_is_console_first_and_uses_http_operation_names() { let output = cli().arg("--help").output().expect("run CLI help"); assert!(output.status.success()); let stdout = String::from_utf8_lossy(&output.stdout); - assert!(stdout.contains("--system-ip")); - assert!(stdout.contains("--system-port")); - for removed in ["--config", "--json", "--log-root", "Group-0 leader"] { + assert!(stdout.contains("--cluster")); + assert!(stdout.contains("--management-endpoint")); + for removed in [ + "--config", + "--json", + "--log-root", + "--registry", + "--system-ip", + "--system-port", + "Group-0 leader", + ] { assert!( !stdout.contains(removed), "unexpected {removed} in help: {stdout}" @@ -172,7 +186,13 @@ fn help_is_console_first_and_uses_http_operation_names() { .output() .expect("run put help"); let stdout = String::from_utf8_lossy(&output.stdout); - for option in ["--root", "--file", "--text", "--random-size"] { + for option in [ + "--cluster", + "--management-endpoint", + "--file", + "--text", + "--random-size", + ] { assert!(stdout.contains(option), "missing {option}: {stdout}"); } assert!(!stdout.contains("--data-dir"), "legacy option exposed: {stdout}"); @@ -183,7 +203,7 @@ fn simple_s3_command_uses_console_without_file_log() { let root = tempdir("no-file-log"); let output = cli() .current_dir(&root) - .args(["s3", "bucket", "list", "--root", "missing"]) + .args(["s3", "bucket", "list"]) .output() .expect("run list against missing cluster"); assert!(!output.status.success()); @@ -199,13 +219,10 @@ fn bucket_get_prints_headers_and_formats_xml() { let xml = br#"test"#; let (endpoint, captured) = mock_http_once("application/xml", xml); let root = tempdir("bucket-get"); - let cluster_root = root.join("data"); - write_cluster_record(&cluster_root, &endpoint); let output = cli() .current_dir(&root) - .args(["s3", "bucket", "get", "test", "--root"]) - .arg(&cluster_root) + .args(["--management-endpoint", &endpoint, "s3", "bucket", "get", "test"]) .output() .expect("run bucket get"); assert!( @@ -220,6 +237,11 @@ fn bucket_get_prints_headers_and_formats_xml() { assert!(stdout.contains("\n test\n"), "stdout={stdout}"); let stderr = String::from_utf8_lossy(&output.stderr); assert!(stderr.contains("HTTP REQUEST")); + assert!(request + .to_ascii_lowercase() + .contains("authorization: aws4-hmac-sha256")); + assert!(stderr.contains("authorization: ")); + assert!(!stderr.contains("test-secret-never-print")); assert!(stderr.contains("HTTP RESPONSE")); assert!(stderr.contains("x-test-header: visible")); assert!(stderr.contains("body: XML")); @@ -231,13 +253,18 @@ fn bucket_get_prints_headers_and_formats_xml() { fn object_put_text_sets_content_type_and_shows_text_body() { let (endpoint, captured) = mock_http_once("application/xml", b""); let root = tempdir("put-text"); - let cluster_root = root.join("data"); - write_cluster_record(&cluster_root, &endpoint); let output = cli() .current_dir(&root) - .args(["s3", "object", "put", "bucket", "key", "--root"]) - .arg(&cluster_root) + .args([ + "--management-endpoint", + &endpoint, + "s3", + "object", + "put", + "bucket", + "key", + ]) .args(["--text", "object content"]) .output() .expect("run object put --text"); @@ -265,13 +292,18 @@ fn object_put_text_sets_content_type_and_shows_text_body() { fn object_put_random_size_sends_exact_binary_length() { let (endpoint, captured) = mock_http_once("application/xml", b""); let root = tempdir("put-random"); - let cluster_root = root.join("data"); - write_cluster_record(&cluster_root, &endpoint); let output = cli() .current_dir(&root) - .args(["s3", "object", "put", "bucket", "random.bin", "--root"]) - .arg(&cluster_root) + .args([ + "--management-endpoint", + &endpoint, + "s3", + "object", + "put", + "bucket", + "random.bin", + ]) .args(["--random-size", "257"]) .output() .expect("run object put --random-size"); @@ -291,3 +323,72 @@ fn object_put_random_size_sends_exact_binary_length() { assert!(stderr.contains("body: binary (257 bytes, not shown)")); let _ = std::fs::remove_dir_all(root); } + +#[test] +fn s3_lifecycle_and_private_root_selectors_are_removed() { + for (args, expected) in [ + ( + vec!["s3", "cluster", "start"], + "unrecognized subcommand 'cluster'", + ), + ( + vec!["s3", "bucket", "list", "--root", "unused"], + "unexpected argument '--root'", + ), + ] { + let output = cli().args(args).output().unwrap(); + assert_eq!(output.status.code(), Some(2)); + assert!( + String::from_utf8_lossy(&output.stderr).contains(expected), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn s3_mutation_rejects_mixed_cluster_identities_before_sending_data() { + let left = common::TestTargetServer::start(&uuid::Uuid::new_v4().to_string()).await; + let right = common::TestTargetServer::start(&uuid::Uuid::new_v4().to_string()).await; + let output = cli() + .args([ + "--management-endpoint", + &left.origin, + "--management-endpoint", + &right.origin, + "s3", + "object", + "put", + "bucket", + "key", + "--text", + "untouched", + ]) + .env("DOCKER_HOST", "unix:///unavailable-test-runtime") + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&output.stderr).contains("disagree on cluster identity")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn s3_credentials_must_resolve_before_reading_an_upload_or_writing_data() { + let selected = common::TestTargetServer::start(&uuid::Uuid::new_v4().to_string()).await; + let output = cli() + .args([ + "--management-endpoint", + &selected.origin, + "s3", + "object", + "put", + "bucket", + "key", + "--file", + "/path/that/must/not/be/read", + ]) + .env("DOCKER_HOST", "unix:///unavailable-test-runtime") + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&output.stderr).contains("selected cluster rejected credential access")); +} diff --git a/app/crowdb-diskdb/src/health.rs b/app/crowdb-diskdb/src/health.rs index cac9a1f19..a35e1a37a 100644 --- a/app/crowdb-diskdb/src/health.rs +++ b/app/crowdb-diskdb/src/health.rs @@ -35,6 +35,8 @@ pub struct ReadyResponse { /// `true` when `phase == "up"` and not degraded — ready to serve /// mutating RPCs. Convenience flag for load-balancer probes. pub ready: bool, + /// Disks whose uncertain persistence requires a newly fenced reconstruction. + pub recovery_required: Vec, } /// Build the health router bound to the given container. @@ -52,7 +54,8 @@ pub fn router(container: Arc) -> Router { async fn ready(State(state): State) -> (StatusCode, Json) { let phase = state.container.lifecycle_phase(); let degraded = state.container.is_degraded(); - let ready = phase == StartupPhase::Up && !degraded; + let recovery_required = crate::liveness::recovery::uncertain_writes(&state.container); + let ready = phase == StartupPhase::Up && !degraded && recovery_required.is_empty(); let code = if ready { StatusCode::OK } else { @@ -64,6 +67,7 @@ async fn ready(State(state): State) -> (StatusCode, Json, disk: Arc, + completion: Option>, ) { let zone_count = disk_value.zone_count; let zone_size_units = disk_value.zone_size_units; @@ -305,5 +308,10 @@ impl KeepAlive { disk.rebuild_active_zones(zone_rotate_count); dg.rebuild_allocating_disks(); info!(disk = ?disk_id, status = ?final_status, "init-state zone load complete"); + if let Some(completion) = completion { + // Publish the completed state immediately; the preceding sync may + // have advertised this disk while its zones were still loading. + completion.notify_one(); + } } } diff --git a/app/crowdb-diskdb/src/liveness/keepalive/scheduler.rs b/app/crowdb-diskdb/src/liveness/keepalive/scheduler.rs index 2c5b05caf..34e29e249 100644 --- a/app/crowdb-diskdb/src/liveness/keepalive/scheduler.rs +++ b/app/crowdb-diskdb/src/liveness/keepalive/scheduler.rs @@ -359,7 +359,16 @@ impl BackgroundTask for KeepAlive { Trigger::TimerFn(interval_fn) } } - None => Trigger::Timer(self.config.interval), + None => match &self.sync_trigger { + Some(notify) => { + let interval = self.config.interval; + Trigger::TimerOrEvent { + interval_fn: Box::new(move || interval), + notify: Arc::clone(notify), + } + } + None => Trigger::Timer(self.config.interval), + }, } } diff --git a/app/crowdb-diskdb/src/liveness/recovery.rs b/app/crowdb-diskdb/src/liveness/recovery.rs new file mode 100644 index 000000000..e03c4227c --- /dev/null +++ b/app/crowdb-diskdb/src/liveness/recovery.rs @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Unknown writes recover only across a newly fenced process session. + +use std::time::Duration; + +use crate::model::disk_group_container::DdbDiskGroupContainer; +use crowdb_protocol::common::DiskId; +use serde::Serialize; + +#[derive(Debug, Serialize)] +pub struct UncertainDiskWrite { + pub disk_group_id: u64, + pub disk_id: DiskId, + pub reason: &'static str, +} + +/// Snapshot uncertainty without opening a closed gate or changing disk state. +#[must_use] +pub fn uncertain_writes(container: &DdbDiskGroupContainer) -> Vec { + container + .disk_group_ids() + .into_iter() + .filter_map(|id| container.get_disk_group(id)) + .flat_map(|group| { + group + .disk_snapshot() + .iter() + .filter(|disk| disk.write_gate.outcome_unknown()) + .map(|disk| UncertainDiskWrite { + disk_group_id: group.disk_group_id, + disk_id: disk.disk_id, + reason: "metadata-write-outcome-unknown: waiting for a new process session, ownership fence and allocation-state reconstruction", + }) + .collect::>() + }) + .collect() +} + +/// A supervisor may restart the process; current gates remain closed until exit. +pub async fn wait_required(container: &DdbDiskGroupContainer) -> Vec { + let mut interval = tokio::time::interval(Duration::from_secs(1)); + loop { + interval.tick().await; + let writes = uncertain_writes(container); + if !writes.is_empty() { + container.enter_degraded_mode(); + return writes; + } + } +} diff --git a/app/crowdb-diskdb/src/main.rs b/app/crowdb-diskdb/src/main.rs index 692eddf31..e1e76bfcb 100644 --- a/app/crowdb-diskdb/src/main.rs +++ b/app/crowdb-diskdb/src/main.rs @@ -290,22 +290,15 @@ async fn main() { temp_failure_timeout_secs: config.load().heartbeat.temp_failure_timeout_secs, }; let notify_enabled = config.load().notify.notify_enabled; - let sync_trigger = if notify_enabled { - Some(Arc::new(tokio::sync::Notify::new())) - } else { - None - }; + // Local completion events wake publication even when remote watch is off. + let sync_trigger = Arc::new(tokio::sync::Notify::new()); let keepalive = KeepAlive::new(hw, svc, container.clone(), keepalive_cfg) .with_ownership_fencing() .with_cas_retry_metric(Arc::clone(&metrics.allocate_retry_cas_bit)) .with_config_handle(Arc::clone(&config)) .with_rpc_endpoint(config.load().server.rpc_listen_addr.clone()) .with_metrics(metrics.clone()); - let keepalive = if let Some(ref trigger) = sync_trigger { - keepalive.with_sync_trigger(Arc::clone(trigger)) - } else { - keepalive - }; + let keepalive = keepalive.with_sync_trigger(Arc::clone(&sync_trigger)); info!("running blocking initial keep-alive tick"); let init_outcome = keepalive.tick().await; @@ -381,8 +374,10 @@ async fn main() { let load_kv = Arc::clone(&dg_kv); let load_cfg = (**config.load()).clone(); let load_metrics = metrics.clone(); + let loaded = Arc::clone(&sync_trigger); let load_handle = tokio::spawn(async move { run_zone_load(load_kv, load_container, &load_cfg, &load_metrics).await; + loaded.notify_one(); }); // Build the background-task runner with keepalive + compaction. @@ -434,10 +429,7 @@ async fn main() { // keys). The timer remains as a safety-net poller. let notify_handle = if notify_enabled { let watch_client = WatchNotifyClient::from_shared(Arc::clone(&kv_client)); - let trigger = sync_trigger - .as_ref() - .map(Arc::clone) - .expect("sync_trigger set when notify_enabled"); + let trigger = Arc::clone(&sync_trigger); let notify_stop = stop.clone(); Some(tokio::spawn(async move { let handler = NotifyHandler::new(watch_client, trigger); @@ -479,10 +471,14 @@ async fn main() { .expect("HTTP health server error"); }); - // Wait for shutdown signal. let rpc_server_stop = Arc::clone(&rpc_server); - shutdown_signal().await; - info!("received shutdown signal"); + tokio::select! { + () = shutdown_signal() => info!("received shutdown signal"), + writes = crowdb_diskdb::liveness::recovery::wait_required(&container) => { + error!(?writes, + "metadata write outcome is unknown; stopping for supervisor restart under a new fenced process session"); + } + } rpc_service.close_free_admission().await; rpc_server_stop.stop(); stop.notify_waiters(); diff --git a/app/crowdb-diskdb/src/model/disk/write_gate.rs b/app/crowdb-diskdb/src/model/disk/write_gate.rs index 2ff47876f..a892a916f 100644 --- a/app/crowdb-diskdb/src/model/disk/write_gate.rs +++ b/app/crowdb-diskdb/src/model/disk/write_gate.rs @@ -34,6 +34,11 @@ impl DiskWriteGate { self.0.load(Ordering::Acquire) & CLOSED != 0 } + /// A cancelled or unconfirmed persistence still needs authoritative recovery. + pub fn outcome_unknown(&self) -> bool { + self.0.load(Ordering::Acquire) & UNCERTAIN != 0 + } + /// An acknowledgement may only follow this exact closed, drained state. pub fn quiescent(&self) -> bool { self.0.load(Ordering::Acquire) == CLOSED diff --git a/app/crowdb-diskdb/src/model/disk_group.rs b/app/crowdb-diskdb/src/model/disk_group.rs index 53eaf94b5..5f629b18f 100644 --- a/app/crowdb-diskdb/src/model/disk_group.rs +++ b/app/crowdb-diskdb/src/model/disk_group.rs @@ -547,6 +547,7 @@ impl DdbDiskGroup { let mut busy_bytes = 0u64; let mut allocatable_capacity_bytes = 0u64; let mut allocatable_busy_bytes = 0u64; + let mut allocatable_disk_count = 0u32; let mut disk_usages: Vec = Vec::with_capacity(disks_guard.len()); let mut usage_known = true; for disk in disks_guard.iter() { @@ -556,11 +557,10 @@ impl DdbDiskGroup { && u.capacity_bytes > 0; capacity_bytes += u.capacity_bytes; busy_bytes += u.busy_bytes; - if membership - .allocating - .iter() - .any(|candidate| candidate.disk_id == disk.disk_id) - { + // Administrative closure or an unknown write can change a gate + // before the next membership refresh. Report current admission. + if disk.allocatable() { + allocatable_disk_count += 1; allocatable_capacity_bytes += u.capacity_bytes; allocatable_busy_bytes += u.busy_bytes; } @@ -568,8 +568,6 @@ impl DdbDiskGroup { } #[allow(clippy::cast_possible_truncation)] let disk_count = disks_guard.len() as u32; - #[allow(clippy::cast_possible_truncation)] - let allocatable_disk_count = self.membership.load().allocating.len() as u32; let free_bytes = capacity_bytes.saturating_sub(busy_bytes); let allocatable_free_bytes = allocatable_capacity_bytes.saturating_sub(allocatable_busy_bytes); DiskGroupUsage { diff --git a/app/crowdb-diskdb/src/service/diskdb_rpc_service/allocation_failure.rs b/app/crowdb-diskdb/src/service/diskdb_rpc_service/allocation_failure.rs new file mode 100644 index 000000000..9b949824c --- /dev/null +++ b/app/crowdb-diskdb/src/service/diskdb_rpc_service/allocation_failure.rs @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Distinguish unavailable disks from exhausted allocatable capacity. + +use super::{AllocateParams, FBDiskdbRetCode}; + +pub(super) fn describe(params: &AllocateParams) -> (FBDiskdbRetCode, String) { + let usage = params.dg.aggregate_usage(); + let unavailable = !params.dg.allocatable() || usage.allocatable_disk_count == 0; + let code = if unavailable { + FBDiskdbRetCode::Unavailable + } else { + FBDiskdbRetCode::NoSpace + }; + let states: Vec<_> = params + .dg + .disk_snapshot() + .iter() + .map(|disk| { + format!( + "{:?}: status={:?}, write_gate_closed={}, write_outcome_unknown={}", + disk.disk_id, + disk.effective_status(), + disk.write_gate.closed(), + disk.write_gate.outcome_unknown() + ) + }) + .collect(); + let message = + format!( + "DiskDB allocation {}: disk_group={}, group_status={:?}, requested_blocks={}, units_per_block={}, \ + allocatable_disks={}/{}, capacity_bytes={}, busy_bytes={}, free_bytes={}, \ + allocatable_free_bytes={}, disks=[{}]", + if unavailable { "unavailable" } else { "has no eligible space" }, + params.dg.disk_group_id, + params.dg.status(), + params.count, + params.unit_count, + usage.allocatable_disk_count, + usage.disk_count, + usage.capacity_bytes, + usage.busy_bytes, + usage.free_bytes, + usage.allocatable_free_bytes, + states.join("; ") + ); + (code, message) +} diff --git a/app/crowdb-diskdb/src/service/diskdb_rpc_service/mutations.rs b/app/crowdb-diskdb/src/service/diskdb_rpc_service/mutations.rs index 92058511c..bb9a51312 100644 --- a/app/crowdb-diskdb/src/service/diskdb_rpc_service/mutations.rs +++ b/app/crowdb-diskdb/src/service/diskdb_rpc_service/mutations.rs @@ -317,7 +317,10 @@ impl DiskdbRpcService { } Err(AllocError::NoSpace) => { metrics.allocate_errors_total.inc(); - metrics.allocate_no_space_errors.inc(); + let (code, message) = super::allocation_failure::describe(¶ms); + if code == FBDiskdbRetCode::NoSpace { + metrics.allocate_no_space_errors.inc(); + } let disks = params.dg.disk_snapshot(); let diagnostics: Vec<_> = disks .iter() @@ -348,15 +351,10 @@ impl DiskdbRpcService { requested_units_per_block = params.unit_count, allow_disk_reuse = params.allow_disk_reuse, ?diagnostics, - "diskdb allocation has no space after compaction fallback" - ); - let ctrl = build_allocate_response( - req_id, - create_nano, - FBDiskdbRetCode::NoSpace, - Some("no space available"), - &[], + %message, + "diskdb allocation failed after compaction fallback" ); + let ctrl = build_allocate_response(req_id, create_nano, code, Some(&message), &[]); submit_fb_response(&server, conn_handle, ctrl, msg_type, req_id); } Err(AllocError::Persistence) => { diff --git a/app/crowdb-diskdb/src/service/diskdb_rpc_service/service.rs b/app/crowdb-diskdb/src/service/diskdb_rpc_service/service.rs index 1f5e67197..a32203fad 100644 --- a/app/crowdb-diskdb/src/service/diskdb_rpc_service/service.rs +++ b/app/crowdb-diskdb/src/service/diskdb_rpc_service/service.rs @@ -267,6 +267,8 @@ impl DiskdbRpcService { #[path = "admin.rs"] mod admin; +#[path = "allocation_failure.rs"] +mod allocation_failure; #[path = "mutations.rs"] mod mutations; #[path = "queries.rs"] diff --git a/app/crowdb-diskdb/tests/disk_initialization_test.rs b/app/crowdb-diskdb/tests/disk_initialization_test.rs index 86c5a06ce..4dc6e9a13 100644 --- a/app/crowdb-diskdb/tests/disk_initialization_test.rs +++ b/app/crowdb-diskdb/tests/disk_initialization_test.rs @@ -21,6 +21,7 @@ async fn provisional_init_waits_for_activation_and_only_then_loads_allocatable_z let id = DiskId { high: 4, low: 5 }; let hw = prepare_hardware(&metadata, HwStatus::Init).await; let container = Arc::new(DdbDiskGroupContainer::new(999)); + let completion = Arc::new(tokio::sync::Notify::new()); let keepalive = KeepAlive::new( metadata.make_hardware_client(), metadata.make_service_registry_client(), @@ -33,7 +34,8 @@ async fn provisional_init_waits_for_activation_and_only_then_loads_allocatable_z temp_failure_timeout_secs: 900, }, ) - .with_ddb_kv_client(metadata.make_ddb_kv_client()); + .with_ddb_kv_client(metadata.make_ddb_kv_client()) + .with_sync_trigger(completion.clone()); for _ in 0..2 { let outcome = keepalive.tick().await; assert_eq!(outcome.disks_added, 0); @@ -48,6 +50,26 @@ async fn provisional_init_waits_for_activation_and_only_then_loads_allocatable_z wait_for_disks_ready(&container, 3, 1, 1).await; let group = container.get_disk_group(3).unwrap(); assert!(group.disk_snapshot()[0].allocatable()); + tokio::time::timeout(Duration::from_millis(100), completion.notified()) + .await + .expect("completed zone loading must wake publication without another heartbeat interval"); + keepalive.tick().await; + let instances = metadata + .make_service_registry_client() + .read_all_diskdb_instances() + .await + .unwrap(); + let summary = &instances[0] + .1 + .extra + .as_ref() + .unwrap() + .diskdb + .as_ref() + .unwrap() + .group_usages[0]; + assert!(summary.usage_known); + assert_eq!(summary.allocatable_disk_count, 1); } #[test] diff --git a/app/crowdb-diskdb/tests/health_test.rs b/app/crowdb-diskdb/tests/health_test.rs index b14d0ca8a..1d09a7e9c 100644 --- a/app/crowdb-diskdb/tests/health_test.rs +++ b/app/crowdb-diskdb/tests/health_test.rs @@ -13,6 +13,73 @@ use crowdb_diskdb::health; use crowdb_diskdb::liveness::lifecycle::StartupPhase; use crowdb_diskdb::model::disk_group_container::DdbDiskGroupContainer; +#[tokio::test] +async fn uncertain_metadata_write_closes_readiness_and_requests_fenced_process_recovery() { + use crowdb_diskdb::{ + liveness::recovery, + model::{disk::DdbDisk, disk_group::DdbDiskGroup}, + }; + use crowdb_protocol::{common::DiskId, diskdb::rpc::DiskValue}; + + let container = Arc::new(DdbDiskGroupContainer::new(1)); + container.set_lifecycle_phase(StartupPhase::Up); + let group = Arc::new(DdbDiskGroup::new(7, 1, 1)); + let disk = Arc::new(DdbDisk::new( + DiskId { high: 2, low: 3 }, + 7, + 1, + 1, + DiskValue::default(), + )); + group.add_disk(disk.clone()); + container.replace_disk_group(&group); + drop(disk.write_gate.enter().unwrap()); + + let response = health::router(container.clone()) + .oneshot( + Request::builder() + .uri("/ready") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + let body = body_string(response).await; + assert!(body.contains("metadata-write-outcome-unknown"), "{body}"); + assert!(body.contains("\"disk_group_id\":7"), "{body}"); + assert!( + !container.is_degraded(), + "the health read must not initiate recovery" + ); + let writes = recovery::wait_required(&container).await; + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].disk_id, disk.disk_id); + assert!(container.is_degraded()); + assert!(disk.write_gate.closed()); + assert!( + !disk.write_gate.quiescent(), + "unknown writes cannot acknowledge disk removal" + ); +} + +#[test] +fn administrative_exclusion_does_not_request_process_recovery() { + use crowdb_diskdb::{ + liveness::recovery, + model::{disk::DdbDisk, disk_group::DdbDiskGroup}, + }; + use crowdb_protocol::{common::DiskId, diskdb::rpc::DiskValue}; + let container = DdbDiskGroupContainer::new(1); + let group = Arc::new(DdbDiskGroup::new(7, 1, 1)); + let disk = Arc::new(DdbDisk::new(DiskId::default(), 7, 1, 1, DiskValue::default())); + group.add_disk(disk.clone()); + container.replace_disk_group(&group); + disk.exclude(); + assert!(recovery::uncertain_writes(&container).is_empty()); + assert!(disk.write_gate.closed() && disk.write_gate.quiescent()); +} + async fn body_string(resp: axum::response::Response) -> String { let bytes = to_bytes(resp.into_body(), 1 << 20).await.expect("body"); String::from_utf8(bytes.to_vec()).expect("utf8") diff --git a/app/crowdb-diskdb/tests/query_handler_test.rs b/app/crowdb-diskdb/tests/query_handler_test.rs index 840c1ce49..54ec58f49 100644 --- a/app/crowdb-diskdb/tests/query_handler_test.rs +++ b/app/crowdb-diskdb/tests/query_handler_test.rs @@ -206,7 +206,7 @@ async fn disk_rpc_reports_effective_status() { kv.clone(), StorageDefaults::default(), Arc::new(ZoneLoader::new(kv.clone(), 4)), - Arc::new(RecalcEngine::new(kv, container)), + Arc::new(RecalcEngine::new(kv, container.clone())), ScanState::new(), Arc::new(DiskdbMetrics::disabled()), Arc::new(ArcSwap::from_pointee(DdbConfig::default())), @@ -247,5 +247,40 @@ async fn disk_rpc_reports_effective_status() { status as i32 ); } + container.set_lifecycle_phase(crowdb_diskdb::liveness::lifecycle::StartupPhase::Up); + for member in dg.disk_snapshot().iter() { + if member.disk_id == disk.disk_id { + drop(member.write_gate.enter().unwrap()); + } else { + member.exclude(); + } + } + let request = crowdb_protocol::diskdb::rpc::AllocateBlocksRequest { + disk_group_id: DG, + unit_count: 1, + count: 1, + owner_chunk: Some(crowdb_protocol::common::ChunkId { high: 7, low: 8 }), + ..Default::default() + }; + let error = transport.allocate_blocks(&endpoint, &request).await.unwrap_err(); + assert!( + matches!(error, crowdb_diskdb_client::DiskdbClientError::Unreachable(_)), + "{error}" + ); + let detail = error.to_string(); + assert!(detail.contains("allocatable_disks=0/2"), "{detail}"); + assert!(detail.contains("write_outcome_unknown=true"), "{detail}"); + assert!(detail.contains("free_bytes=1048576"), "{detail}"); + + let full = make_dg(); + full.allocate_blocks_reusing_disks(1, ZONE_CAP * ZONE_COUNT * 2, &[], 100, ZONE_ROTATE) + .unwrap(); + container.replace_disk_group(&full); + let error = transport.allocate_blocks(&endpoint, &request).await.unwrap_err(); + assert!( + matches!(error, crowdb_diskdb_client::DiskdbClientError::NoSpace(_)), + "{error}" + ); + assert!(error.to_string().contains("free_bytes=0"), "{error}"); server.stop(); } diff --git a/app/crowdb-diskio/src/dio_config.cpp b/app/crowdb-diskio/src/dio_config.cpp index 247af4c24..d81196471 100644 --- a/app/crowdb-diskio/src/dio_config.cpp +++ b/app/crowdb-diskio/src/dio_config.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include namespace crowdb::diskio { @@ -56,7 +58,7 @@ static bool parse_u32(const char *s, uint32_t &out) { char *end = nullptr; long v = std::strtol(s, &end, 10); - if (end == s || *end != '\0' || v < 0 || v > 0xFFFFFFFF) { + if (end == s || *end != '\0' || v < 0 || std::cmp_greater(v, std::numeric_limits::max())) { return false; } out = static_cast(v); @@ -175,7 +177,8 @@ bool DioConfig::parse_args(int argc, char *argv[], DioConfig &out, std::string & err = "--fault-latency expects :"; return false; } - uint32_t lo, hi; + uint32_t lo; + uint32_t hi; if (!parse_u32(spec.substr(0, colon).c_str(), lo) || !parse_u32(spec.substr(colon + 1).c_str(), hi)) { err = "invalid --fault-latency value"; return false; @@ -371,6 +374,11 @@ bool DioConfig::validate(std::string &err) const err = "max_write_request_age_ms must be > 0"; return false; } + if (max_write_request_age_ms > protocol::kMaxWriteRequestAgeMs || + max_clock_skew_ms > protocol::kMaxWriteClockSkewMs) { + err = "write request age and clock skew must not exceed the shared recovery bounds"; + return false; + } for (const auto &d : disks) { if (d.id.is_zero()) { err = "disk id is zero"; diff --git a/app/crowdb-diskio/src/dio_config.h b/app/crowdb-diskio/src/dio_config.h index 25808aa57..d582a9907 100644 --- a/app/crowdb-diskio/src/dio_config.h +++ b/app/crowdb-diskio/src/dio_config.h @@ -53,8 +53,8 @@ struct DioConfig // Shared stale-write exclusion budget. DiskIO rejects a write whose // wall-clock creation time is older than this age plus peer clock skew. - uint64_t max_write_request_age_ms = protocol::kDefaultMaxWriteRequestAgeMs; - uint64_t max_clock_skew_ms = protocol::kDefaultMaxClockSkewMs; + uint64_t max_write_request_age_ms = protocol::kMaxWriteRequestAgeMs; + uint64_t max_clock_skew_ms = protocol::kMaxWriteClockSkewMs; // Optional fault injection for dummy disks. std::optional dummy_props; diff --git a/app/crowdb-diskio/src/disk/types.h b/app/crowdb-diskio/src/disk/types.h index 32a82d3e1..4f95ea33e 100644 --- a/app/crowdb-diskio/src/disk/types.h +++ b/app/crowdb-diskio/src/disk/types.h @@ -70,6 +70,7 @@ enum class IoRetCode : int16_t { InvalidAlignment = 5, ConnectionError = 6, OldRequest = 8, + FsyncFailed = 9, }; } // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/engine/aligned_writer.cpp b/app/crowdb-diskio/src/engine/aligned_writer.cpp index 9ed36e424..16293abf9 100644 --- a/app/crowdb-diskio/src/engine/aligned_writer.cpp +++ b/app/crowdb-diskio/src/engine/aligned_writer.cpp @@ -3,7 +3,6 @@ #include "engine/aligned_writer.h" -#include "crowdb-common/log.h" #include "disk/disk.h" #include @@ -12,14 +11,13 @@ #include #include #include +#include namespace crowdb::diskio { namespace { -alignas(4096) const std::array ZERO_BYTES{}; - bool is_power_of_two(size_t value) { return value != 0 && (value & (value - 1)) == 0; @@ -31,7 +29,7 @@ std::shared_ptr allocate_aligned(size_t alignment, size_t size) if (::posix_memalign(&memory, alignment, size) != 0) { return {}; } - return std::shared_ptr(static_cast(memory), [](uint8_t *ptr) { ::free(ptr); }); + return {static_cast(memory), [](uint8_t *ptr) { ::free(ptr); }}; } } // namespace @@ -73,7 +71,8 @@ size_t AlignedWriter::CacheKeyHash::operator()(const CacheKey &key) const size_t AlignedWriter::shard_index(const Request &request) const { - return CacheKeyHash{}(CacheKey{request.disk->id(), request.ordering_phys_offset}) % SHARD_COUNT; + return CacheKeyHash{}(CacheKey{.disk_id = request.disk->id(), .block_offset = request.ordering_phys_offset}) % + SHARD_COUNT; } void AlignedWriter::submit(std::shared_ptr disk, off_t phys_offset, const uint8_t *data, size_t size, @@ -84,7 +83,8 @@ void AlignedWriter::submit(std::shared_ptr disk, off_t phys_offset, const } void AlignedWriter::submit_ordered(std::shared_ptr disk, off_t phys_offset, const uint8_t *data, size_t size, - uint64_t ordering_phys_offset, std::function on_complete) + uint64_t ordering_phys_offset, std::function on_complete, + std::chrono::steady_clock::time_point deadline) { if (stopping_.load(std::memory_order_acquire)) { if (on_complete) { @@ -98,19 +98,14 @@ void AlignedWriter::submit_ordered(std::shared_ptr disk, off_t phys_offset } return; } - if (disk->block_size() == 1 || size == 0) { - Disk *disk_ptr = disk.get(); - IoEngine *engine = disk_ptr->engine(); - engine->submit_write(disk_ptr, phys_offset, data, size, - [disk = std::move(disk), callback = std::move(on_complete)](int result) mutable { - if (callback) { - callback(result); - } - }); - return; - } - auto *request = new Request{std::move(disk), phys_offset, data, size, ordering_phys_offset, std::move(on_complete)}; - size_t index = shard_index(*request); + auto *request = new Request{.disk = std::move(disk), + .phys_offset = phys_offset, + .data = data, + .size = size, + .ordering_phys_offset = ordering_phys_offset, + .deadline = deadline, + .on_complete = std::move(on_complete)}; + size_t index = shard_index(*request); if (!shards_[index]->pending.try_push(request)) { if (request->on_complete) { request->on_complete(-EAGAIN); @@ -154,17 +149,25 @@ void AlignedWriter::process_next(size_t index) void AlignedWriter::process(size_t index, Request *request) { + if (std::chrono::steady_clock::now() >= request->deadline) { + finish(index, request, -ETIMEDOUT); + return; + } size_t block_size = request->disk->block_size(); if (block_size == 0) { finish(index, request, -EINVAL); return; } - if (block_size == 1 || request->size == 0) { + if (request->size == 0) { + finish(index, request, 0); + return; + } + if (block_size == 1) { request->disk->engine()->submit_write(request->disk.get(), request->phys_offset, request->data, request->size, [this, index, request](int result) { finish(index, request, result); }); return; } - if (!is_power_of_two(block_size) || block_size < sizeof(void *) || block_size > MAX_ZERO_BYTES) { + if (!is_power_of_two(block_size) || block_size < sizeof(void *) || block_size > MAX_BLOCK_BYTES) { finish(index, request, -EINVAL); return; } @@ -184,7 +187,7 @@ void AlignedWriter::process(size_t index, Request *request) if (offset_in_block == 0 && request->size % block_size == 0 && pointer_aligned) { request->disk->engine()->submit_write(request->disk.get(), request->phys_offset, request->data, request->size, [this, index, request, block_size](int result) { - if (result == static_cast(request->size)) { + if (std::cmp_equal(result, request->size)) { invalidate_cache(*shards_[index], *request, request->phys_offset, request->size, block_size); } @@ -198,69 +201,11 @@ void AlignedWriter::process(size_t index, Request *request) finish(index, request, -ENOMEM); return; } - uint8_t *bytes = buffer.get(); - for (size_t offset = 0; offset < aligned_size; offset += block_size) { - std::memcpy(bytes + offset, ZERO_BYTES.data(), block_size); - } - - auto &cache = shards_[index]->partial_blocks; - CacheKey head_key{request->disk->id(), static_cast(aligned_offset)}; - auto head = cache.find(head_key); - if (head != cache.end() && head->second.disk.lock().get() != request->disk.get()) { - cache.erase(head); - head = cache.end(); - } - if (head != cache.end()) { - std::memcpy(bytes, head->second.bytes.data(), block_size); - } - if (aligned_size > block_size) { - CacheKey tail_key{request->disk->id(), static_cast(aligned_offset) + aligned_size - block_size}; - auto tail = cache.find(tail_key); - if (tail != cache.end()) { - if (tail != cache.end() && tail->second.disk.lock().get() != request->disk.get()) { - cache.erase(tail); - tail = cache.end(); - } - std::memcpy(bytes + aligned_size - block_size, tail->second.bytes.data(), block_size); - } - } - std::memcpy(bytes + offset_in_block, request->data, request->size); - - if (offset_in_block == 0 || head != cache.end()) { - submit_buffer(index, request, std::move(buffer), aligned_offset, aligned_size, block_size); - return; - } - - CRB_LOG_WARN("unaligned disk write cache miss; recovering block: disk_high={} disk_low={} offset={} block_size={}", - request->disk->id().high, request->disk->id().low, request->phys_offset, block_size); - request->disk->engine()->submit_read( - request->disk.get(), aligned_offset, bytes, block_size, static_cast(aligned_offset), - [this, index, request, buffer = std::move(buffer), aligned_offset, aligned_size, block_size](int result) { - if (result != static_cast(block_size)) { - finish(index, request, result < 0 ? result : -EIO); - return; - } - std::memcpy(buffer.get() + (request->phys_offset - aligned_offset), request->data, request->size); - submit_buffer(index, request, buffer, aligned_offset, aligned_size, block_size); - }); -} - -void AlignedWriter::submit_buffer(size_t index, Request *request, std::shared_ptr buffer, off_t aligned_offset, - size_t aligned_size, size_t block_size) -{ - uint8_t *bytes = buffer.get(); - request->disk->engine()->submit_write( - request->disk.get(), aligned_offset, bytes, aligned_size, - [this, index, request, buffer = std::move(buffer), aligned_offset, aligned_size, block_size](int result) { - auto &shard = *shards_[index]; - if (result == static_cast(aligned_size)) { - update_cache(shard, *request, buffer.get(), aligned_offset, aligned_size, block_size); - finish(index, request, static_cast(request->size)); - return; - } - invalidate_cache(shard, *request, aligned_offset, aligned_size, block_size); - finish(index, request, result < 0 ? result : -EIO); - }); + merge_boundaries( + index, request, + MergeBuffer{ + .bytes = std::move(buffer), .offset = aligned_offset, .size = aligned_size, .block_size = block_size}, + 0); } void AlignedWriter::update_cache(Shard &shard, const Request &request, const uint8_t *buffer, off_t aligned_offset, @@ -275,17 +220,17 @@ void AlignedWriter::update_cache(Shard &shard, const Request &request, const uin shard.partial_blocks.erase(shard.partial_blocks.begin()); } uint64_t tail_offset = static_cast(aligned_offset) + aligned_size - block_size; - shard.partial_blocks[CacheKey{request.disk->id(), tail_offset}] = - CachedBlock{request.disk, std::vector(buffer + aligned_size - block_size, buffer + aligned_size)}; + shard.partial_blocks[CacheKey{.disk_id = request.disk->id(), .block_offset = tail_offset}] = CachedBlock{ + .disk = request.disk, .bytes = std::vector(buffer + aligned_size - block_size, buffer + aligned_size)}; } void AlignedWriter::invalidate_cache(Shard &shard, const Request &request, off_t aligned_offset, size_t aligned_size, size_t block_size) { - uint64_t block_offset = static_cast(aligned_offset); + auto block_offset = static_cast(aligned_offset); uint64_t end = block_offset + aligned_size; while (block_offset < end) { - shard.partial_blocks.erase(CacheKey{request.disk->id(), block_offset}); + shard.partial_blocks.erase(CacheKey{.disk_id = request.disk->id(), .block_offset = block_offset}); block_offset += block_size; } } diff --git a/app/crowdb-diskio/src/engine/aligned_writer.h b/app/crowdb-diskio/src/engine/aligned_writer.h index 31977061c..33be3c1e0 100644 --- a/app/crowdb-diskio/src/engine/aligned_writer.h +++ b/app/crowdb-diskio/src/engine/aligned_writer.h @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -24,7 +25,7 @@ class Disk; class AlignedWriter { public: - static constexpr size_t MAX_ZERO_BYTES = 2 * 1024 * 1024; + static constexpr size_t MAX_BLOCK_BYTES = 2 * 1024 * 1024; explicit AlignedWriter(); ~AlignedWriter(); @@ -35,18 +36,28 @@ class AlignedWriter void submit(std::shared_ptr disk, off_t phys_offset, const uint8_t *data, size_t size, std::function on_complete); void submit_ordered(std::shared_ptr disk, off_t phys_offset, const uint8_t *data, size_t size, - uint64_t ordering_phys_offset, std::function on_complete); + uint64_t ordering_phys_offset, std::function on_complete, + std::chrono::steady_clock::time_point deadline = std::chrono::steady_clock::time_point::max()); void stop(); private: struct Request { - std::shared_ptr disk; - off_t phys_offset; - const uint8_t *data; + std::shared_ptr disk; + off_t phys_offset; + const uint8_t *data; + size_t size; + uint64_t ordering_phys_offset; + std::chrono::steady_clock::time_point deadline; + std::function on_complete; + }; + + struct MergeBuffer + { + std::shared_ptr bytes; + off_t offset; size_t size; - uint64_t ordering_phys_offset; - std::function on_complete; + size_t block_size; }; struct CacheKey @@ -85,17 +96,17 @@ class AlignedWriter std::array, SHARD_COUNT> shards_; std::atomic stopping_{false}; - size_t shard_index(const Request &request) const; - void try_start(size_t shard_index); - void process_next(size_t shard_index); - void process(size_t shard_index, Request *request); - void submit_buffer(size_t shard_index, Request *request, std::shared_ptr buffer, off_t aligned_offset, - size_t aligned_size, size_t block_size); - void finish(size_t shard_index, Request *request, int result); - void update_cache(Shard &shard, const Request &request, const uint8_t *buffer, off_t aligned_offset, - size_t aligned_size, size_t block_size); - void invalidate_cache(Shard &shard, const Request &request, off_t aligned_offset, size_t aligned_size, - size_t block_size); + [[nodiscard]] size_t shard_index(const Request &request) const; + void try_start(size_t shard_index); + void process_next(size_t shard_index); + void process(size_t shard_index, Request *request); + void merge_boundaries(size_t shard_index, Request *request, MergeBuffer buffer, size_t boundary); + void submit_buffer(size_t shard_index, Request *request, MergeBuffer buffer); + void finish(size_t shard_index, Request *request, int result); + void update_cache(Shard &shard, const Request &request, const uint8_t *buffer, off_t aligned_offset, + size_t aligned_size, size_t block_size); + void invalidate_cache(Shard &shard, const Request &request, off_t aligned_offset, size_t aligned_size, + size_t block_size); }; } // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/engine/aligned_writer_merge.cpp b/app/crowdb-diskio/src/engine/aligned_writer_merge.cpp new file mode 100644 index 000000000..c0e38599a --- /dev/null +++ b/app/crowdb-diskio/src/engine/aligned_writer_merge.cpp @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "disk/disk.h" +#include "engine/aligned_writer.h" + +#include +#include +#include + +namespace crowdb::diskio +{ + +void AlignedWriter::merge_boundaries(size_t index, Request *request, MergeBuffer buffer, size_t boundary) +{ + const bool partial_head = request->phys_offset != buffer.offset; + const bool partial_tail = static_cast(request->phys_offset) + request->size != + static_cast(buffer.offset) + buffer.size; + for (; boundary < 2; ++boundary) { + const bool needed = boundary == 0 ? partial_head || (partial_tail && buffer.size == buffer.block_size) + : partial_tail && buffer.size > buffer.block_size; + if (!needed) { + continue; + } + const size_t offset = boundary == 0 ? 0 : buffer.size - buffer.block_size; + const auto position = buffer.offset + static_cast(offset); + auto &cache = shards_[index]->partial_blocks; + auto cached = + cache.find(CacheKey{.disk_id = request->disk->id(), .block_offset = static_cast(position)}); + if (cached != cache.end() && cached->second.disk.lock().get() != request->disk.get()) { + cache.erase(cached); + cached = cache.end(); + } + if (cached != cache.end()) { + std::memcpy(buffer.bytes.get() + offset, cached->second.bytes.data(), buffer.block_size); + continue; + } + auto *destination = buffer.bytes.get() + offset; + const size_t block_size = buffer.block_size; + request->disk->engine()->submit_read( + request->disk.get(), position, destination, block_size, static_cast(position), + [this, index, request, buffer = std::move(buffer), boundary, block_size](int result) mutable { + if (std::cmp_not_equal(result, block_size)) { + finish(index, request, result < 0 ? result : -EIO); + return; + } + merge_boundaries(index, request, std::move(buffer), boundary + 1); + }); + return; + } + std::memcpy(buffer.bytes.get() + (request->phys_offset - buffer.offset), request->data, request->size); + submit_buffer(index, request, std::move(buffer)); +} + +void AlignedWriter::submit_buffer(size_t index, Request *request, MergeBuffer buffer) +{ + if (std::chrono::steady_clock::now() >= request->deadline) { + finish(index, request, -ETIMEDOUT); + return; + } + auto *bytes = buffer.bytes.get(); + const off_t offset = buffer.offset; + const size_t size = buffer.size; + request->disk->engine()->submit_write( + request->disk.get(), offset, bytes, size, [this, index, request, buffer = std::move(buffer)](int result) { + auto &shard = *shards_[index]; + if (std::cmp_equal(result, buffer.size)) { + update_cache(shard, *request, buffer.bytes.get(), buffer.offset, buffer.size, buffer.block_size); + finish(index, request, static_cast(request->size)); + return; + } + invalidate_cache(shard, *request, buffer.offset, buffer.size, buffer.block_size); + finish(index, request, result < 0 ? result : -EIO); + }); +} + +} // namespace crowdb::diskio diff --git a/app/crowdb-diskio/src/rpc/dio_server.cpp b/app/crowdb-diskio/src/rpc/dio_server.cpp index 230c87825..830d43bdd 100644 --- a/app/crowdb-diskio/src/rpc/dio_server.cpp +++ b/app/crowdb-diskio/src/rpc/dio_server.cpp @@ -56,16 +56,9 @@ uint64_t elapsed_nanos(std::chrono::steady_clock::time_point started) std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); } -uint64_t unix_time_ms() -{ - return static_cast( - std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) - .count()); -} - } // namespace -DiskioServer::DiskioServer(std::shared_ptr disk_set, crowdb::rpc::SocketTransport *transport, +DiskioServer::DiskioServer(std::shared_ptr disk_set, crowdb::rpc::Transport *transport, uint64_t max_write_request_age_ms, uint64_t max_clock_skew_ms) : disk_set_(std::move(disk_set)), transport_(transport), @@ -80,12 +73,13 @@ DiskioServer::DiskioServer(std::shared_ptr disk_set, crowdb::rpc::Socke // Build a diskio response control buffer (flatbuffer). crowdb::rpc::Buffer *DiskioServer::build_response_ctrl(crowdb::rpc::BufferPool *pool, uint64_t request_id, - uint64_t rpc_create_nano, int16_t ret_code, uint16_t msg_type) + uint64_t rpc_create_nano, int16_t ret_code, uint16_t msg_type, + bool sync_completed) { auto fb_ret = static_cast(ret_code); flatbuffers::FlatBufferBuilder fbb(64); if (msg_type == static_cast(rproto::FBMsgType_EDiskWriteResponse)) { - auto off = dproto::CreateFBDiskWriteResponse(fbb, request_id, rpc_create_nano, fb_ret); + auto off = dproto::CreateFBDiskWriteResponse(fbb, request_id, rpc_create_nano, fb_ret, sync_completed); fbb.Finish(off); } else if (msg_type == static_cast(rproto::FBMsgType_EDiskReadResponse)) { @@ -124,89 +118,6 @@ static DiskId parse_disk_id(const crowdb::rpc::proto::FBInt128 *fb_id) return {fb_id->high(), fb_id->low()}; } -crowdb::rpc::OutFrame *DiskioServer::handle_write(crowdb::rpc::Frame *request, crowdb::rpc::Connection *conn) -{ - uint64_t req_id = request->request_id; - uint64_t create_nano = request->rpc_create_nano; - uint16_t msg_type = static_cast(rproto::FBMsgType_EDiskWriteResponse); - - auto *fb_req = ::flatbuffers::GetRoot(request->control.data()); - if (fb_req == nullptr || request->control.size() < 4) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); - return nullptr; - } - DiskId did = parse_disk_id(fb_req->disk_id()); - uint32_t zone_index = fb_req->zone_index(); - uint64_t zone_offset = fb_req->zone_offset(); - uint32_t size = fb_req->size(); - uint64_t ordering_zone_offset = fb_req->ordering_zone_offset(); - uint64_t write_create_time_ms = fb_req->write_create_time_ms(); - - const uint64_t allowance = max_write_request_age_ms_ > UINT64_MAX - max_clock_skew_ms_ - ? UINT64_MAX - : max_write_request_age_ms_ + max_clock_skew_ms_; - const uint64_t now_ms = unix_time_ms(); - if (write_create_time_ms == 0 || (now_ms >= write_create_time_ms && now_ms - write_create_time_ms > allowance)) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, - static_cast(dproto::FBDiskIoRetCode_OldRequest)); - return nullptr; - } - - auto disk = disk_set_->find_disk(did); - if (disk == nullptr) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, - static_cast(dproto::FBDiskIoRetCode_DiskNotExist)); - return nullptr; - } - - Zone *zone = disk->find_zone(zone_index); - if (zone == nullptr) { - delete request; - send_error_response(conn, req_id, create_nano, msg_type, - static_cast(dproto::FBDiskIoRetCode_ZoneNotExist)); - return nullptr; - } - off_t phys_offset = static_cast(zone->base_offset + zone_offset); - - crowdb::rpc::Buffer *data_buf = nullptr; - if (request->data_buf != nullptr && size > 0) { - data_buf = request->data_buf->ref_clone(); - } - delete request; - - if (data_buf == nullptr && size > 0) { - send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); - return nullptr; - } - uint64_t ordering_phys_offset = zone->base_offset + ordering_zone_offset; - auto started = std::chrono::steady_clock::now(); - aligned_writer_.submit_ordered(disk, phys_offset, data_buf ? data_buf->data : nullptr, size, ordering_phys_offset, - [this, conn, req_id, create_nano, msg_type, data_buf, size, started](int res) { - write_latency_->observe(elapsed_nanos(started)); - int16_t ret_code = static_cast(dproto::FBDiskIoRetCode_Success); - if (res < 0) { - ret_code = static_cast(dproto::FBDiskIoRetCode_IoError); - } - else if (static_cast(res) < size) { - ret_code = static_cast(dproto::FBDiskIoRetCode_PartialWrite); - } - auto *pool = conn->pool(); - auto *ctrl = build_response_ctrl(pool, req_id, create_nano, ret_code, msg_type); - auto *out = crowdb::rpc::build_out_frame(req_id, msg_type, ctrl, nullptr); - out->create_nano = static_cast( - std::chrono::steady_clock::now().time_since_epoch().count()); - transport_->submit(conn, out); - if (data_buf != nullptr) { - data_buf->release(); - } - }); - - return nullptr; -} - crowdb::rpc::OutFrame *DiskioServer::handle_read(crowdb::rpc::Frame *request, crowdb::rpc::Connection *conn) { uint64_t req_id = request->request_id; diff --git a/app/crowdb-diskio/src/rpc/dio_server.h b/app/crowdb-diskio/src/rpc/dio_server.h index de64c877d..5eaedb7e0 100644 --- a/app/crowdb-diskio/src/rpc/dio_server.h +++ b/app/crowdb-diskio/src/rpc/dio_server.h @@ -39,9 +39,9 @@ namespace crowdb::diskio class DiskioServer { public: - DiskioServer(std::shared_ptr disk_set, crowdb::rpc::SocketTransport *transport, - uint64_t max_write_request_age_ms = protocol::kDefaultMaxWriteRequestAgeMs, - uint64_t max_clock_skew_ms = protocol::kDefaultMaxClockSkewMs); + DiskioServer(std::shared_ptr disk_set, crowdb::rpc::Transport *transport, + uint64_t max_write_request_age_ms = protocol::kMaxWriteRequestAgeMs, + uint64_t max_clock_skew_ms = protocol::kMaxWriteClockSkewMs); // Handler functions (registered with RpcServer::register_handler). // Each parses the flatbuffer control from the Frame, looks up the @@ -56,7 +56,7 @@ class DiskioServer private: std::shared_ptr disk_set_; - crowdb::rpc::SocketTransport *transport_; + crowdb::rpc::Transport *transport_; AlignedWriter aligned_writer_; crowdb::common::metrics::LatencyHistogram *read_latency_; crowdb::common::metrics::LatencyHistogram *write_latency_; @@ -68,7 +68,8 @@ class DiskioServer // ret_code is a proto::FBDiskIoRetCode value (int16_t to avoid // pulling the generated header into this .h). crowdb::rpc::Buffer *build_response_ctrl(crowdb::rpc::BufferPool *pool, uint64_t request_id, - uint64_t rpc_create_nano, int16_t ret_code, uint16_t msg_type); + uint64_t rpc_create_nano, int16_t ret_code, uint16_t msg_type, + bool sync_completed = false); // Send an error response synchronously (disk not found, parse error). void send_error_response(crowdb::rpc::Connection *conn, uint64_t request_id, uint64_t rpc_create_nano, diff --git a/app/crowdb-diskio/src/rpc/dio_server_write.cpp b/app/crowdb-diskio/src/rpc/dio_server_write.cpp new file mode 100644 index 000000000..01f135c4a --- /dev/null +++ b/app/crowdb-diskio/src/rpc/dio_server_write.cpp @@ -0,0 +1,179 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-common/metrics/metrics.h" +#include "crowdb-rpc/server/message.h" +#include "disk/disk.h" +#include "rpc/dio_server.h" + +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace crowdb::diskio +{ +namespace dproto = crowdb::diskio::proto; +namespace rproto = crowdb::rpc::proto; + +namespace +{ +uint64_t unix_time_ms() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count()); +} + +uint64_t elapsed_nanos(std::chrono::steady_clock::time_point started) +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()); +} + +std::optional write_deadline(uint64_t created_ms, uint64_t max_age_ms, + uint64_t skew_ms) +{ + const auto steady_now = std::chrono::steady_clock::now(); + const auto now_ms = unix_time_ms(); + if (created_ms == 0 || max_age_ms == 0 || (created_ms > now_ms && created_ms - now_ms > skew_ms) || + max_age_ms > UINT64_MAX - skew_ms) { + return std::nullopt; + } + const auto allowance = max_age_ms + skew_ms; + if (created_ms > UINT64_MAX - allowance || created_ms + allowance <= now_ms) { + return std::nullopt; + } + const auto remaining_ms = created_ms + allowance - now_ms; + const auto max_wait_ms = + std::chrono::duration_cast(std::chrono::steady_clock::time_point::max() - steady_now) + .count(); + if (std::cmp_greater(remaining_ms, max_wait_ms)) { + return std::nullopt; + } + return steady_now + std::chrono::milliseconds(static_cast(remaining_ms)); +} +} // namespace + +crowdb::rpc::OutFrame *DiskioServer::handle_write(crowdb::rpc::Frame *request, crowdb::rpc::Connection *conn) +{ + uint64_t req_id = request->request_id; + uint64_t create_nano = request->rpc_create_nano; + auto msg_type = static_cast(rproto::FBMsgType_EDiskWriteResponse); + + flatbuffers::Verifier verifier(request->control.data(), request->control.size()); + if (!verifier.VerifyBuffer(nullptr)) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); + return nullptr; + } + const auto *fb_req = ::flatbuffers::GetRoot(request->control.data()); + const auto *identity = fb_req->disk_id(); + DiskId did = identity != nullptr ? DiskId{.high = identity->high(), .low = identity->low()} : DiskId{}; + uint32_t zone_index = fb_req->zone_index(); + uint64_t zone_offset = fb_req->zone_offset(); + uint32_t size = fb_req->size(); + uint64_t ordering_zone_offset = fb_req->ordering_zone_offset(); + uint64_t write_create_time_ms = fb_req->write_create_time_ms(); + const bool sync = fb_req->sync(); + + const auto deadline = write_deadline(write_create_time_ms, max_write_request_age_ms_, max_clock_skew_ms_); + if (!deadline) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, + static_cast(dproto::FBDiskIoRetCode_OldRequest)); + return nullptr; + } + + auto disk = disk_set_->find_disk(did); + if (disk == nullptr) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, + static_cast(dproto::FBDiskIoRetCode_DiskNotExist)); + return nullptr; + } + + Zone *zone = disk->find_zone(zone_index); + if (zone == nullptr) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, + static_cast(dproto::FBDiskIoRetCode_ZoneNotExist)); + return nullptr; + } + if (zone->base_offset < 0 || zone->capacity < 0 || ordering_zone_offset > zone_offset || + std::cmp_greater(zone_offset, zone->capacity) || size > static_cast(zone->capacity) - zone_offset || + std::cmp_greater(zone_offset, std::numeric_limits::max() - zone->base_offset) || + (size > 0 && (request->data_buf == nullptr || request->data_buf->len != size))) { + delete request; + send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); + return nullptr; + } + off_t phys_offset = zone->base_offset + static_cast(zone_offset); + + crowdb::rpc::Buffer *data_buf = nullptr; + if (request->data_buf != nullptr && size > 0) { + data_buf = request->data_buf->ref_clone(); + } + delete request; + + if (data_buf == nullptr && size > 0) { + send_error_response(conn, req_id, create_nano, msg_type, static_cast(dproto::FBDiskIoRetCode_IoError)); + return nullptr; + } + auto data = std::shared_ptr(data_buf, [](crowdb::rpc::Buffer *buffer) { + if (buffer != nullptr) { + buffer->release(); + } + }); + auto complete = [this, connection = conn->shared_from_this(), data, req_id, create_nano, + msg_type](int16_t code, bool synchronized) { + auto *control = build_response_ctrl(connection->pool(), req_id, create_nano, code, msg_type, synchronized); + if (control == nullptr) { + return; + } + auto *out = crowdb::rpc::build_out_frame(req_id, msg_type, control, nullptr); + out->create_nano = static_cast(std::chrono::steady_clock::now().time_since_epoch().count()); + if (!transport_->submit(connection.get(), out)) { + control->release(); + delete out; + } + }; + uint64_t ordering_phys_offset = zone->base_offset + ordering_zone_offset; + auto started = std::chrono::steady_clock::now(); + aligned_writer_.submit_ordered( + disk, phys_offset, data_buf != nullptr ? data_buf->data : nullptr, size, ordering_phys_offset, + [this, disk, complete = std::move(complete), sync, size, started](int res) mutable { + write_latency_->observe(elapsed_nanos(started)); + if (res == -ETIMEDOUT) { + complete(static_cast(dproto::FBDiskIoRetCode_OldRequest), false); + } + else if (res < 0) { + complete(static_cast(dproto::FBDiskIoRetCode_IoError), false); + } + else if (std::cmp_less(res, size)) { + complete(static_cast(dproto::FBDiskIoRetCode_PartialWrite), false); + } + else if (!sync) { + complete(static_cast(dproto::FBDiskIoRetCode_Success), false); + } + else { + auto flush_started = std::chrono::steady_clock::now(); + disk->engine()->submit_fsync( + disk.get(), [this, disk, complete = std::move(complete), flush_started](int result) mutable { + fsync_latency_->observe(elapsed_nanos(flush_started)); + complete(static_cast(result < 0 ? dproto::FBDiskIoRetCode_FsyncFailed + : dproto::FBDiskIoRetCode_Success), + result >= 0); + }); + } + }, + *deadline); + return nullptr; +} + +} // namespace crowdb::diskio diff --git a/app/crowdb-diskio/tests/aligned_writer_order_test.cpp b/app/crowdb-diskio/tests/aligned_writer_order_test.cpp new file mode 100644 index 000000000..ec12985bc --- /dev/null +++ b/app/crowdb-diskio/tests/aligned_writer_order_test.cpp @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/aligned_writer.h" + +#include + +#include +#include +#include +#include +#include + +namespace +{ +using crowdb::diskio::AlignedWriter; +using crowdb::diskio::test::TestDisk; +using crowdb::diskio::test::TestRecordingEngine; + +TEST(AlignedWriter, ByteAddressedWritesKeepTheirSegmentCompletionOrder) +{ + TestRecordingEngine engine(4096, true); + auto disk = std::make_shared(&engine, 1); + AlignedWriter writer; + std::vector data(32, 7); + std::vector completed; + writer.submit_ordered(disk, 0, data.data(), data.size(), 0, [&](int) { completed.push_back(1); }); + writer.submit_ordered(disk, 32, data.data(), data.size(), 0, [&](int) { completed.push_back(2); }); + EXPECT_EQ(engine.pending_writes.size(), 1); + engine.complete_one_write(); + EXPECT_EQ(completed, (std::vector{1})); + while (!engine.pending_writes.empty()) { + engine.complete_one_write(); + } + EXPECT_EQ(completed, (std::vector{1, 2})); +} + +TEST(AlignedWriter, EmptyOrderedBarrierWaitsForWritesWithoutIssuingMediaIo) +{ + TestRecordingEngine engine(4096, true); + auto disk = std::make_shared(&engine, 1); + AlignedWriter writer; + std::vector data(32, 7); + std::vector completed; + writer.submit_ordered(disk, 0, data.data(), data.size(), 0, [&](int) { completed.push_back(1); }); + writer.submit_ordered(disk, 0, nullptr, 0, 0, [&](int result) { + EXPECT_EQ(result, 0); + completed.push_back(2); + }); + EXPECT_EQ(engine.pending_writes.size(), 1); + EXPECT_TRUE(completed.empty()); + engine.complete_one_write(); + while (!engine.pending_writes.empty()) { + engine.complete_one_write(); + } + EXPECT_EQ(completed, (std::vector{1, 2})); + EXPECT_EQ(engine.write_sizes, (std::vector{32})); +} + +TEST(AlignedWriter, ExpiredQueuedWritesCannotRunAfterThePriorCompletion) +{ + for (const size_t block_size : {size_t{1}, size_t{4096}}) { + TestRecordingEngine engine(8192, true); + auto disk = std::make_shared(&engine, block_size); + AlignedWriter writer; + std::vector data(32, 7); + std::vector completed; + writer.submit_ordered(disk, 0, data.data(), data.size(), 0, [&](int result) { completed.push_back(result); }); + writer.submit_ordered( + disk, 32, data.data(), data.size(), 0, [&](int result) { completed.push_back(result); }, + std::chrono::steady_clock::time_point::min()); + EXPECT_TRUE(completed.empty()); + EXPECT_EQ(engine.pending_writes.size(), 1); + engine.complete_one_write(); + EXPECT_EQ(completed, (std::vector{32, -ETIMEDOUT})); + EXPECT_TRUE(engine.pending_writes.empty()); + EXPECT_EQ(engine.write_sizes.size(), 1); + while (!engine.pending_writes.empty()) { + engine.complete_one_write(); + } + } +} + +TEST(AlignedWriter, ExpiredBoundaryReadCannotStartAMediaWrite) +{ + TestRecordingEngine engine(8192); + engine.defer_reads = true; + auto disk = std::make_shared(&engine, 4096); + AlignedWriter writer; + std::vector data(32, 7); + int completed = 0; + writer.submit_ordered( + disk, 0, data.data(), data.size(), 0, [&](int result) { completed = result; }, + std::chrono::steady_clock::now() + std::chrono::milliseconds(50)); + EXPECT_EQ(engine.pending_reads.size(), 1); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + while (!engine.pending_reads.empty()) { + engine.complete_one_read(); + } + EXPECT_EQ(completed, -ETIMEDOUT); + EXPECT_TRUE(engine.write_sizes.empty()); +} +} // namespace diff --git a/app/crowdb-diskio/tests/aligned_writer_range_test.cpp b/app/crowdb-diskio/tests/aligned_writer_range_test.cpp new file mode 100644 index 000000000..a2be7e3e7 --- /dev/null +++ b/app/crowdb-diskio/tests/aligned_writer_range_test.cpp @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/aligned_writer.h" + +#include + +#include +#include +#include +#include + +namespace +{ +using crowdb::diskio::AlignedWriter; +using crowdb::diskio::test::TestDisk; +using crowdb::diskio::test::TestRecordingEngine; + +TEST(AlignedWriter, DelayedDuplicatePreservesLaterBytesAfterCacheReset) +{ + TestRecordingEngine engine(8192); + auto disk = std::make_shared(&engine, 4096); + std::vector first(13, 0x11); + std::vector second(13, 0x22); + { + AlignedWriter writer; + writer.submit_ordered(disk, 0, first.data(), first.size(), 0, [](int result) { ASSERT_EQ(result, 13); }); + writer.submit_ordered(disk, 13, second.data(), second.size(), 0, [](int result) { ASSERT_EQ(result, 13); }); + } + AlignedWriter reopened; + reopened.submit_ordered(disk, 0, first.data(), first.size(), 0, [](int result) { ASSERT_EQ(result, 13); }); + EXPECT_TRUE(std::equal(first.begin(), first.end(), engine.storage.begin())); + EXPECT_TRUE(std::equal(second.begin(), second.end(), engine.storage.begin() + 13)); +} + +TEST(AlignedWriter, MissingHeadAndTailPreserveBothBoundaryBlocks) +{ + TestRecordingEngine engine(3 * 4096); + std::fill(engine.storage.begin(), engine.storage.end(), 0x7a); + auto disk = std::make_shared(&engine, 4096); + AlignedWriter writer; + std::vector data(8192, 0x33); + writer.submit_ordered(disk, 5, data.data(), data.size(), 0, [](int result) { ASSERT_EQ(result, 8192); }); + EXPECT_EQ(engine.read_count, 2); + EXPECT_TRUE( + std::all_of(engine.storage.begin(), engine.storage.begin() + 5, [](uint8_t value) { return value == 0x7a; })); + EXPECT_TRUE(std::equal(data.begin(), data.end(), engine.storage.begin() + 5)); + EXPECT_TRUE( + std::all_of(engine.storage.begin() + 8197, engine.storage.end(), [](uint8_t value) { return value == 0x7a; })); +} + +TEST(AlignedWriter, CachedHeadDoesNotAuthorizeZeroingAnUncachedTail) +{ + TestRecordingEngine engine(8192); + std::fill(engine.storage.begin(), engine.storage.end(), 0x7a); + auto disk = std::make_shared(&engine, 4096); + AlignedWriter writer; + std::vector first(13, 0x11); + std::vector second(4096, 0x22); + writer.submit_ordered(disk, 0, first.data(), first.size(), 0, [](int result) { ASSERT_EQ(result, 13); }); + writer.submit_ordered(disk, 13, second.data(), second.size(), 0, [](int result) { ASSERT_EQ(result, 4096); }); + EXPECT_EQ(engine.read_count, 2); + EXPECT_TRUE(std::equal(first.begin(), first.end(), engine.storage.begin())); + EXPECT_TRUE(std::equal(second.begin(), second.end(), engine.storage.begin() + 13)); + EXPECT_TRUE( + std::all_of(engine.storage.begin() + 4109, engine.storage.end(), [](uint8_t value) { return value == 0x7a; })); +} + +TEST(AlignedWriter, FailedOrShortBoundaryReadCannotWriteUnknownBytes) +{ + for (const int read_result : {-EIO, 2048}) { + TestRecordingEngine engine(3 * 4096); + std::fill(engine.storage.begin(), engine.storage.end(), 0x7a); + engine.failed_read_offset = 8192; + engine.failed_read_result = read_result; + const auto retained = engine.storage; + auto disk = std::make_shared(&engine, 4096); + AlignedWriter writer; + std::vector data(8192, 0x33); + int result = 0; + writer.submit_ordered(disk, 5, data.data(), data.size(), 0, [&](int completed) { result = completed; }); + EXPECT_EQ(result, -EIO); + EXPECT_EQ(engine.read_count, 2); + EXPECT_TRUE(engine.write_sizes.empty()); + EXPECT_EQ(engine.storage, retained); + } +} +} // namespace diff --git a/app/crowdb-diskio/tests/aligned_writer_test.cpp b/app/crowdb-diskio/tests/aligned_writer_test.cpp index f1da3a1e3..05be02403 100644 --- a/app/crowdb-diskio/tests/aligned_writer_test.cpp +++ b/app/crowdb-diskio/tests/aligned_writer_test.cpp @@ -1,8 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -#include "disk/disk.h" -#include "engine/aligned_writer.h" +#include "common/aligned_writer.h" #include #include @@ -23,116 +22,12 @@ namespace { -class RecordingEngine final : public crowdb::diskio::IoEngine -{ - public: - explicit RecordingEngine(size_t capacity, bool defer_writes = false) - : storage(capacity, 0), - defer_writes(defer_writes) - { - } - - void submit_write(crowdb::diskio::Disk *, off_t offset, const uint8_t *data, size_t size, - std::function on_complete) override - { - write_offsets.push_back(offset); - write_sizes.push_back(size); - if (offset < 0 || static_cast(offset) + size > storage.size()) { - on_complete(-1); - return; - } - std::memcpy(storage.data() + offset, data, size); - if (defer_writes) { - pending_writes.emplace_back(size, std::move(on_complete)); - } - else { - on_complete(static_cast(size)); - } - } - - void submit_read(crowdb::diskio::Disk *, off_t offset, uint8_t *buf, size_t size, uint64_t, - std::function on_complete) override - { - read_count++; - if (offset < 0 || static_cast(offset) + size > storage.size()) { - on_complete(-1); - return; - } - std::memcpy(buf, storage.data() + offset, size); - on_complete(static_cast(size)); - } - - void submit_fsync(crowdb::diskio::Disk *, std::function on_complete) override - { - on_complete(0); - } - - void complete_one_write() - { - ASSERT_FALSE(pending_writes.empty()); - auto [size, callback] = std::move(pending_writes.front()); - pending_writes.pop_front(); - callback(static_cast(size)); - } - - std::vector storage; - bool defer_writes; - size_t read_count{0}; - std::vector write_offsets; - std::vector write_sizes; - std::deque>> pending_writes; -}; - -class TestDisk final : public crowdb::diskio::Disk -{ - public: - TestDisk(RecordingEngine *engine, size_t block_size) : engine_(engine), block_size_(block_size) - { - } - - [[nodiscard]] crowdb::diskio::DiskType type() const override - { - return crowdb::diskio::DiskType::Mem; - } - - [[nodiscard]] int fd() const override - { - return 1; - } - - [[nodiscard]] bool is_o_direct() const override - { - return block_size_ > 1; - } - - [[nodiscard]] size_t block_size() const override - { - return block_size_; - } - - crowdb::diskio::IoEngine *engine() override - { - return engine_; - } - - [[nodiscard]] crowdb::diskio::DiskId id() const override - { - return {7, 9}; - } - - crowdb::diskio::Zone *find_zone(uint32_t) override - { - return nullptr; - } - - private: - RecordingEngine *engine_; - size_t block_size_; -}; +using crowdb::diskio::test::TestDisk; +using crowdb::diskio::test::TestRecordingEngine; TEST(AlignedWriter, BlockSizeOneUsesNoOpPath) { - RecordingEngine engine(4096); + TestRecordingEngine engine(4096); auto disk = std::make_shared(&engine, 1); crowdb::diskio::AlignedWriter writer; std::vector data{1, 2, 3}; @@ -148,7 +43,7 @@ TEST(AlignedWriter, BlockSizeOneUsesNoOpPath) TEST(AlignedWriter, SequentialPartialWritesReuseCachedBlock) { - RecordingEngine engine(8192); + TestRecordingEngine engine(8192); auto disk = std::make_shared(&engine, 4096); crowdb::diskio::AlignedWriter writer; std::vector first(1024, 0x11); @@ -161,7 +56,7 @@ TEST(AlignedWriter, SequentialPartialWritesReuseCachedBlock) EXPECT_EQ(first_result, 1024); EXPECT_EQ(second_result, 1024); - EXPECT_EQ(engine.read_count, 0); + EXPECT_EQ(engine.read_count, 1); ASSERT_EQ(engine.write_sizes.size(), 2); EXPECT_EQ(engine.write_sizes[0], 4096); EXPECT_EQ(engine.write_sizes[1], 4096); @@ -173,7 +68,7 @@ TEST(AlignedWriter, SequentialPartialWritesReuseCachedBlock) TEST(AlignedWriter, CacheMissReadsAndMergesExistingBlock) { - RecordingEngine engine(8192); + TestRecordingEngine engine(8192); std::fill(engine.storage.begin(), engine.storage.begin() + 4096, 0x7a); auto disk = std::make_shared(&engine, 4096); crowdb::diskio::AlignedWriter writer; @@ -194,7 +89,7 @@ TEST(AlignedWriter, CacheMissReadsAndMergesExistingBlock) TEST(AlignedWriter, FullAlignedWriteInvalidatesPartialCache) { - RecordingEngine engine(8192); + TestRecordingEngine engine(8192); auto disk = std::make_shared(&engine, 4096); crowdb::diskio::AlignedWriter writer; std::vector partial(1024, 0x11); @@ -208,7 +103,7 @@ TEST(AlignedWriter, FullAlignedWriteInvalidatesPartialCache) writer.submit(disk, 0, aligned.get(), 4096, [](int result) { ASSERT_EQ(result, 4096); }); writer.submit(disk, 1024, continuation.data(), continuation.size(), [](int result) { ASSERT_EQ(result, 1024); }); - EXPECT_EQ(engine.read_count, 1); + EXPECT_EQ(engine.read_count, 2); EXPECT_TRUE(std::all_of(engine.storage.begin(), engine.storage.begin() + 1024, [](uint8_t value) { return value == 0x44; })); EXPECT_TRUE(std::all_of(engine.storage.begin() + 1024, engine.storage.begin() + 2048, @@ -219,7 +114,7 @@ TEST(AlignedWriter, FullAlignedWriteInvalidatesPartialCache) TEST(AlignedWriter, FullRangeWriteInvalidatesInteriorPartialCache) { - RecordingEngine engine(16384); + TestRecordingEngine engine(16384); auto disk = std::make_shared(&engine, 4096); crowdb::diskio::AlignedWriter writer; std::vector partial(1024, 0x11); @@ -233,7 +128,7 @@ TEST(AlignedWriter, FullRangeWriteInvalidatesInteriorPartialCache) writer.submit(disk, 0, aligned.get(), 12288, [](int result) { ASSERT_EQ(result, 12288); }); writer.submit(disk, 5120, continuation.data(), continuation.size(), [](int result) { ASSERT_EQ(result, 1024); }); - EXPECT_EQ(engine.read_count, 1); + EXPECT_EQ(engine.read_count, 2); EXPECT_TRUE(std::all_of(engine.storage.begin() + 4096, engine.storage.begin() + 5120, [](uint8_t value) { return value == 0x44; })); EXPECT_TRUE(std::all_of(engine.storage.begin() + 5120, engine.storage.begin() + 6144, @@ -244,7 +139,7 @@ TEST(AlignedWriter, FullRangeWriteInvalidatesInteriorPartialCache) TEST(AlignedWriter, DestructionWaitsForActiveCompletion) { - RecordingEngine engine(8192, true); + TestRecordingEngine engine(8192, true); auto disk = std::make_shared(&engine, 4096); auto writer = std::make_unique(); std::vector data(1024, 0x11); @@ -264,7 +159,7 @@ TEST(AlignedWriter, DestructionWaitsForActiveCompletion) TEST(AlignedWriter, SameBlockWritesCompleteInSubmissionOrder) { - RecordingEngine engine(8192, true); + TestRecordingEngine engine(8192, true); auto disk = std::make_shared(&engine, 4096); crowdb::diskio::AlignedWriter writer; std::vector first(1024, 0x11); @@ -285,7 +180,7 @@ TEST(AlignedWriter, SameBlockWritesCompleteInSubmissionOrder) TEST(AlignedWriter, OrderedWritesShareOneSubmissionQueue) { - RecordingEngine engine(8192, true); + TestRecordingEngine engine(8192, true); auto disk = std::make_shared(&engine, 4096); crowdb::diskio::AlignedWriter writer; std::vector first_data(1024, 0x11); diff --git a/app/crowdb-diskio/tests/common/aligned_writer.h b/app/crowdb-diskio/tests/common/aligned_writer.h new file mode 100644 index 000000000..7c10b32c3 --- /dev/null +++ b/app/crowdb-diskio/tests/common/aligned_writer.h @@ -0,0 +1,153 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "disk/disk.h" +#include "engine/aligned_writer.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace crowdb::diskio::test +{ + +class TestRecordingEngine final : public crowdb::diskio::IoEngine +{ + public: + explicit TestRecordingEngine(size_t capacity, bool defer_writes = false) + : storage(capacity, 0), + defer_writes(defer_writes) + { + } + + void submit_write(crowdb::diskio::Disk * /*disk*/, off_t offset, const uint8_t *data, size_t size, + std::function on_complete) override + { + write_offsets.push_back(offset); + write_sizes.push_back(size); + if (offset < 0 || static_cast(offset) + size > storage.size()) { + on_complete(-1); + return; + } + std::memcpy(storage.data() + offset, data, size); + if (defer_writes) { + pending_writes.emplace_back(size, std::move(on_complete)); + } + else { + on_complete(static_cast(size)); + } + } + + void submit_read(crowdb::diskio::Disk * /*disk*/, off_t offset, uint8_t *buf, size_t size, + uint64_t /*test_pattern_offset*/, std::function on_complete) override + { + read_count++; + if (failed_read_offset == offset) { + on_complete(failed_read_result); + return; + } + if (offset < 0 || static_cast(offset) + size > storage.size()) { + on_complete(-1); + return; + } + if (defer_reads) { + pending_reads.emplace_back([this, offset, buf, size, callback = std::move(on_complete)] { + std::memcpy(buf, storage.data() + offset, size); + callback(static_cast(size)); + }); + return; + } + std::memcpy(buf, storage.data() + offset, size); + on_complete(static_cast(size)); + } + + void submit_fsync(crowdb::diskio::Disk * /*disk*/, std::function on_complete) override + { + on_complete(0); + } + + void complete_one_write() + { + ASSERT_FALSE(pending_writes.empty()); + auto [size, callback] = std::move(pending_writes.front()); + pending_writes.pop_front(); + callback(static_cast(size)); + } + + void complete_one_read() + { + ASSERT_FALSE(pending_reads.empty()); + auto callback = std::move(pending_reads.front()); + pending_reads.pop_front(); + callback(); + } + + std::vector storage; + bool defer_writes; + bool defer_reads{false}; + std::deque> pending_reads; + std::optional failed_read_offset; + int failed_read_result{-EIO}; + size_t read_count{0}; + std::vector write_offsets; + std::vector write_sizes; + std::deque>> pending_writes; +}; + +class TestDisk final : public crowdb::diskio::Disk +{ + public: + TestDisk(TestRecordingEngine *engine, size_t block_size) : engine_(engine), block_size_(block_size) + { + } + + [[nodiscard]] crowdb::diskio::DiskType type() const override + { + return crowdb::diskio::DiskType::Mem; + } + + [[nodiscard]] int fd() const override + { + return 1; + } + + [[nodiscard]] bool is_o_direct() const override + { + return block_size_ > 1; + } + + [[nodiscard]] size_t block_size() const override + { + return block_size_; + } + + crowdb::diskio::IoEngine *engine() override + { + return engine_; + } + + [[nodiscard]] crowdb::diskio::DiskId id() const override + { + return {.high = 7, .low = 9}; + } + + crowdb::diskio::Zone *find_zone(uint32_t /*zone_index*/) override + { + return nullptr; + } + + private: + TestRecordingEngine *engine_; + size_t block_size_; +}; + +} // namespace crowdb::diskio::test diff --git a/app/crowdb-diskio/tests/common/synchronized_write.h b/app/crowdb-diskio/tests/common/synchronized_write.h new file mode 100644 index 000000000..8dfe30b56 --- /dev/null +++ b/app/crowdb-diskio/tests/common/synchronized_write.h @@ -0,0 +1,220 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "disk/disk.h" +#include "rpc/dio_server.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace crowdb::diskio::test +{ +namespace dproto = crowdb::diskio::proto; +namespace dio = crowdb::diskio; +namespace rpc = crowdb::rpc; + +class TestWriteEngine final : public dio::IoEngine +{ + public: + struct PendingWrite + { + const uint8_t *data; + size_t size; + std::function complete; + }; + + void submit_write(dio::Disk * /*disk*/, off_t /*offset*/, const uint8_t *data, size_t size, + std::function complete) override + { + writes.push_back({.data = data, .size = size, .complete = std::move(complete)}); + } + + void submit_read(dio::Disk * /*disk*/, off_t /*offset*/, uint8_t *data, size_t size, uint64_t /*pattern*/, + std::function complete) override + { + std::memset(data, 0, size); + complete(static_cast(size)); + } + + void submit_fsync(dio::Disk * /*disk*/, std::function complete) override + { + flushes.push_back(std::move(complete)); + } + + void write_done(int result) + { + ASSERT_FALSE(writes.empty()); + auto pending = std::move(writes.front()); + writes.pop_front(); + pending.complete(result); + } + + void flush_done(int result) + { + ASSERT_FALSE(flushes.empty()); + auto complete = std::move(flushes.front()); + flushes.pop_front(); + complete(result); + } + + std::deque writes; + std::deque> flushes; +}; + +class TestWriteDisk final : public dio::Disk +{ + public: + explicit TestWriteDisk(dio::IoEngine *engine) : engine_(engine) + { + zones_.push_back({.zone_index = 0, .base_offset = 0, .capacity = 4096}); + } + + [[nodiscard]] dio::DiskType type() const override + { + return dio::DiskType::Mem; + } + + [[nodiscard]] int fd() const override + { + return 1; + } + + [[nodiscard]] bool is_o_direct() const override + { + return block > 1; + } + + [[nodiscard]] size_t block_size() const override + { + return block; + } + + dio::IoEngine *engine() override + { + return engine_; + } + + [[nodiscard]] dio::DiskId id() const override + { + return {.high = 1, .low = 2}; + } + + dio::Zone *find_zone(uint32_t index) override + { + return index == 0 ? &zones_.front() : nullptr; + } + + size_t block = 1; + + private: + dio::IoEngine *engine_; +}; + +class TestWriteTransport final : public rpc::Transport +{ + public: + ~TestWriteTransport() override + { + for (auto *frame : replies) { + frame->control->release(); + frame->release_data(); + delete frame; + } + } + + bool submit(rpc::Connection * /*connection*/, rpc::OutFrame *frame) override + { + ++attempts; + if (reject) { + return false; + } + replies.push_back(frame); + return true; + } + + rpc::Buffer *register_buffer(rpc::Buffer *buffer) override + { + return buffer; + } + + void shutdown() override + { + } + + [[nodiscard]] const dproto::FBDiskWriteResponse *reply(size_t index = 0) const + { + return flatbuffers::GetRoot(replies.at(index)->control->data); + } + + bool reject = false; + size_t attempts = 0; + std::vector replies; +}; + +class TestSynchronizedWrite : public testing::Test +{ + protected: + TestSynchronizedWrite() + : disks(std::make_shared()), + disk(std::make_shared(&engine)), + connection(std::make_shared(1, "write-test", &pool)), + server(disks, &transport) + { + payload.fill(7); + disks->add(disk); + } + + rpc::Frame *request(bool sync, uint64_t offset = 0, uint64_t id = 41, uint32_t size = 32, + std::optional created_ms = std::nullopt) + { + flatbuffers::FlatBufferBuilder builder; + crowdb::rpc::proto::FBInt128 identity(1, 2); + auto now = static_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()) + .count()); + auto control = dproto::CreateFBDiskWriteRequest(builder, id, 0, &identity, 0, offset, size, 0, + created_ms.value_or(now), sync); + builder.Finish(control); + auto *frame = new rpc::Frame; + frame->request_id = id; + frame->control.assign(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); + if (size == 0) { + return frame; + } + auto *buffer = new rpc::Buffer; + buffer->data = payload.data(); + buffer->len = size; + buffer->capacity = 32; + buffer->ref = new std::atomic(1); + buffer->free_ctx = &released; + buffer->free_cb = [](void *context) { ++*static_cast(context); }; + frame->data_buf = buffer; + return frame; + } + + rpc::SystemBufferPool pool; + TestWriteTransport transport; + TestWriteEngine engine; + std::shared_ptr disks; + std::shared_ptr disk; + std::shared_ptr connection; + dio::DiskioServer server; + std::array payload{}; + size_t released = 0; +}; + +} // namespace crowdb::diskio::test diff --git a/app/crowdb-diskio/tests/dio_config_test.cpp b/app/crowdb-diskio/tests/dio_config_test.cpp index 5ae60700e..bd8fb305c 100644 --- a/app/crowdb-diskio/tests/dio_config_test.cpp +++ b/app/crowdb-diskio/tests/dio_config_test.cpp @@ -82,8 +82,8 @@ rpc_workers = 7 node_id = 9 dummy_disk_type = "mem" o_direct = false -max_write_request_age_ms = 40000 -max_clock_skew_ms = 2000 +max_write_request_age_ms = 20000 +max_clock_skew_ms = 500 fault_latency_min_ms = 2 fault_latency_max_ms = 8 fault_error_rate = 0.25 @@ -119,12 +119,13 @@ zone_capacity = 4096 EXPECT_EQ(config.node_id, 9U); EXPECT_EQ(config.dummy_disk_type, DummyDiskType::Mem); EXPECT_FALSE(config.o_direct); - EXPECT_EQ(config.max_write_request_age_ms, 40000U); - EXPECT_EQ(config.max_clock_skew_ms, 2000U); + EXPECT_EQ(config.max_write_request_age_ms, 20000U); + EXPECT_EQ(config.max_clock_skew_ms, 500U); ASSERT_TRUE(config.dummy_props.has_value()); - EXPECT_EQ(config.dummy_props->latency_min_ms, 2U); - EXPECT_EQ(config.dummy_props->latency_max_ms, 8U); - EXPECT_DOUBLE_EQ(config.dummy_props->error_rate, 0.25); + const auto properties = config.dummy_props.value(); + EXPECT_EQ(properties.latency_min_ms, 2U); + EXPECT_EQ(properties.latency_max_ms, 8U); + EXPECT_DOUBLE_EQ(properties.error_rate, 0.25); EXPECT_EQ(config.thread_pool_size, 6U); EXPECT_EQ(config.sq_entries, 512U); EXPECT_EQ(config.kv_seeds.size(), 2U); @@ -143,6 +144,21 @@ zone_capacity = 4096 EXPECT_EQ(config.disks[0].zones[0].capacity, 4096); } +TEST(DioConfigTest, RejectsPoliciesBeyondSharedRecoveryBounds) +{ + DioConfig config; + std::string error; + config.max_write_request_age_ms = crowdb::protocol::kMaxWriteRequestAgeMs + 1; + EXPECT_FALSE(config.validate(error)); + EXPECT_EQ(error, "write request age and clock skew must not exceed the shared recovery bounds"); + config.max_write_request_age_ms = crowdb::protocol::kMaxWriteRequestAgeMs; + config.max_clock_skew_ms = crowdb::protocol::kMaxWriteClockSkewMs + 1; + EXPECT_FALSE(config.validate(error)); + config.max_write_request_age_ms = 100; + config.max_clock_skew_ms = 0; + EXPECT_TRUE(config.validate(error)) << error; +} + TEST(DioConfigTest, CliOverridesFileRegardlessOfArgumentOrder) { TempConfig file("[server]\nlisten_port = 13042\nrpc_workers = 7\n[engine]\nthread_pool_size = 6\n"); diff --git a/app/crowdb-diskio/tests/dio_server_drain_test.cpp b/app/crowdb-diskio/tests/dio_server_drain_test.cpp new file mode 100644 index 000000000..b7ea4c253 --- /dev/null +++ b/app/crowdb-diskio/tests/dio_server_drain_test.cpp @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/synchronized_write.h" + +#include + +#include +#include +#include +#include + +namespace +{ +namespace dproto = crowdb::diskio::proto; +using DiskioWriteDrain = crowdb::diskio::test::TestSynchronizedWrite; + +TEST_F(DiskioWriteDrain, EmptySynchronizationWaitsForPriorByteWritesAndItsFlush) +{ + server.handle_write(request(false), connection.get()); + server.handle_write(request(true, 0, 42, 0), connection.get()); + EXPECT_EQ(engine.writes.size(), 1); + EXPECT_TRUE(engine.flushes.empty()); + EXPECT_TRUE(transport.replies.empty()); + engine.write_done(32); + EXPECT_TRUE(engine.writes.empty()); + EXPECT_EQ(engine.flushes.size(), 1); + EXPECT_EQ(transport.replies.size(), 1); + engine.flush_done(0); + ASSERT_EQ(transport.replies.size(), 2); + EXPECT_EQ(transport.reply(1)->id(), 42); + EXPECT_EQ(transport.reply(1)->ret_code(), dproto::FBDiskIoRetCode_Success); + EXPECT_TRUE(transport.reply(1)->sync_completed()); + EXPECT_EQ(released, 1); +} + +TEST_F(DiskioWriteDrain, MissingStaleAndUnboundedFutureTimesAreRejectedBeforeIo) +{ + for (const uint64_t created_ms : {uint64_t{0}, uint64_t{1}, std::numeric_limits::max()}) { + server.handle_write(request(true, 0, 41, 32, created_ms), connection.get()); + } + EXPECT_TRUE(engine.writes.empty()); + EXPECT_TRUE(engine.flushes.empty()); + ASSERT_EQ(transport.replies.size(), 3); + for (size_t index = 0; index < 3; ++index) { + EXPECT_EQ(transport.reply(index)->ret_code(), dproto::FBDiskIoRetCode_OldRequest); + EXPECT_FALSE(transport.reply(index)->sync_completed()); + } + EXPECT_EQ(released, 3); +} + +TEST_F(DiskioWriteDrain, AQueuedExpiredWriteCannotRunBeforeTheFreshDrain) +{ + crowdb::diskio::DiskioServer guarded(disks, &transport, 50, 1); + guarded.handle_write(request(false, 0, 41), connection.get()); + guarded.handle_write(request(false, 32, 42), connection.get()); + EXPECT_EQ(engine.writes.size(), 1); + EXPECT_TRUE(transport.replies.empty()); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + guarded.handle_write(request(true, 0, 43, 0), connection.get()); + engine.write_done(32); + EXPECT_TRUE(engine.writes.empty()); + EXPECT_EQ(engine.flushes.size(), 1); + while (!engine.writes.empty()) { + engine.write_done(32); + } + while (!engine.flushes.empty()) { + engine.flush_done(0); + } + ASSERT_EQ(transport.replies.size(), 3); + EXPECT_EQ(transport.reply()->id(), 41); + EXPECT_EQ(transport.reply(1)->id(), 42); + EXPECT_EQ(transport.reply(1)->ret_code(), dproto::FBDiskIoRetCode_OldRequest); + EXPECT_FALSE(transport.reply(1)->sync_completed()); + EXPECT_EQ(transport.reply(2)->id(), 43); + EXPECT_TRUE(transport.reply(2)->sync_completed()); +} +} // namespace diff --git a/app/crowdb-diskio/tests/dio_server_write_test.cpp b/app/crowdb-diskio/tests/dio_server_write_test.cpp new file mode 100644 index 000000000..5963d4d4e --- /dev/null +++ b/app/crowdb-diskio/tests/dio_server_write_test.cpp @@ -0,0 +1,164 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "common/synchronized_write.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ +namespace dproto = crowdb::diskio::proto; +namespace dio = crowdb::diskio; +namespace rpc = crowdb::rpc; + +using DiskioSynchronizedWrite = crowdb::diskio::test::TestSynchronizedWrite; + +TEST_F(DiskioSynchronizedWrite, ReplyWaitsForBothWriteAndFlush) +{ + server.handle_write(request(true), connection.get()); + ASSERT_EQ(engine.writes.size(), 1); + EXPECT_TRUE(engine.flushes.empty()); + EXPECT_TRUE(transport.replies.empty()); + engine.write_done(32); + ASSERT_EQ(engine.flushes.size(), 1); + EXPECT_TRUE(transport.replies.empty()); + EXPECT_EQ(released, 0); + engine.flush_done(0); + ASSERT_EQ(transport.replies.size(), 1); + EXPECT_EQ(transport.reply()->ret_code(), dproto::FBDiskIoRetCode_Success); + EXPECT_TRUE(transport.reply()->sync_completed()); + EXPECT_EQ(released, 1); +} + +TEST_F(DiskioSynchronizedWrite, BufferedWriteDoesNotFlush) +{ + server.handle_write(request(false), connection.get()); + engine.write_done(32); + EXPECT_TRUE(engine.flushes.empty()); + ASSERT_EQ(transport.replies.size(), 1); + EXPECT_FALSE(transport.reply()->sync_completed()); + EXPECT_EQ(released, 1); +} + +TEST_F(DiskioSynchronizedWrite, FailedWriteDoesNotFlushOrSucceed) +{ + server.handle_write(request(true), connection.get()); + engine.write_done(-EIO); + EXPECT_TRUE(engine.flushes.empty()); + ASSERT_EQ(transport.replies.size(), 1); + EXPECT_EQ(transport.reply()->ret_code(), dproto::FBDiskIoRetCode_IoError); + EXPECT_FALSE(transport.reply()->sync_completed()); + EXPECT_EQ(released, 1); +} + +TEST_F(DiskioSynchronizedWrite, PartialWriteDoesNotFlushOrSucceed) +{ + server.handle_write(request(true), connection.get()); + engine.write_done(31); + EXPECT_TRUE(engine.flushes.empty()); + ASSERT_EQ(transport.replies.size(), 1); + EXPECT_EQ(transport.reply()->ret_code(), dproto::FBDiskIoRetCode_PartialWrite); + EXPECT_FALSE(transport.reply()->sync_completed()); +} + +TEST_F(DiskioSynchronizedWrite, FailedFlushReturnsDedicatedFailure) +{ + server.handle_write(request(true), connection.get()); + engine.write_done(32); + engine.flush_done(-EIO); + ASSERT_EQ(transport.replies.size(), 1); + EXPECT_EQ(transport.reply()->ret_code(), dproto::FBDiskIoRetCode_FsyncFailed); + EXPECT_FALSE(transport.reply()->sync_completed()); + EXPECT_EQ(released, 1); +} + +TEST_F(DiskioSynchronizedWrite, FlushRetainsRemovedDiskConnectionAndPayload) +{ + std::weak_ptr weak_disk = disk; + std::weak_ptr weak_connection = connection; + server.handle_write(request(true), connection.get()); + ASSERT_TRUE(disks->remove_disk(disk->id())); + disk.reset(); + connection.reset(); + engine.write_done(32); + EXPECT_FALSE(weak_disk.expired()); + EXPECT_FALSE(weak_connection.expired()); + EXPECT_EQ(released, 0); + engine.flush_done(0); + EXPECT_TRUE(weak_disk.expired()); + EXPECT_TRUE(weak_connection.expired()); + EXPECT_EQ(released, 1); + EXPECT_EQ(transport.attempts, 1); +} + +TEST_F(DiskioSynchronizedWrite, RejectedResponseStillReleasesOwners) +{ + transport.reject = true; + std::weak_ptr weak_connection = connection; + server.handle_write(request(true), connection.get()); + connection.reset(); + engine.write_done(32); + engine.flush_done(0); + EXPECT_EQ(transport.attempts, 1); + EXPECT_TRUE(transport.replies.empty()); + EXPECT_EQ(released, 1); + EXPECT_TRUE(weak_connection.expired()); +} + +TEST_F(DiskioSynchronizedWrite, InvalidControlPayloadAndZoneAreRejectedBeforeIo) +{ + auto *malformed = request(true); + malformed->control.resize(3); + server.handle_write(malformed, connection.get()); + auto *short_payload = request(true); + short_payload->data_buf->len = 16; + server.handle_write(short_payload, connection.get()); + server.handle_write(request(true, 4090), connection.get()); + EXPECT_TRUE(engine.writes.empty()); + EXPECT_TRUE(engine.flushes.empty()); + ASSERT_EQ(transport.replies.size(), 3); + for (size_t index = 0; index != 3; ++index) { + EXPECT_EQ(transport.reply(index)->ret_code(), dproto::FBDiskIoRetCode_IoError); + EXPECT_FALSE(transport.reply(index)->sync_completed()); + } + EXPECT_EQ(released, 3); +} + +TEST_F(DiskioSynchronizedWrite, SameStripWritesAndTheirFlushesRetainOrder) +{ + disk->block = 4096; + server.handle_write(request(true, 0, 41), connection.get()); + server.handle_write(request(true, 32, 42), connection.get()); + ASSERT_EQ(engine.writes.size(), 1); + engine.write_done(4096); + ASSERT_EQ(engine.writes.size(), 1); + EXPECT_TRUE(std::all_of(engine.writes.front().data, engine.writes.front().data + 64, + [](uint8_t value) { return value == 7; })); + ASSERT_EQ(engine.flushes.size(), 1); + EXPECT_TRUE(transport.replies.empty()); + engine.flush_done(0); + ASSERT_EQ(transport.replies.size(), 1); + EXPECT_EQ(transport.reply()->id(), 41); + engine.write_done(4096); + engine.flush_done(0); + ASSERT_EQ(transport.replies.size(), 2); + EXPECT_EQ(transport.reply(1)->id(), 42); + EXPECT_TRUE(transport.reply()->sync_completed()); + EXPECT_TRUE(transport.reply(1)->sync_completed()); + EXPECT_EQ(released, 2); +} +} // namespace diff --git a/app/crowdb-kv-server/Cargo.toml b/app/crowdb-kv-server/Cargo.toml index 24ec3a4dd..f06ee6a43 100644 --- a/app/crowdb-kv-server/Cargo.toml +++ b/app/crowdb-kv-server/Cargo.toml @@ -36,12 +36,25 @@ arc-swap = "1.9" utoipa = "5" bytes = "1" sha2 = "0.10" +uuid = { version = "1", features = ["v4"] } [dev-dependencies] reqwest = { version = "0.12", features = ["json"] } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../../lib/crowdb-test-harness" } +[[test]] +name = "keepalive_recovery_test" +path = "tests/keepalive_recovery_test.rs" + +[[test]] +name = "group_creation_test" +path = "tests/group_creation_test.rs" + +[[test]] +name = "member_routes_test" +path = "tests/member_routes_test.rs" + [[test]] name = "chunk_dynamic_monitor_test" path = "tests/chunk_dynamic_monitor_test.rs" diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs index cdb0a9ad1..193b3ca19 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv.rs @@ -24,6 +24,7 @@ use super::{DomainMonitorDriver, DomainMonitorFuture}; mod balance; mod catalog; +mod grants; pub struct ChunkKvRangeMonitorDriver; @@ -55,12 +56,15 @@ impl DomainMonitorDriver for ChunkKvRangeMonitorDriver { descriptor: &'a DomainMonitorDescriptor, ) -> DomainMonitorFuture<'a> { Box::pin(async move { + if catalog::refresh_endpoints(control, descriptor).await? { + return Ok(()); + } plan_dead_owner_transfer(control, descriptor).await?; advance_dead_owner_exclusion(control, descriptor).await?; balance::plan(control, descriptor).await?; authorize_live_source_fences(control, descriptor).await?; publish_ready_transitions(control).await?; - issue_serving_grants(control, descriptor).await + grants::issue(control, descriptor).await }) } } @@ -91,6 +95,17 @@ async fn plan_dead_owner_transfer( transitioning_partitions.insert(transition.partition_id); } } + // A split may already have durable child authority even while the old + // parent remains published. A competing transfer cannot infer its recovery + // boundary from the parent catalog entry alone. + for (split, _) in read_splits(control).await? { + if !matches!(split.phase, SplitPhase::CatalogCommitted | SplitPhase::Aborted) { + transitioning_partitions.insert(split.parent_id); + transitioning_partitions.insert(split.child.partition_id); + busy_owners.insert(split.parent_owner.instance_id); + busy_owners.insert(split.child.owner.instance_id); + } + } let healthy_before = now_ms.saturating_sub(descriptor.suspect_after_ms); let dead_before = now_ms.saturating_sub(descriptor.dead_after_ms); let mut targets: Vec<_> = instances @@ -313,106 +328,6 @@ async fn publish_ready_transitions(control: &Group0ControlPlane) -> Result<(), S Ok(()) } -async fn issue_serving_grants( - control: &Group0ControlPlane, - descriptor: &DomainMonitorDescriptor, -) -> Result<(), String> { - let Some(catalog) = catalog::load_current(control).await? else { - return Ok(()); - }; - let ready_instances = read_ready_instances(control, descriptor).await?; - let mut assignments: HashMap> = HashMap::new(); - for entry in catalog.pages.iter().flat_map(|page| &page.entries) { - let assignment = ServingAssignment { - partition_id: entry.partition_id, - owner_epoch: entry.owner_epoch, - }; - if entry.state == ChunkKvRangeCatalogPartitionState::Serving - && ready_instances - .get(&entry.owner.instance_id) - .is_some_and(|ready| ready.contains(&assignment)) - { - assignments - .entry(entry.owner.instance_id) - .or_default() - .push(assignment); - } - } - let now_ms = wall_time_ms(); - for (instance_id, assignments) in assignments { - let path = ServingGrantKey { instance_id }.to_path(); - let current = control - .get(path.as_bytes()) - .await - .map_err(|error| operation_error(&error))?; - let prior_sequence = match current.value.as_deref() { - Some(bytes) => { - let grant: ServingGrant = serde_json::from_slice(bytes).map_err(|error| error.to_string())?; - grant.validate().map_err(|error| error.to_string())?; - grant.lease_sequence - } - None => 0, - }; - let mut grant = ServingGrant { - instance_id, - lease_sequence: prior_sequence - .checked_add(1) - .ok_or_else(|| "serving grant sequence overflowed".to_string())?, - catalog_generation: catalog.head.generation, - issued_at_ms: now_ms, - expires_at_ms: now_ms - .checked_add(descriptor.lease_duration_ms) - .ok_or_else(|| "serving grant deadline overflowed".to_string())?, - assignments, - assignment_digest: [0; 32], - }; - grant.seal(); - grant.validate().map_err(|error| error.to_string())?; - let encoded = serde_json::to_vec(&grant).map_err(|error| error.to_string())?; - control - .compare_and_put(Bytes::from(path), Bytes::from(encoded), current.revision) - .await - .map_err(|error| operation_error(&error))?; - } - Ok(()) -} - -async fn read_ready_instances( - control: &Group0ControlPlane, - descriptor: &DomainMonitorDescriptor, -) -> Result>, String> { - let prefix = InstanceKey::text_prefix_for_service(&descriptor.service_registry_name); - let items = control - .scan_all_prefix(Bytes::from(prefix), 256) - .await - .map_err(|error| operation_error(&error))?; - let healthy_after = wall_time_ms().saturating_sub(descriptor.suspect_after_ms); - let mut ready = HashMap::new(); - for item in items { - let value: InstanceValue = serde_json::from_slice(&item.value).map_err(|error| error.to_string())?; - if value.last_heartbeat_ms < healthy_after { - continue; - } - let Some(extra) = value.extra.and_then(|extra| extra.chunk_kv) else { - return Err(format!( - "chunk-KV instance {} omitted its chunk-KV observation", - value.instance_id - )); - }; - let assignments = extra - .hosted - .into_iter() - .filter(|hosted| !hosted.recovering) - .map(|hosted| ServingAssignment { - partition_id: hosted.partition_id, - owner_epoch: hosted.owner_epoch, - }) - .collect(); - ready.insert(value.instance_id, assignments); - } - Ok(ready) -} - pub(super) async fn read_instances( control: &Group0ControlPlane, descriptor: &DomainMonitorDescriptor, diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs index b72097a06..eaee82913 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/balance.rs @@ -65,7 +65,7 @@ pub async fn plan(control: &Group0ControlPlane, descriptor: &DomainMonitorDescri && entry.transition_id == Some(transition.transition_id) && entry.partition_id == transition.partition_id && entry.range == transition.range - && entry.owner == transition.target + && entry.owner.instance_id == transition.target.instance_id && entry.owner_epoch == transition.target_epoch && entry.artifact == transition.target_artifact }); diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs index f42cc89d3..065965e33 100644 --- a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog.rs @@ -14,6 +14,9 @@ use crowdb_protocol::key::{ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKe use crate::group0_control_plane::Group0ControlPlane; +mod endpoints; +pub(super) use endpoints::refresh as refresh_endpoints; + pub struct VersionedRangeCatalog { pub head: ChunkKvRangeCatalogHead, pub pages: Vec, @@ -99,7 +102,7 @@ pub async fn publish_transfer( )?; let source = |entry: &ChunkKvRangeCatalogEntry| { let source_serving = transfer_source_matches(entry, transition); - let target_catching_up = entry.owner == transition.target + let target_catching_up = entry.owner.instance_id == transition.target.instance_id && entry.owner_epoch == transition.target_epoch && entry.state == ChunkKvRangeCatalogPartitionState::TargetCatchingUp && entry.artifact == transition.target_artifact @@ -119,11 +122,19 @@ pub async fn publish_transfer( fn transfer_source_matches(entry: &ChunkKvRangeCatalogEntry, transition: &TransferTransition) -> bool { entry.partition_id == transition.partition_id && entry.range == transition.range - && entry.owner == transition.source + && entry.owner.instance_id == transition.source.instance_id && entry.owner_epoch == transition.source_epoch && entry.state == ChunkKvRangeCatalogPartitionState::Serving && entry.artifact == transition.artifact - && (entry.transition_id.is_none() || entry.artifact.tail_overlay.is_none()) + // Lease recovery retains the complete split artifact, including its + // historical overlay. The old split marker is not a competing owner; + // the exact source assignment and expired lease fence this cutover. + && (entry.transition_id.is_none() + || entry.artifact.tail_overlay.is_none() + || matches!( + transition.release_proof, + Some(AuthorityReleaseProof::LeaseExpired { .. }) + )) && entry.transition_id != Some(transition.transition_id) } @@ -329,7 +340,7 @@ pub async fn publish_split( let parent = |entry: &ChunkKvRangeCatalogEntry| { entry.partition_id == transition.parent_id && entry.range == transition.parent_range - && entry.owner == transition.parent_owner + && entry.owner.instance_id == transition.parent_owner.instance_id && entry.owner_epoch == transition.parent_epoch && entry.state == ChunkKvRangeCatalogPartitionState::Serving && entry.artifact == transition.parent_artifact @@ -536,7 +547,12 @@ fn published_assignment_matches( current: &ChunkKvRangeCatalogEntry, desired: &ChunkKvRangeCatalogEntry, ) -> bool { - if current == desired { + // Connections may move while ownership and the fixed transition stay intact. + let mut current = current.clone(); + if current.owner.instance_id == desired.owner.instance_id { + current.owner.rpc_endpoint.clone_from(&desired.owner.rpc_endpoint); + } + if current == *desired { return true; } // Materialization may advance the catalog after publication but before its @@ -549,7 +565,7 @@ fn published_assignment_matches( { return false; } - let mut materialized = current.clone(); + let mut materialized = current; materialized .artifact .tail_overlay @@ -608,7 +624,7 @@ mod tests { source_partition_id: Id128 { high: 9, low: 10 }, source_epoch: 3, source_stream_name: StreamName { high: 11, low: 12 }, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 16, base_root_manifest_generation: 1, @@ -689,7 +705,7 @@ mod tests { source_partition_id: partition_id, source_epoch: 3, source_stream_name: StreamName { high: 8, low: 9 }, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 16, base_root_manifest_generation: 1, diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog/endpoints.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog/endpoints.rs new file mode 100644 index 000000000..2fac491e0 --- /dev/null +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/catalog/endpoints.rs @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Publish current addresses for recovered owners without transferring partitions. + +use super::super::{read_instances, wall_time_ms}; +use super::{load_current, publish, replace_one}; +use crate::group0_control_plane::Group0ControlPlane; +use crowdb_protocol::chunk_kv::DomainMonitorDescriptor; + +pub(in crate::background::domain_monitor::chunk_kv) async fn refresh( + control: &Group0ControlPlane, + descriptor: &DomainMonitorDescriptor, +) -> Result { + let Some(catalog) = load_current(control).await? else { + return Ok(false); + }; + let instances = read_instances(control, descriptor).await?; + let healthy_after = wall_time_ms().saturating_sub(descriptor.suspect_after_ms); + for entry in catalog.pages.iter().flat_map(|page| &page.entries) { + let Some(instance) = instances.get(&entry.owner.instance_id) else { + continue; + }; + let hosted = instance + .extra + .as_ref() + .and_then(|extra| extra.chunk_kv.as_ref()) + .is_some_and(|extra| { + extra.hosted.iter().any(|hosted| { + hosted.partition_id == entry.partition_id && hosted.owner_epoch == entry.owner_epoch + }) + }); + if instance.last_heartbeat_ms < healthy_after + || !hosted + || instance.rpc_endpoint == entry.owner.rpc_endpoint + { + continue; + } + instance + .rpc_endpoint + .parse::() + .map_err(|_| "recovered chunk-KV owner has an invalid RPC address")?; + let mut replacement = entry.clone(); + replacement.owner.rpc_endpoint.clone_from(&instance.rpc_endpoint); + let partition = entry.partition_id; + let revision = catalog.head_revision; + let (head, pages) = replace_one( + catalog.head, + catalog.pages.clone(), + |entry| entry.partition_id == partition, + vec![replacement], + )?; + publish(control, head, pages, revision).await?; + return Ok(true); + } + Ok(false) +} diff --git a/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs new file mode 100644 index 000000000..e8ee91fda --- /dev/null +++ b/app/crowdb-kv-server/src/background/domain_monitor/chunk_kv/grants.rs @@ -0,0 +1,173 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Grant publication serialized with process replacement and renewal. + +use super::{catalog, operation_error, wall_time_ms}; +use crate::group0_control_plane::Group0ControlPlane; +use bytes::Bytes; +use crowdb_kv::cluster::group_operations::KvGroupMutation; +use crowdb_protocol::{ + chunk_kv::{ChunkKvRangeCatalogPartitionState, DomainMonitorDescriptor, ServingAssignment, ServingGrant}, + chunk_stream::{StreamProcessLease, StreamPurpose}, + common::InstanceValue, + key::{InstanceKey, ServingGrantKey, StreamProcessLeaseKey, TextKey}, +}; +use std::collections::{HashMap, HashSet}; + +struct ReadyInstance { + process_epoch: u64, + assignments: HashSet, +} + +pub(super) async fn issue( + control: &Group0ControlPlane, + descriptor: &DomainMonitorDescriptor, +) -> Result<(), String> { + let Some(catalog) = catalog::load_current(control).await? else { + return Ok(()); + }; + let ready = read_ready(control, descriptor).await?; + let mut assignments: HashMap> = HashMap::new(); + for entry in catalog.pages.iter().flat_map(|page| &page.entries) { + let assignment = ServingAssignment { + partition_id: entry.partition_id, + owner_epoch: entry.owner_epoch, + }; + if entry.state == ChunkKvRangeCatalogPartitionState::Serving + && ready + .get(&entry.owner.instance_id) + .is_some_and(|instance| instance.assignments.contains(&assignment)) + { + assignments + .entry(entry.owner.instance_id) + .or_default() + .push(assignment); + } + } + for (instance_id, assignments) in assignments { + publish( + control, + descriptor, + instance_id, + ready[&instance_id].process_epoch, + catalog.head.generation, + assignments, + ) + .await?; + } + Ok(()) +} + +async fn publish( + control: &Group0ControlPlane, + descriptor: &DomainMonitorDescriptor, + instance_id: u64, + process_epoch: u64, + generation: u64, + assignments: Vec, +) -> Result<(), String> { + let lease_path = StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id, + } + .to_path(); + let current = control + .get(lease_path.as_bytes()) + .await + .map_err(|error| operation_error(&error))?; + let Some(bytes) = current.value.as_deref() else { + return Ok(()); + }; + let mut lease: StreamProcessLease = serde_json::from_slice(bytes).map_err(|error| error.to_string())?; + lease.validate().map_err(|error| error.to_string())?; + let now = wall_time_ms(); + if lease.identity.purpose != StreamPurpose::Wal || lease.identity.instance_id != instance_id { + return Err("WAL process lease does not match its key".into()); + } + if lease.identity.process_epoch != process_epoch || now >= lease.expires_at_ms { + return Ok(()); + } + let mut grant = ServingGrant { + instance_id, + process_epoch, + lease_sequence: lease + .grant_sequence + .checked_add(1) + .ok_or("serving grant sequence overflowed")?, + catalog_generation: generation, + issued_at_ms: now, + expires_at_ms: now + .checked_add(descriptor.lease_duration_ms) + .ok_or("serving grant deadline overflowed")? + .min(lease.expires_at_ms), + assignments, + assignment_digest: [0; 32], + }; + grant.seal(); + grant.validate().map_err(|error| error.to_string())?; + lease.grant_sequence = grant.lease_sequence; + lease.grant_expires_at_ms = lease.grant_expires_at_ms.max(grant.expires_at_ms); + let mutations = [ + KvGroupMutation::Put { + key: Bytes::from(lease_path.clone()), + value: Bytes::from(serde_json::to_vec(&lease).map_err(|error| error.to_string())?), + }, + KvGroupMutation::Put { + key: Bytes::from(ServingGrantKey { instance_id }.to_path()), + value: Bytes::from(serde_json::to_vec(&grant).map_err(|error| error.to_string())?), + }, + ]; + control + .compare_and_batch(&mutations, Bytes::from(lease_path), current.revision) + .await + .map_err(|error| operation_error(&error))?; + Ok(()) +} + +async fn read_ready( + control: &Group0ControlPlane, + descriptor: &DomainMonitorDescriptor, +) -> Result, String> { + let items = control + .scan_all_prefix( + Bytes::from(InstanceKey::text_prefix_for_service( + &descriptor.service_registry_name, + )), + 256, + ) + .await + .map_err(|error| operation_error(&error))?; + let healthy_after = wall_time_ms().saturating_sub(descriptor.suspect_after_ms); + let mut ready = HashMap::new(); + for item in items { + let value: InstanceValue = serde_json::from_slice(&item.value).map_err(|error| error.to_string())?; + if value.last_heartbeat_ms < healthy_after { + continue; + } + let extra = value + .extra + .and_then(|extra| extra.chunk_kv) + .ok_or_else(|| format!("chunk-KV instance {} omitted its observation", value.instance_id))?; + if extra.process_epoch == 0 { + return Err("chunk-KV observation has no process epoch".into()); + } + let assignments = extra + .hosted + .into_iter() + .filter(|hosted| !hosted.recovering) + .map(|hosted| ServingAssignment { + partition_id: hosted.partition_id, + owner_epoch: hosted.owner_epoch, + }) + .collect(); + ready.insert( + value.instance_id, + ReadyInstance { + process_epoch: extra.process_epoch, + assignments, + }, + ); + } + Ok(ready) +} diff --git a/app/crowdb-kv-server/src/background/keepalive.rs b/app/crowdb-kv-server/src/background/keepalive.rs index 166e7f1ff..345288259 100644 --- a/app/crowdb-kv-server/src/background/keepalive.rs +++ b/app/crowdb-kv-server/src/background/keepalive.rs @@ -44,11 +44,17 @@ impl KeepAliveLoop { // Before Group 0 exists, this node can seed its own RPC endpoint. // Once Group 0 exists, use its management seeds for discovery. let bootstrap_local = group0_management_seeds.is_empty(); - let mgmt_seeds = if bootstrap_local { + let mut mgmt_seeds = if bootstrap_local { vec![rpc_endpoint.clone()] } else { group0_management_seeds }; + // The local management listener survives a complete address change. + // Retained peer seeds can all be unreachable before the first fresh + // registration enables normal committed endpoint publication. + if !mgmt_seeds.contains(&rpc_endpoint) { + mgmt_seeds.insert(0, rpc_endpoint.clone()); + } let handle = tokio::spawn(async move { let kv_client = CrowdbKvClient::new(ClientConfig::new(mgmt_seeds.clone())); if bootstrap_local { diff --git a/app/crowdb-kv-server/src/cli.rs b/app/crowdb-kv-server/src/cli.rs index cba578b2a..d7e63ad0a 100644 --- a/app/crowdb-kv-server/src/cli.rs +++ b/app/crowdb-kv-server/src/cli.rs @@ -26,6 +26,9 @@ pub struct Cli { /// (fixed on-disk layout). Required on every start. #[arg(long)] pub root: std::path::PathBuf, + /// Owning monitor root for authenticated member address recovery. + #[arg(long)] + pub node_recovery_root: Option, /// Optional TOML config for startup tunables in first-boot and restore modes. /// When omitted, tunables use `CrowDBConfig::default()`. diff --git a/app/crowdb-kv-server/src/main.rs b/app/crowdb-kv-server/src/main.rs index 195a807d3..e60dc75d9 100644 --- a/app/crowdb-kv-server/src/main.rs +++ b/app/crowdb-kv-server/src/main.rs @@ -132,6 +132,10 @@ async fn main() { None => CrowDBConfig::default(), }; config.apply_root(&args.root); + if let Some(root) = &args.node_recovery_root { + crowdb_kv_server::recovery::member_routes::configure(&config.config_root, root) + .expect("invalid monitor member recovery source"); + } // CLI tunable overrides. config.wal_backend = args.wal_backend.clone(); @@ -158,6 +162,7 @@ async fn main() { )), ); + let _member_routes = crowdb_kv_server::recovery::member_routes::start(Arc::clone(®istry)); // Spawn a config file watcher for diff logging. Only when --config is // passed; with no toml there is nothing to watch. let watcher_old_config = Arc::new(std::sync::Mutex::new(config)); diff --git a/app/crowdb-kv-server/src/mgmt.rs b/app/crowdb-kv-server/src/mgmt.rs index 027bbd1f5..749a931cc 100644 --- a/app/crowdb-kv-server/src/mgmt.rs +++ b/app/crowdb-kv-server/src/mgmt.rs @@ -11,6 +11,7 @@ //! - [`topology`] — topology export + metrics endpoints mod discovery; +pub(crate) mod group_creation; mod group_ops; pub mod operation_registry; mod replica_ops; @@ -177,6 +178,7 @@ pub fn router(state: RegistryArc) -> Router { post(group_ops::wipe_user_data), ) .route("/stores/:sid/groups/:gid/ready", get(group_ops::group_readiness)) + .route("/stores/:sid/groups/:gid/creation", get(group_creation::status)) .route("/operations/:id", get(group_ops::get_operation)) .route("/topology", get(topology::export_topology)) .route("/top", get(topology::export_topology)) diff --git a/app/crowdb-kv-server/src/mgmt/group_creation.rs b/app/crowdb-kv-server/src/mgmt/group_creation.rs new file mode 100644 index 000000000..b55412479 --- /dev/null +++ b/app/crowdb-kv-server/src/mgmt/group_creation.rs @@ -0,0 +1,249 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Durable creation ownership precedes ordinary-group WAL and engine effects. + +use std::{ + fs::{self, File, OpenOptions}, + io::{Read, Write}, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::{Path, PathBuf}, +}; + +use axum::{ + extract::{Path as AxumPath, State}, + http::StatusCode, + Json, +}; +use crowdb_protocol::mgmt::{AddGroupRequest, GroupCreationStatus}; + +use super::{err_json, system_bootstrap::ManagementError, RegistryArc}; + +pub(super) async fn status( + State(state): State, + AxumPath((store, group)): AxumPath<(u64, u64)>, +) -> Result, ManagementError> { + read(&state.config.config_root, store, group)? + .map(Json) + .ok_or_else(|| err_json(StatusCode::NOT_FOUND, "group creation ownership is absent")) +} + +pub(super) async fn accept( + state: &RegistryArc, + store: u64, + request: &AddGroupRequest, +) -> Result<(), ManagementError> { + if store == 0 && request.group_id == 0 { + return if request.deployment.is_none() { + Ok(()) + } else { + Err(err_json( + StatusCode::BAD_REQUEST, + "system group must use system bootstrap ownership", + )) + }; + } + let expected = GroupCreationStatus { + version: 1, + store_id: store, + group_id: request.group_id, + replica_id: request.replica_id, + deployment: request.deployment.clone(), + retired: false, + }; + validate(&expected)?; + let root = &state.config.config_root; + if let Some(current) = read(root, store, request.group_id)? { + return matches(¤t, &expected); + } + if state + .get_store(store) + .is_some_and(|store| store.get_group(request.group_id).is_some()) + { + return if request.deployment.is_none() { + Ok(()) + } else { + Err(err_json( + StatusCode::CONFLICT, + "existing group has no matching deployment ownership", + )) + }; + } + let persisted = crowdb_kv::cluster::node_config::NodeConfigStore::new(root) + .load() + .await + .map_err(storage_error)?; + if persisted + .group(store, request.group_id) + .is_some_and(|group| group.replica_id != request.replica_id) + { + return Err(err_json( + StatusCode::CONFLICT, + "persisted group replica identity differs", + )); + } + if request.deployment.is_some() + && (persisted.group(store, request.group_id).is_some() + || crate::recovery::startup::store_crowdb_tree_path( + &state.config.data_root, + store, + request.group_id, + ) + .exists() + || crate::recovery::startup::store_wal_root(&state.config.wal_root, store) + .join(format!("group{}", request.group_id)) + .exists()) + { + return Err(err_json( + StatusCode::CONFLICT, + "existing group has no matching deployment ownership", + )); + } + fs::create_dir_all(root).map_err(storage_error)?; + publish(root, &expected, false)?; + let current = read(root, store, request.group_id)? + .ok_or_else(|| err_json(StatusCode::CONFLICT, "group creation ownership disappeared"))?; + matches(¤t, &expected) +} + +pub(super) fn retire(state: &RegistryArc, store: u64, group: u64) -> Result<(), ManagementError> { + if let Some(mut current) = read(&state.config.config_root, store, group)? { + if !current.retired { + current.retired = true; + publish(&state.config.config_root, ¤t, true)?; + } + } + Ok(()) +} + +pub(super) fn finish_removal(state: &RegistryArc, store: u64, group: u64) -> Result<(), ManagementError> { + let root = &state.config.config_root; + if read(root, store, group)?.is_some_and(|current| current.deployment.is_none() && current.retired) { + fs::remove_file(path(root, store, group)).map_err(storage_error)?; + File::open(root) + .and_then(|file| file.sync_all()) + .map_err(storage_error)?; + } + Ok(()) +} + +pub(crate) fn check_restore(root: &Path, store: u64, group: u64, replica: u64) -> std::io::Result<()> { + let status = read(root, store, group).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "invalid durable group creation ownership", + ) + })?; + if status.is_some_and(|status| status.retired || status.replica_id != replica) { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "group creation is retired or has another replica identity", + )); + } + Ok(()) +} + +fn matches(current: &GroupCreationStatus, expected: &GroupCreationStatus) -> Result<(), ManagementError> { + if current != expected { + return Err(err_json( + StatusCode::CONFLICT, + "group creation is retired or owned by another operation", + )); + } + Ok(()) +} + +fn path(root: &Path, store: u64, group: u64) -> PathBuf { + root.join(format!("group-creation-{store}-{group}.json")) +} + +fn read(root: &Path, store: u64, group: u64) -> Result, ManagementError> { + let path = path(root, store, group); + let metadata = match fs::symlink_metadata(&path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(storage_error(error)), + }; + if !metadata.file_type().is_file() + || metadata.len() > 4096 + || metadata.permissions().mode() & 0o777 != 0o600 + { + return Err(err_json( + StatusCode::CONFLICT, + "invalid durable group creation ownership", + )); + } + let mut bytes = Vec::new(); + File::open(path) + .map_err(storage_error)? + .take(4097) + .read_to_end(&mut bytes) + .map_err(storage_error)?; + if bytes.len() > 4096 { + return Err(err_json( + StatusCode::CONFLICT, + "group ownership exceeds its size bound", + )); + } + let status: GroupCreationStatus = serde_json::from_slice(&bytes) + .map_err(|_| err_json(StatusCode::CONFLICT, "invalid durable group creation ownership"))?; + validate(&status)?; + if status.store_id != store || status.group_id != group { + return Err(err_json( + StatusCode::CONFLICT, + "group creation ownership path differs", + )); + } + Ok(Some(status)) +} + +fn validate(status: &GroupCreationStatus) -> Result<(), ManagementError> { + let valid_identity = status.deployment.as_ref().map_or(true, |identity| { + [&identity.cluster_id, &identity.operation_id] + .iter() + .all(|value| { + uuid::Uuid::parse_str(value).is_ok_and(|id| !id.is_nil() && id.to_string() == value.as_str()) + }) + }); + if status.version != 1 || status.replica_id == 0 || !valid_identity { + return Err(err_json(StatusCode::CONFLICT, "invalid group creation identity")); + } + Ok(()) +} + +fn publish(root: &Path, status: &GroupCreationStatus, replace: bool) -> Result<(), ManagementError> { + let temporary = root.join(format!(".group-creation-{}.tmp", uuid::Uuid::new_v4())); + let result = (|| { + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(&serde_json::to_vec(status).map_err(std::io::Error::other)?)?; + file.sync_all()?; + let target = path(root, status.store_id, status.group_id); + if replace { + fs::rename(&temporary, target)?; + } else { + match fs::hard_link(&temporary, target) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + } + File::open(root)?.sync_all() + })(); + match fs::remove_file(temporary) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) if result.is_ok() => return Err(storage_error(error)), + Ok(()) | Err(_) => {} + } + result.map_err(storage_error) +} + +fn storage_error(error: impl std::fmt::Display) -> ManagementError { + err_json( + StatusCode::INTERNAL_SERVER_ERROR, + format!("group ownership persistence failed: {error}"), + ) +} diff --git a/app/crowdb-kv-server/src/mgmt/group_ops.rs b/app/crowdb-kv-server/src/mgmt/group_ops.rs index d0eb08d57..f0800c4b5 100644 --- a/app/crowdb-kv-server/src/mgmt/group_ops.rs +++ b/app/crowdb-kv-server/src/mgmt/group_ops.rs @@ -204,11 +204,9 @@ pub(super) async fn add_group( Path(sid): Path, Json(req): Json, ) -> Result)> { - let _system_execution = if sid == 0 && req.group_id == 0 { - Some(super::system_bootstrap::begin(&state)?) - } else { - None - }; + // Reuse the existing node-local initialization exclusion for ordinary + // creation too; duplicate runners may submit the same sealed operation. + let _execution = super::system_bootstrap::begin(&state)?; if sid == 0 && req.group_id == 0 && super::system_bootstrap::has_owner(&state)? { return Err(err_json( StatusCode::CONFLICT, @@ -219,6 +217,7 @@ pub(super) async fn add_group( .get_store(sid) .ok_or_else(|| err_json(StatusCode::NOT_FOUND, format!("store {sid} not found")))?; + super::group_creation::accept(&state, sid, &req).await?; if store.get_group(req.group_id).is_some() { return Err(err_json( StatusCode::CONFLICT, @@ -470,6 +469,7 @@ pub(super) async fn remove_group( Path((sid, gid)): Path<(u64, u64)>, headers: HeaderMap, ) -> Result)> { + let _execution = super::system_bootstrap::begin(&state)?; let store = state .get_store(sid) .ok_or_else(|| err_json(StatusCode::NOT_FOUND, format!("store {sid} not found")))?; @@ -493,6 +493,7 @@ pub(super) async fn remove_group( )); } info!(s = sid, g = gid, "removing PxGroup via management API"); + super::group_creation::retire(&state, sid, gid)?; if !store.remove_group(gid) { return Err(err_json( StatusCode::NOT_FOUND, @@ -523,6 +524,8 @@ pub(super) async fn remove_group( tracing::warn!(s = sid, g = gid, error = %e, "failed to update node_config; continuing"); } + super::group_creation::finish_removal(&state, sid, gid)?; + info!( s = sid, g = gid, diff --git a/app/crowdb-kv-server/src/mgmt/replica_ops.rs b/app/crowdb-kv-server/src/mgmt/replica_ops.rs index b04b48342..b1cc1bd6b 100644 --- a/app/crowdb-kv-server/src/mgmt/replica_ops.rs +++ b/app/crowdb-kv-server/src/mgmt/replica_ops.rs @@ -151,6 +151,7 @@ pub(super) async fn add_remote_replicas( format!("failed to persist membership before publication: {error}"), )); } + let _ = crate::recovery::member_routes::apply(&new_group, &state.config.config_root, sid); store.add_group(new_group); info!( @@ -270,6 +271,7 @@ pub(super) async fn remove_remote_replica( format!("failed to persist membership before publication: {error}"), )); } + let _ = crate::recovery::member_routes::apply(&new_group, &state.config.config_root, sid); store.add_group(new_group); info!( @@ -387,6 +389,7 @@ pub(super) async fn batch_add_remote_replicas( format!("failed to persist membership before publication: {error}"), )); } + let _ = crate::recovery::member_routes::apply(&new_group, &state.config.config_root, sid); store.add_group(new_group); info!( diff --git a/app/crowdb-kv-server/src/mgmt/system_bootstrap.rs b/app/crowdb-kv-server/src/mgmt/system_bootstrap.rs index a96df745c..4a5cef094 100644 --- a/app/crowdb-kv-server/src/mgmt/system_bootstrap.rs +++ b/app/crowdb-kv-server/src/mgmt/system_bootstrap.rs @@ -44,7 +44,7 @@ pub(super) fn begin(state: &RegistryArc) -> Result 1 && new_group.local_replica().current_term_snapshot() == 0 { + new_group.local_replica().become_follower(0); + } new_group } diff --git a/app/crowdb-kv-server/src/recovery/member_routes.rs b/app/crowdb-kv-server/src/recovery/member_routes.rs new file mode 100644 index 000000000..d4fab9d4a --- /dev/null +++ b/app/crowdb-kv-server/src/recovery/member_routes.rs @@ -0,0 +1,220 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Monitor-owned endpoint evidence changes connections, never membership. + +use crate::store_registry::KvStoreRegistry; +use crowdb_kv::cluster::group::PxGroup; +use crowdb_protocol::mgmt::{ + node::recovery::{MemberEndpoint, MemberEndpoints, RecoveryGroups}, + GroupCreationStatus, SystemBootstrapIdentity, +}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeMap, + fs::{self, File}, + io::{self, Read, Write}, + net::SocketAddr, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Source { + root: PathBuf, + boot_id: String, +} + +#[derive(Deserialize)] +struct Accepted { + node_id: u64, + bootstrap: SystemBootstrapIdentity, +} + +#[derive(Deserialize)] +struct BootstrapOwner { + identity: SystemBootstrapIdentity, + replica_id: u64, +} + +/// # Errors +/// Rejects invalid local monitor state and private-file publication failures. +pub fn configure(config_root: &Path, root: &Path) -> io::Result<()> { + let endpoint: MemberEndpoint = read(&root.join("node-endpoint.json"))?; + fs::create_dir_all(config_root)?; + let source = Source { + root: root.to_path_buf(), + boot_id: endpoint.boot_id, + }; + let temporary = config_root.join(format!(".member-source-{}.tmp", uuid::Uuid::new_v4())); + let mut file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(&serde_json::to_vec(&source)?)?; + file.sync_all()?; + fs::rename(temporary, config_root.join("member-route-source.json"))?; + File::open(config_root)?.sync_all() +} + +/// Fence old addresses before this group can start its election driver. +/// +/// # Errors +/// Rejects corrupt local recovery ownership; absent evidence leaves routes blocked. +pub fn apply(group: &PxGroup, config_root: &Path, store_id: u64) -> io::Result<()> { + let source: Source = match read(&config_root.join("member-route-source.json")) { + Ok(source) => source, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => { + group.set_recovery_routes(&BTreeMap::new()); + return Err(error); + } + }; + let routes = resolve(group, config_root, store_id, &source); + match routes { + Ok(routes) => { + group.set_recovery_routes(&routes); + Ok(()) + } + Err(error) => { + group.set_recovery_routes(&BTreeMap::new()); + Err(error) + } + } +} + +fn resolve( + group: &PxGroup, + config_root: &Path, + store: u64, + source: &Source, +) -> io::Result> { + let accepted: Accepted = read(&source.root.join("accepted-node.json"))?; + let groups: RecoveryGroups = read(&source.root.join("recovery-groups.json"))?; + if groups.bootstrap != accepted.bootstrap { + return Err(invalid("recovery ownership differs")); + } + let mapping = groups + .groups + .iter() + .find(|owner| owner.store_id == store && owner.group_id == group.group_id()) + .ok_or_else(|| invalid("replica ownership is not available for recovery"))?; + if !mapping + .members + .iter() + .any(|member| member.replica_id == group.local_replica().id && member.node_id == accepted.node_id) + { + return Err(invalid("local recovery replica identity differs")); + } + if store == 0 && group.group_id() == 0 { + let owner: BootstrapOwner = read(&config_root.join("system-bootstrap.json"))?; + if owner.identity != accepted.bootstrap || owner.replica_id != group.local_replica().id { + return Err(invalid("system recovery owner differs")); + } + } else { + match read::( + &config_root.join(format!("group-creation-{store}-{}.json", group.group_id())), + ) { + Ok(owner) + if !owner.retired + && owner.deployment == mapping.deployment + && owner.replica_id == group.local_replica().id => {} + Err(error) if error.kind() == io::ErrorKind::NotFound && mapping.deployment.is_none() => {} + _ => return Err(invalid("group recovery owner differs")), + } + } + let endpoints: MemberEndpoints = match read(&source.root.join("member-endpoints.json")) { + Ok(endpoints) => endpoints, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(BTreeMap::new()), + Err(error) => return Err(error), + }; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(io::Error::other)? + .as_millis(); + if endpoints.version != 1 + || endpoints.bootstrap != accepted.bootstrap + || endpoints.observer_boot_id != source.boot_id + || now.saturating_sub(u128::from(endpoints.observed_at_unix_ms)) > 15000 + || u128::from(endpoints.observed_at_unix_ms) > now + 1000 + || !endpoints + .members + .iter() + .any(|member| member.node_id == accepted.node_id && member.boot_id == source.boot_id) + { + return Ok(BTreeMap::new()); + } + let mut routes = BTreeMap::new(); + for (replica, endpoint, _) in group.remote_replica_info() { + let Some(owner) = mapping.members.iter().find(|member| member.replica_id == replica) else { + continue; + }; + let Some(member) = endpoints.members.iter().find(|member| { + member.node_id == owner.node_id && !endpoints.conflicts.contains(&member.discovery_id) + }) else { + continue; + }; + let port = endpoint + .trim_start_matches("http://") + .trim_start_matches("https://") + .parse::() + .map_err(|_| invalid("invalid original replica endpoint"))? + .port(); + let Some(address) = member + .addresses + .iter() + .find(|address| address.is_ipv4()) + .or_else(|| member.addresses.first()) + else { + continue; + }; + if address.is_unspecified() || address.is_multicast() { + return Err(invalid("invalid verified address")); + } + routes.insert(replica, SocketAddr::new(*address, port)); + } + Ok(routes) +} + +pub struct Worker(tokio::task::JoinHandle<()>); +impl Drop for Worker { + fn drop(&mut self) { + self.0.abort(); + } +} + +#[must_use] +pub fn start(registry: Arc) -> Worker { + Worker(tokio::spawn(async move { + let mut tick = tokio::time::interval(Duration::from_secs(1)); + loop { + tick.tick().await; + for (store_id, store) in registry.stores_snapshot().iter() { + for group_id in store.group_ids() { + if let Some(group) = store.get_group(group_id) { + if let Err(error) = apply(&group, ®istry.config.config_root, *store_id) { + tracing::debug!(store_id, group_id, %error, "member connection routes remain blocked"); + } + } + } + } + } + })) +} + +fn read(path: &Path) -> io::Result { + let metadata = fs::symlink_metadata(path)?; + if !metadata.is_file() || metadata.permissions().mode() & 0o777 != 0o600 || metadata.len() > 1024 * 1024 { + return Err(invalid("invalid private recovery record")); + } + let mut bytes = Vec::new(); + File::open(path)?.take(1024 * 1024 + 1).read_to_end(&mut bytes)?; + serde_json::from_slice(&bytes).map_err(io::Error::other) +} +fn invalid(message: &str) -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, message) +} diff --git a/app/crowdb-kv-server/src/recovery/reconcile.rs b/app/crowdb-kv-server/src/recovery/reconcile.rs index 041764b91..6e55c37df 100644 --- a/app/crowdb-kv-server/src/recovery/reconcile.rs +++ b/app/crowdb-kv-server/src/recovery/reconcile.rs @@ -184,6 +184,11 @@ fn execute_reconcile(plan: &[ReconcileAction], registry: &KvStoreRegistry) { if let Some(store) = registry.get_store(action.store_id) { if let Some(group) = store.get_group(action.group_id) { let new_group = rebuild_group_with_new_remotes(&group, &action.seed_remotes); + let _ = super::member_routes::apply( + &new_group, + ®istry.config.config_root, + action.store_id, + ); store.add_group(new_group); seeded += 1; info!( diff --git a/app/crowdb-kv-server/src/recovery/startup.rs b/app/crowdb-kv-server/src/recovery/startup.rs index 7c18c5c9b..93e6756b4 100644 --- a/app/crowdb-kv-server/src/recovery/startup.rs +++ b/app/crowdb-kv-server/src/recovery/startup.rs @@ -178,6 +178,7 @@ pub async fn create_group_with_wal( wal_backend: Arc, crowtree_backend: CrowdbTreeBackend, ) -> io::Result { + crate::mgmt::group_creation::check_restore(&config.config_root, store_id, group_id, replica_id)?; persist_creation_identity(&config.config_root, store_id, group_id, replica_id).await?; let mut wal_config = WalConfig::with_root(store_wal_root(&config.wal_root, store_id)); if std::env::var("CROWDB_KV_WAL_TEXT").as_deref() == Ok("1") { @@ -231,5 +232,8 @@ pub async fn create_group_with_wal( .saturating_add(1) .max(1); group.set_next_slot(next_slot); + if let Err(error) = super::member_routes::apply(&group, &config.config_root, store_id) { + tracing::debug!(store_id, group_id, %error, "restored member routes are blocked pending verification"); + } Ok(group) } diff --git a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs index c8fb91ac2..6cd9f43ec 100644 --- a/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs +++ b/app/crowdb-kv-server/tests/chunk_kv_balance_planning_test.rs @@ -22,6 +22,66 @@ struct TestBalancePlanning { descriptor: DomainMonitorDescriptor, } +#[tokio::test] +async fn dead_owner_transfer_cannot_fence_an_active_split_parent() { + use crowdb_protocol::chunk_kv::{SplitPhase, SplitTransition}; + use crowdb_protocol::common::InstanceValue; + use crowdb_protocol::key::InstanceKey; + let fixture = TestBalancePlanning::new().await; + balance_planning::recent_split(&fixture.control).await; + let rows = fixture + .control + .scan_all_prefix(Bytes::from(ChunkKvSplitKey::text_prefix_all()), 16) + .await + .unwrap(); + let mut split: SplitTransition = serde_json::from_slice(&rows[0].value).unwrap(); + split.phase = SplitPhase::ParentPreparing; + split.failure = None; + split.validate().unwrap(); + fixture + .control + .compare_and_put( + rows[0].key.clone(), + Bytes::from(serde_json::to_vec(&split).unwrap()), + rows[0].revision, + ) + .await + .unwrap(); + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 1, + } + .to_path(); + let row = fixture.control.get(path.as_bytes()).await.unwrap(); + let mut instance: InstanceValue = serde_json::from_slice(row.value.as_deref().unwrap()).unwrap(); + instance.last_heartbeat_ms = 0; + fixture + .control + .compare_and_put( + Bytes::from(path), + Bytes::from(serde_json::to_vec(&instance).unwrap()), + row.revision, + ) + .await + .unwrap(); + fixture.tick().await; + assert!( + fixture + .control + .scan_all_prefix(Bytes::from(ChunkKvTransferKey::text_prefix_all()), 16,) + .await + .unwrap() + .is_empty(), + "a competing transfer would fence the recovering split's WAL" + ); + let persisted = fixture.control.get(&rows[0].key).await.unwrap(); + let persisted: SplitTransition = serde_json::from_slice(persisted.value.as_deref().unwrap()).unwrap(); + assert_eq!( + persisted, split, + "unavailability must not abort or rewrite the split proof" + ); +} + #[tokio::test] async fn recent_split_attempt_does_not_postpone_first_placement() { let fixture = TestBalancePlanning::new().await; diff --git a/app/crowdb-kv-server/tests/common/balance_distribution.rs b/app/crowdb-kv-server/tests/common/balance_distribution.rs index 0fc69f58b..875077518 100644 --- a/app/crowdb-kv-server/tests/common/balance_distribution.rs +++ b/app/crowdb-kv-server/tests/common/balance_distribution.rs @@ -56,6 +56,7 @@ pub async fn seed(control: &Group0ControlPlane, distribution: &[(u64, u64)], gen ]; for owner in [1, 2] { let mut extra = ChunkKvExtra { + process_epoch: 1, capacity_bytes: 10_000, ..Default::default() }; diff --git a/app/crowdb-kv-server/tests/common/balance_planning.rs b/app/crowdb-kv-server/tests/common/balance_planning.rs index 8c338be66..d98251c4b 100644 --- a/app/crowdb-kv-server/tests/common/balance_planning.rs +++ b/app/crowdb-kv-server/tests/common/balance_planning.rs @@ -93,6 +93,7 @@ async fn seed_instances(control: &Group0ControlPlane, partition_ids: &[Id128]) { ( 1, ChunkKvExtra { + process_epoch: 1, capacity_bytes: 1_000, durable_bytes: 200, request_rate: 0, @@ -111,6 +112,7 @@ async fn seed_instances(control: &Group0ControlPlane, partition_ids: &[Id128]) { ( 2, ChunkKvExtra { + process_epoch: 1, capacity_bytes: 1_000, ..ChunkKvExtra::default() }, @@ -175,7 +177,7 @@ pub async fn recent_split(control: &Group0ControlPlane) { parent_epoch: 1, parent_next_epoch: 2, parent_artifact: artifact(50), - retained_parent_artifact: artifact(60), + retained_parent_artifact: artifact(50), split_key: b"f".to_vec(), child: SplitChildAssignment { partition_id: Id128 { high: 91, low: 2 }, diff --git a/app/crowdb-kv-server/tests/common/catalog_publication.rs b/app/crowdb-kv-server/tests/common/catalog_publication.rs index b3afd76a9..89f108060 100644 --- a/app/crowdb-kv-server/tests/common/catalog_publication.rs +++ b/app/crowdb-kv-server/tests/common/catalog_publication.rs @@ -12,11 +12,18 @@ struct TestPublication { #[tokio::test] async fn expired_lease_transfer_publishes_directly_from_serving_source() { - for prior_marker in [None, Some(Id128 { high: 88, low: 99 })] { + for (prior_marker, retained_overlay) in [ + (None, false), + (Some(Id128 { high: 88, low: 99 }), false), + (Some(Id128 { high: 88, low: 99 }), true), + ] { let (control, mut ready) = assert_ready_transfer_publication().await; ready.release_proof = Some(AuthorityReleaseProof::LeaseExpired { activation_not_before_ms: 2, }); + if retained_overlay { + ready.artifact.tail_overlay = ready.target_artifact.tail_overlay.clone(); + } ready.target_artifact = ready.artifact.clone(); ready.readiness_proof.as_mut().unwrap().artifact = ready.artifact.clone(); ready.catchup_proof.as_mut().unwrap().artifact = ready.artifact.clone(); @@ -86,6 +93,17 @@ async fn expired_lease_transfer_publishes_directly_from_serving_source() { .await; assert_eq!(grant.catalog_generation, published.generation); assert_eq!(grant.assignments[0].owner_epoch, ready.target_epoch); + let page: ChunkKvRangeCatalogPage = get_json( + &control, + &ChunkKvRangeCatalogPageKey { + generation: published.pages[0].page_generation, + page_index: 0, + } + .to_path(), + ) + .await; + assert_eq!(page.entries[0].artifact, ready.artifact); + assert_eq!(page.entries[0].owner, ready.target); } } diff --git a/app/crowdb-kv-server/tests/common/endpoint_publication.rs b/app/crowdb-kv-server/tests/common/endpoint_publication.rs new file mode 100644 index 000000000..c87bfe346 --- /dev/null +++ b/app/crowdb-kv-server/tests/common/endpoint_publication.rs @@ -0,0 +1,182 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; + +#[tokio::test] +async fn recovered_owner_address_advances_catalog_without_transferring_data() { + let control = endpoint_fixture().await; + let head: ChunkKvRangeCatalogHead = get_json(&control, &ChunkKvRangeCatalogHeadKey.to_path()).await; + let page_path = ChunkKvRangeCatalogPageKey { + generation: head.pages[0].page_generation, + page_index: head.pages[0].page_index, + } + .to_path(); + let page: ChunkKvRangeCatalogPage = get_json(&control, &page_path).await; + let prior = page.entries[0].clone(); + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: prior.owner.instance_id, + } + .to_path(); + let value = control.get(path.as_bytes()).await.unwrap(); + let mut instance: InstanceValue = serde_json::from_slice(value.value.as_ref().unwrap()).unwrap(); + instance.rpc_endpoint = "192.0.2.91:7401".into(); + instance.extra.as_mut().unwrap().chunk_kv.as_mut().unwrap().hosted = vec![HostedPartition { + partition_id: prior.partition_id, + owner_epoch: prior.owner_epoch, + recovering: true, + }]; + control + .compare_and_put( + Bytes::from(path), + Bytes::from(serde_json::to_vec(&instance).unwrap()), + value.revision, + ) + .await + .unwrap(); + let mut descriptor = descriptor(); + descriptor.domain = "chunk-kv".into(); + descriptor.service_registry_name = "chunk-kv".into(); + descriptor.failure_policy = DomainFailurePolicy::OperatorOnly; + ChunkKvRangeMonitorDriver::new() + .tick(&control, &descriptor) + .await + .unwrap(); + let current: ChunkKvRangeCatalogHead = get_json(&control, &ChunkKvRangeCatalogHeadKey.to_path()).await; + assert_eq!(current.generation, head.generation + 1); + let updated: ChunkKvRangeCatalogPage = get_json( + &control, + &ChunkKvRangeCatalogPageKey { + generation: current.pages[0].page_generation, + page_index: current.pages[0].page_index, + } + .to_path(), + ) + .await; + let mut expected = prior; + expected.owner.rpc_endpoint = instance.rpc_endpoint; + assert_eq!(updated.entries, vec![expected]); + let retained: ChunkKvRangeCatalogPage = get_json(&control, &page_path).await; + assert_eq!(retained, page, "old immutable pages remain intact"); + ChunkKvRangeMonitorDriver::new() + .tick(&control, &descriptor) + .await + .unwrap(); + let repeated: ChunkKvRangeCatalogHead = get_json(&control, &ChunkKvRangeCatalogHeadKey.to_path()).await; + assert_eq!( + repeated, current, + "same address does not churn catalog generations" + ); +} + +#[tokio::test] +async fn unproven_owner_address_cannot_rewrite_the_catalog() { + for case in ["expired", "wrong-epoch", "unhosted"] { + let control = endpoint_fixture().await; + let head: ChunkKvRangeCatalogHead = get_json(&control, &ChunkKvRangeCatalogHeadKey.to_path()).await; + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 1, + } + .to_path(); + let record = control.get(path.as_bytes()).await.unwrap(); + let mut instance: InstanceValue = serde_json::from_slice(record.value.as_ref().unwrap()).unwrap(); + instance.rpc_endpoint = "192.0.2.92:7401".into(); + let hosted = &mut instance.extra.as_mut().unwrap().chunk_kv.as_mut().unwrap().hosted; + match case { + "expired" => instance.last_heartbeat_ms = 0, + "wrong-epoch" => hosted[0].owner_epoch += 1, + "unhosted" => hosted.clear(), + _ => unreachable!(), + } + control + .compare_and_put( + Bytes::from(path), + Bytes::from(serde_json::to_vec(&instance).unwrap()), + record.revision, + ) + .await + .unwrap(); + let mut descriptor = descriptor(); + descriptor.domain = "chunk-kv".into(); + descriptor.service_registry_name = "chunk-kv".into(); + descriptor.failure_policy = DomainFailurePolicy::OperatorOnly; + ChunkKvRangeMonitorDriver::new() + .tick(&control, &descriptor) + .await + .unwrap(); + let observed: ChunkKvRangeCatalogHead = + get_json(&control, &ChunkKvRangeCatalogHeadKey.to_path()).await; + assert_eq!(observed, head, "{case} observation is not authority to rebind"); + } +} + +async fn endpoint_fixture() -> Group0ControlPlane { + let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); + let control = Group0ControlPlane::acquire(&store).await.unwrap(); + let mut page = ChunkKvRangeCatalogPage { + generation: 1, + page_index: 0, + checksum: [0; 32], + entries: vec![ChunkKvRangeCatalogEntry { + partition_id: Id128 { high: 1, low: 2 }, + range: KeyRange { + start: vec![], + end: None, + }, + owner: OwnerDescriptor { + instance_id: 1, + rpc_endpoint: "127.0.0.1:17001".into(), + }, + owner_epoch: 1, + state: ChunkKvRangeCatalogPartitionState::Serving, + artifact: PartitionArtifact { + tree_id: 3, + stream_name: StreamName { high: 4, low: 5 }, + tail_overlay: None, + }, + transition_id: None, + }], + }; + page.seal().unwrap(); + let mut head = ChunkKvRangeCatalogHead { + generation: 1, + previous_generation: None, + checksum: [0; 32], + pages: vec![ChunkKvRangeCatalogPageRef { + page_generation: 1, + page_index: 0, + first_key: vec![], + page_checksum: page.checksum, + }], + }; + head.seal().unwrap(); + put_json( + &control, + ChunkKvRangeCatalogPageKey { + generation: 1, + page_index: 0, + } + .to_path(), + &page, + ) + .await; + put_json(&control, ChunkKvRangeCatalogHeadKey.to_path(), &head).await; + put_chunk_kv_instance( + &control, + 1, + ChunkKvExtra { + process_epoch: 1, + hosted: vec![HostedPartition { + partition_id: page.entries[0].partition_id, + owner_epoch: 1, + recovering: false, + }], + ..ChunkKvExtra::default() + }, + ) + .await; + control +} diff --git a/app/crowdb-kv-server/tests/common/grant_process.rs b/app/crowdb-kv-server/tests/common/grant_process.rs new file mode 100644 index 000000000..52304f0d9 --- /dev/null +++ b/app/crowdb-kv-server/tests/common/grant_process.rs @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_kv_server::background::domain_monitor::ChunkKvRangeMonitorDriver; +use crowdb_protocol::{ + chunk_stream::{StreamProcessLease, StreamPurpose}, + key::StreamProcessLeaseKey, +}; + +async fn replace_json(control: &Group0ControlPlane, path: String, value: &T) { + let revision = control.get(path.as_bytes()).await.unwrap().revision; + control + .compare_and_put( + Bytes::from(path), + Bytes::from(serde_json::to_vec(value).unwrap()), + revision, + ) + .await + .unwrap(); +} + +#[tokio::test] +async fn grant_and_process_expiry_fence_publish_together_and_reject_stale_observations() { + let (control, _) = assert_ready_transfer_publication().await; + let path = ServingGrantKey { instance_id: 2 }.to_path(); + let grant: ServingGrant = get_json(&control, &path).await; + let lease_path = StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id: 2, + } + .to_path(); + let mut lease: StreamProcessLease = get_json(&control, &lease_path).await; + assert_eq!(lease.grant_sequence, grant.lease_sequence); + assert!(lease.grant_expires_at_ms >= grant.expires_at_ms); + assert_eq!( + control.get(path.as_bytes()).await.unwrap().revision, + control.get(lease_path.as_bytes()).await.unwrap().revision + ); + lease.identity.process_epoch += 1; + lease.identity.boot_nonce = [2; 16]; + lease.prior_exclusion_deadline_ms = lease.expires_at_ms; + replace_json(&control, lease_path.clone(), &lease).await; + let mut policy = descriptor(); + policy.domain = "chunk-kv".into(); + policy.service_registry_name = "chunk-kv".into(); + policy.failure_policy = DomainFailurePolicy::OperatorOnly; + ChunkKvRangeMonitorDriver::new() + .tick(&control, &policy) + .await + .unwrap(); + let unchanged: ServingGrant = get_json(&control, &path).await; + assert_eq!( + unchanged, grant, + "old observation cannot grant the replacement boot" + ); + let instance_path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 2, + } + .to_path(); + let mut instance: InstanceValue = get_json(&control, &instance_path).await; + instance + .extra + .as_mut() + .unwrap() + .chunk_kv + .as_mut() + .unwrap() + .process_epoch = lease.identity.process_epoch; + replace_json(&control, instance_path, &instance).await; + ChunkKvRangeMonitorDriver::new() + .tick(&control, &policy) + .await + .unwrap(); + let current: ServingGrant = get_json(&control, &path).await; + assert_eq!(current.process_epoch, 2); + assert!(current.lease_sequence > grant.lease_sequence); + let accepted: StreamProcessLease = get_json(&control, &lease_path).await; + assert_eq!(accepted.grant_sequence, current.lease_sequence); + assert!(accepted.grant_expires_at_ms >= current.expires_at_ms); +} diff --git a/app/crowdb-kv-server/tests/common/group_creation.rs b/app/crowdb-kv-server/tests/common/group_creation.rs new file mode 100644 index 000000000..f27d8f227 --- /dev/null +++ b/app/crowdb-kv-server/tests/common/group_creation.rs @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv::{ + cluster::{KvServer, PxKvStore}, + common::config::CrowDBConfig, +}; +use crowdb_kv_server::{ + mgmt::{self, operation_registry::AppState}, + store_registry::KvStoreRegistry, +}; +use serde_json::{json, Value}; +use std::{path::PathBuf, sync::Arc}; + +pub struct TestGroupCreation { + _root: crowdb_test_harness::test_dirs::TestDir, + pub registry: Arc, + pub store: Arc, + pub origin: String, + server: tokio::task::JoinHandle<()>, +} + +impl TestGroupCreation { + pub async fn start(block_data: bool) -> Self { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("native-group-creation"); + let config = CrowDBConfig { + data_root: root.path().join("data"), + wal_root: root.path().join("wal"), + config_root: root.path().join("config"), + crowtree_backend: "file".into(), + wal_backend: "file".into(), + ..CrowDBConfig::for_tests() + }; + if block_data { + std::fs::write(&config.data_root, b"blocked").unwrap(); + } + let registry = Arc::new(KvStoreRegistry::with_config(config)); + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + store.start().await.unwrap(); + registry.add_store(0, &store); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let router = mgmt::router(AppState::new(Arc::clone(®istry))); + let server = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + Self { + _root: root, + registry, + store, + origin, + server, + } + } + + pub fn ownership(&self) -> PathBuf { + self.registry.config.config_root.join("group-creation-0-1.json") + } + + pub async fn request( + &self, + method: reqwest::Method, + path: &str, + body: Option<&Value>, + ) -> (reqwest::StatusCode, Vec) { + let client = reqwest::Client::builder() + .no_proxy() + .timeout(std::time::Duration::from_secs(5)) + .build() + .unwrap(); + let mut request = client.request(method, format!("{}{path}", self.origin)); + if let Some(body) = body { + request = request.json(body); + } + let response = request.send().await.unwrap(); + let status = response.status(); + let bytes = response.bytes().await.unwrap().to_vec(); + (status, bytes) + } + + pub async fn create(&self, body: &Value, expected: reqwest::StatusCode) { + let (status, bytes) = self + .request(reqwest::Method::POST, "/stores/0/groups", Some(body)) + .await; + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&bytes)); + } +} + +impl Drop for TestGroupCreation { + fn drop(&mut self) { + self.server.abort(); + self.store.stop(); + } +} + +pub fn request() -> Value { + json!({"group_id":1,"replica_id":7,"initial_role":"leader","start_election":false, + "deployment":{"cluster_id":uuid::Uuid::new_v4().to_string(),"operation_id":uuid::Uuid::new_v4().to_string()}}) +} diff --git a/app/crowdb-kv-server/tests/common/process_lease.rs b/app/crowdb-kv-server/tests/common/process_lease.rs new file mode 100644 index 000000000..f867005a8 --- /dev/null +++ b/app/crowdb-kv-server/tests/common/process_lease.rs @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_kv_server::group0_control_plane::Group0ControlPlane; +use crowdb_protocol::{ + chunk_stream::{StreamProcessIdentity, StreamProcessLease, StreamPurpose}, + key::{StreamProcessLeaseKey, TextKey}, +}; + +pub struct TestProcessLease; + +impl TestProcessLease { + pub fn idle_instance(instance_id: u64, endpoint: &str) -> crowdb_protocol::common::InstanceValue { + use crowdb_protocol::common::{ChunkKvExtra, InstanceValue, ServiceExtra}; + InstanceValue { + instance_id, + rpc_endpoint: endpoint.into(), + last_heartbeat_ms: u64::MAX, + extra: Some(ServiceExtra { + chunk_kv: Some(ChunkKvExtra { + process_epoch: 1, + ..ChunkKvExtra::default() + }), + ..ServiceExtra::default() + }), + } + } + + pub async fn seed(control: &Group0ControlPlane, instance_id: u64) { + let lease = StreamProcessLease { + identity: StreamProcessIdentity { + purpose: StreamPurpose::Wal, + instance_id, + process_epoch: 1, + boot_nonce: [1; 16], + }, + expires_at_ms: u64::MAX, + prior_exclusion_deadline_ms: 0, + grant_sequence: 0, + grant_expires_at_ms: 0, + }; + control + .compare_and_put( + Bytes::from( + StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id, + } + .to_path(), + ), + Bytes::from(serde_json::to_vec(&lease).unwrap()), + 0, + ) + .await + .unwrap(); + } +} diff --git a/app/crowdb-kv-server/tests/common/system_bootstrap.rs b/app/crowdb-kv-server/tests/common/system_bootstrap.rs new file mode 100644 index 000000000..5bcad0b50 --- /dev/null +++ b/app/crowdb-kv-server/tests/common/system_bootstrap.rs @@ -0,0 +1,127 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv::common::config::CrowDBConfig; +use crowdb_kv_server::{ + mgmt::{self, operation_registry::AppState}, + store_registry::KvStoreRegistry, +}; +use crowdb_protocol::mgmt::{SystemBootstrapIdentity, SystemPrepareRequest}; +use reqwest::StatusCode; +use serde_json::Value; +use std::{path::PathBuf, sync::Arc, time::Duration}; + +pub struct TestSystemBootstrap { + _root: crowdb_test_harness::test_dirs::TestDir, + pub registry: Arc, + pub origin: String, + server: tokio::task::JoinHandle<()>, +} + +impl TestSystemBootstrap { + pub async fn start() -> Self { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("system-prepare"); + let config = CrowDBConfig { + data_root: root.path().join("data"), + wal_root: root.path().join("wal"), + config_root: root.path().join("config"), + ..CrowDBConfig::for_tests() + }; + let registry = Arc::new(KvStoreRegistry::with_config(config)); + let (origin, server) = Self::serve(®istry).await; + Self { + _root: root, + registry, + origin, + server, + } + } + + async fn serve(registry: &Arc) -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let router = mgmt::router(AppState::new(registry.clone())); + let server = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + (origin, server) + } + + pub async fn restart(&mut self) { + self.server.abort(); + self.registry = Arc::new(KvStoreRegistry::with_config(self.registry.config.clone())); + (self.origin, self.server) = Self::serve(&self.registry).await; + } + + pub fn ownership(&self) -> PathBuf { + self.registry.config.config_root.join("system-bootstrap.json") + } + + pub async fn request(&self, path: &str, body: &Value) -> StatusCode { + let response = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(3)) + .build() + .unwrap() + .post(format!("{}{path}", self.origin)) + .json(body) + .send() + .await + .unwrap(); + let status = response.status(); + let bytes = response.bytes().await.unwrap(); + assert!( + matches!(status, StatusCode::OK | StatusCode::CONFLICT), + "{path}: {status}: {}", + String::from_utf8_lossy(&bytes) + ); + status + } + + pub async fn prepare(&self, replica_id: u64, bootstrap: &SystemBootstrapIdentity) -> StatusCode { + self.request( + "/system/prepare", + &serde_json::to_value(SystemPrepareRequest { + replica_id, + bootstrap: bootstrap.clone(), + }) + .unwrap(), + ) + .await + } + + pub fn assert_no_execution(&self) { + assert_eq!(self.registry.store_count(), 0); + assert!(!self.registry.config.data_root.exists()); + assert!(!self.registry.config.wal_root.exists()); + } +} + +impl Drop for TestSystemBootstrap { + fn drop(&mut self) { + self.server.abort(); + } +} + +pub fn identity() -> SystemBootstrapIdentity { + SystemBootstrapIdentity { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + configuration_digest: "a".repeat(64), + } +} + +pub async fn proposal( + nodes: &[TestSystemBootstrap], + selected: [usize; 3], + identity: &SystemBootstrapIdentity, +) -> bool { + let (first, second, third) = tokio::join!( + nodes[selected[0]].prepare(1, identity), + nodes[selected[1]].prepare(2, identity), + nodes[selected[2]].prepare(3, identity) + ); + [first, second, third] + .iter() + .all(|status| *status == StatusCode::OK) +} diff --git a/app/crowdb-kv-server/tests/domain_monitor_test.rs b/app/crowdb-kv-server/tests/domain_monitor_test.rs index 2f1053c46..75a0ce721 100644 --- a/app/crowdb-kv-server/tests/domain_monitor_test.rs +++ b/app/crowdb-kv-server/tests/domain_monitor_test.rs @@ -29,8 +29,15 @@ use crowdb_protocol::key::{ DomainMonitorKey, InstanceKey, ServingGrantKey, TextKey, }; +#[path = "common/grant_process.rs"] +mod grant_process; +#[path = "common/process_lease.rs"] +mod test_process; + #[path = "common/catalog_publication.rs"] mod catalog_publication; +#[path = "common/endpoint_publication.rs"] +mod endpoint_publication; struct CountingDriver { ticks: Arc, @@ -201,6 +208,8 @@ async fn assert_ready_transfer_publication() -> (Group0ControlPlane, TransferTra let (_, store) = registry_with_group_zero(PxLocalReplicaRole::Leader); let control = Group0ControlPlane::acquire(&store).await.unwrap(); + test_process::TestProcessLease::seed(&control, 1).await; + test_process::TestProcessLease::seed(&control, 2).await; let partition_id = Id128 { high: 1, low: 1 }; let transition_id = Id128 { high: 9, low: 9 }; let artifact = PartitionArtifact { @@ -264,7 +273,7 @@ async fn assert_ready_transfer_publication() -> (Group0ControlPlane, TransferTra source_partition_id: partition_id, source_epoch: 1, source_stream_name: artifact.stream_name, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 7, base_root_manifest_generation: 1, @@ -325,6 +334,7 @@ async fn assert_ready_transfer_publication() -> (Group0ControlPlane, TransferTra last_heartbeat_ms: u64::MAX, extra: Some(ServiceExtra { chunk_kv: Some(ChunkKvExtra { + process_epoch: 1, hosted: vec![HostedPartition { partition_id, owner_epoch: 1, @@ -355,6 +365,7 @@ async fn assert_ready_transfer_publication() -> (Group0ControlPlane, TransferTra last_heartbeat_ms: u64::MAX, extra: Some(ServiceExtra { chunk_kv: Some(ChunkKvExtra { + process_epoch: 1, hosted: vec![HostedPartition { partition_id, owner_epoch: 2, @@ -501,15 +512,7 @@ async fn chunk_kv_driver_plans_dead_owner_recovery_once_and_waits_for_target() { instance_id: 2, } .to_path(), - &InstanceValue { - instance_id: 2, - rpc_endpoint: "127.0.0.1:17002".into(), - last_heartbeat_ms: u64::MAX, - extra: Some(ServiceExtra { - chunk_kv: Some(ChunkKvExtra::default()), - ..ServiceExtra::default() - }), - }, + &test_process::TestProcessLease::idle_instance(2, "127.0.0.1:17002"), ) .await; let mut policy = descriptor(); @@ -614,6 +617,7 @@ async fn assert_chunk_kv_split_plan( last_heartbeat_ms: u64::MAX, extra: Some(ServiceExtra { chunk_kv: Some(ChunkKvExtra { + process_epoch: 1, capacity_bytes: 1_000, durable_bytes: 600, request_rate: 0, @@ -677,7 +681,7 @@ async fn assert_chunk_kv_split_plan( source_partition_id: prepared.parent_id, source_epoch: prepared.parent_epoch, source_stream_name: prepared.parent_artifact.stream_name, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 10, base_root_manifest_generation: 1, @@ -698,7 +702,7 @@ async fn assert_chunk_kv_split_plan( child_applied_seq: 11, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay: None, + child_tail_overlay: overlay, }); prepared.validate().unwrap(); @@ -851,6 +855,7 @@ async fn put_balance_fixture(control: &Group0ControlPlane) { ( 1, ChunkKvExtra { + process_epoch: 1, capacity_bytes: 1_000, durable_bytes: 200, request_rate: 0, @@ -869,6 +874,7 @@ async fn put_balance_fixture(control: &Group0ControlPlane) { ( 2, ChunkKvExtra { + process_epoch: 1, capacity_bytes: 1_000, ..ChunkKvExtra::default() }, diff --git a/app/crowdb-kv-server/tests/group_creation_test.rs b/app/crowdb-kv-server/tests/group_creation_test.rs new file mode 100644 index 000000000..ae4b09aba --- /dev/null +++ b/app/crowdb-kv-server/tests/group_creation_test.rs @@ -0,0 +1,215 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/group_creation.rs"] +mod common; +#[path = "common/system_bootstrap.rs"] +mod system_bootstrap; + +use common::{request, TestGroupCreation}; +use crowdb_kv::cluster::{group::PxGroup, PxLocalReplica, PxLocalReplicaRole}; +use reqwest::{Method, StatusCode}; +use serde_json::Value; + +#[tokio::test] +async fn fixed_creation_identity_is_private_and_conflicting_retries_do_not_overwrite_it() { + use std::os::unix::fs::PermissionsExt; + let fixture = TestGroupCreation::start(false).await; + let body = request(); + fixture.create(&body, StatusCode::CREATED).await; + let before = std::fs::read(fixture.ownership()).unwrap(); + assert_eq!( + std::fs::metadata(fixture.ownership()) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + let (status, bytes) = fixture + .request(Method::GET, "/stores/0/groups/1/creation", None) + .await; + assert_eq!(status, StatusCode::OK); + let accepted: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(accepted["deployment"], body["deployment"]); + assert_eq!(accepted["retired"], false); + fixture.create(&body, StatusCode::CONFLICT).await; + let mut changed_replica = body.clone(); + changed_replica["replica_id"] = 8.into(); + let mut changed_cluster = body.clone(); + changed_cluster["deployment"]["cluster_id"] = uuid::Uuid::new_v4().to_string().into(); + for changed in [request(), changed_replica, changed_cluster] { + fixture.create(&changed, StatusCode::CONFLICT).await; + assert_eq!(std::fs::read(fixture.ownership()).unwrap(), before); + } + let mut generic = body.clone(); + generic.as_object_mut().unwrap().remove("deployment"); + fixture.create(&generic, StatusCode::CONFLICT).await; + assert_eq!(fixture.store.get_group(1).unwrap().local_replica().id, 7); +} + +#[tokio::test] +async fn an_existing_unowned_replica_is_not_adopted_even_when_its_numeric_id_matches() { + let fixture = TestGroupCreation::start(false).await; + fixture.store.add_group_without_election(PxGroup::new( + 1, + PxLocalReplica::new(7, PxLocalReplicaRole::Leader), + )); + fixture.create(&request(), StatusCode::CONFLICT).await; + let mut generic = request(); + generic.as_object_mut().unwrap().remove("deployment"); + generic["replica_id"] = 99.into(); + fixture.create(&generic, StatusCode::CONFLICT).await; + assert!(!fixture.ownership().exists()); + assert!(!fixture.registry.config.wal_root.exists()); +} + +#[tokio::test] +async fn failure_after_acceptance_resumes_the_same_operation_without_allocating_another_owner() { + let fixture = TestGroupCreation::start(true).await; + let body = request(); + fixture.create(&body, StatusCode::INTERNAL_SERVER_ERROR).await; + let before = std::fs::read(fixture.ownership()).unwrap(); + fixture.create(&request(), StatusCode::CONFLICT).await; + assert_eq!(std::fs::read(fixture.ownership()).unwrap(), before); + std::fs::remove_file(&fixture.registry.config.data_root).unwrap(); + std::fs::create_dir(&fixture.registry.config.data_root).unwrap(); + fixture.create(&body, StatusCode::CREATED).await; + assert_eq!(std::fs::read(fixture.ownership()).unwrap(), before); +} + +#[tokio::test] +async fn explicit_removal_keeps_the_deployment_tombstone_and_restore_cannot_recreate_it() { + let fixture = TestGroupCreation::start(false).await; + let body = request(); + fixture.create(&body, StatusCode::CREATED).await; + let (status, bytes) = fixture.request(Method::DELETE, "/stores/0/groups/1", None).await; + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); + let tombstone = std::fs::read(fixture.ownership()).unwrap(); + assert_eq!( + serde_json::from_slice::(&tombstone).unwrap()["retired"], + true + ); + fixture.create(&body, StatusCode::CONFLICT).await; + let restored = crowdb_kv_server::recovery::startup::create_group_with_wal( + 0, + 1, + 7, + PxLocalReplicaRole::Follower, + &fixture.registry.config, + fixture.registry.wal_backend.clone(), + fixture.registry.crowtree_backend, + ) + .await; + assert!(matches!(restored, Err(error) if error.kind() == std::io::ErrorKind::PermissionDenied)); + assert_eq!(std::fs::read(fixture.ownership()).unwrap(), tombstone); + assert!(fixture.store.get_group(1).is_none()); +} + +#[tokio::test] +async fn corrupt_owner_cannot_be_replaced_or_open_group_storage() { + let fixture = TestGroupCreation::start(false).await; + std::fs::create_dir_all(&fixture.registry.config.config_root).unwrap(); + std::fs::write(fixture.ownership(), b"corrupt").unwrap(); + fixture.create(&request(), StatusCode::CONFLICT).await; + assert_eq!(std::fs::read(fixture.ownership()).unwrap(), b"corrupt"); + assert!(!fixture.registry.config.wal_root.exists()); + assert!(!fixture.registry.config.data_root.exists()); +} + +#[tokio::test] +async fn asymmetric_coordinators_cannot_accept_two_manifests_before_or_after_restart() { + use system_bootstrap::{identity, proposal, TestSystemBootstrap}; + let mut nodes = Vec::new(); + for _ in 0..4 { + nodes.push(TestSystemBootstrap::start().await); + } + let first = identity(); + let second = identity(); + let (accepted_first, accepted_second) = tokio::join!( + proposal(&nodes, [0, 1, 2], &first), + proposal(&nodes, [1, 2, 3], &second) + ); + assert!(!(accepted_first && accepted_second)); + let retained: Vec<_> = nodes + .iter() + .map(|node| std::fs::read(node.ownership()).unwrap()) + .collect(); + for node in &mut nodes { + node.assert_no_execution(); + node.restart().await; + } + let (accepted_first, accepted_second) = tokio::join!( + proposal(&nodes, [0, 1, 2], &first), + proposal(&nodes, [1, 2, 3], &second) + ); + assert!(!(accepted_first && accepted_second)); + let late = identity(); + assert!(!proposal(&nodes, [0, 1, 3], &late).await); + for (node, bytes) in nodes.iter().zip(retained) { + let rejected = serde_json::json!({"replica_id":1,"start_election":false,"bootstrap":late}); + assert_eq!( + node.request("/system/init", &rejected).await, + StatusCode::CONFLICT + ); + assert_eq!(std::fs::read(node.ownership()).unwrap(), bytes); + node.assert_no_execution(); + } +} + +#[tokio::test] +async fn lost_prepare_reply_resumes_the_exact_durable_owner_without_storage_creation() { + use std::{os::unix::fs::PermissionsExt, time::Duration}; + use system_bootstrap::{identity, TestSystemBootstrap}; + use tokio::io::AsyncWriteExt; + let mut node = TestSystemBootstrap::start().await; + let accepted = identity(); + let body = serde_json::to_vec(&serde_json::json!({"replica_id":2,"bootstrap":accepted})).unwrap(); + let mut connection = tokio::net::TcpStream::connect(node.origin.trim_start_matches("http://")) + .await + .unwrap(); + connection.write_all(format!("POST /system/prepare HTTP/1.1\r\nHost: local\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes()).await.unwrap(); + connection.write_all(&body).await.unwrap(); + // Discard the response once acceptance is durable, as a disconnected coordinator would. + tokio::time::timeout(Duration::from_secs(3), async { + while !node.ownership().exists() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + drop(connection); + let bytes = std::fs::read(node.ownership()).unwrap(); + assert_eq!( + std::fs::metadata(node.ownership()).unwrap().permissions().mode() & 0o777, + 0o600 + ); + node.restart().await; + assert_eq!(node.prepare(2, &accepted).await, StatusCode::OK); + assert_eq!(node.prepare(3, &accepted).await, StatusCode::CONFLICT); + assert_eq!(node.prepare(2, &identity()).await, StatusCode::CONFLICT); + assert_eq!(std::fs::read(node.ownership()).unwrap(), bytes); + node.assert_no_execution(); +} + +#[tokio::test] +async fn wiring_a_fresh_multireplica_group_requires_an_elected_leader() { + let fixture = TestGroupCreation::start(false).await; + fixture.create(&request(), StatusCode::CREATED).await; + let remotes = serde_json::json!([ + {"replica_id":8,"endpoint":"127.0.0.1:1","voting":true}, + {"replica_id":9,"endpoint":"127.0.0.1:2","voting":true} + ]); + let (status, bytes) = fixture + .request(Method::POST, "/stores/0/groups/1/remotes", Some(&remotes)) + .await; + assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); + let group = fixture.store.get_group(1).unwrap(); + assert_eq!(group.quorum(), 2); + assert_eq!( + group.leader_id(), + 0, + "a bootstrap role at term zero is not a quorum-elected leader" + ); + assert_ne!(group.local_replica().role(), PxLocalReplicaRole::Leader); +} diff --git a/app/crowdb-kv-server/tests/keepalive_recovery_test.rs b/app/crowdb-kv-server/tests/keepalive_recovery_test.rs new file mode 100644 index 000000000..d7c039f47 --- /dev/null +++ b/app/crowdb-kv-server/tests/keepalive_recovery_test.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{extract::State, routing::get, Json, Router}; +use crowdb_kv::cluster::{group::PxGroup, KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv::common::config::CrowDBConfig; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_kv_server::{background::keepalive::KeepAliveLoop, store_registry::KvStoreRegistry}; +use crowdb_protocol::common::KvServerIdentity; +use std::{sync::Arc, time::Duration}; + +#[tokio::test] +async fn a_recovered_singleton_registers_its_current_address_with_unreachable_retained_seeds() { + async fn topology(State(store): State>) -> Json { + Json(serde_json::json!({"stores": [store.status()]})) + } + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new( + 0, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + store.start().await.unwrap(); + let registry = Arc::new(KvStoreRegistry::with_config(CrowDBConfig::for_tests())); + registry.add_store(0, &store); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let current = format!("http://{}", listener.local_addr().unwrap()); + let router = Router::new() + .route("/topology", get(topology)) + .with_state(store.clone()); + let http = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + let keepalive = KeepAliveLoop::spawn( + registry, + KvServerIdentity { + instance_id: 7, + node_id: Some(1), + }, + current.clone(), + "192.0.2.1:10100", + vec!["http://127.0.0.1:1".into()], + String::new(), + 1, + ); + let client = CrowdbKvClient::new(ClientConfig::new(vec![current.clone()])); + client.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); + let value = tokio::time::timeout(Duration::from_secs(3), async { + loop { + if let GetOutcome::Found { value, .. } = client + .get(0, 0, b"/srv/kv-server/7", ReadMode::Linearizable, None) + .await + .unwrap() + { + break serde_json::from_slice::(&value).unwrap(); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("local authority must publish the current registration without a reachable old seed"); + assert_eq!(value["rpc_endpoint"], current); + assert_eq!(value["extra"]["kv_server"]["node_id"], 1); + keepalive.stop().await; + http.abort(); + store.shutdown(Duration::from_secs(1)).await; +} diff --git a/app/crowdb-kv-server/tests/member_routes_test.rs b/app/crowdb-kv-server/tests/member_routes_test.rs new file mode 100644 index 000000000..4b8f65d72 --- /dev/null +++ b/app/crowdb-kv-server/tests/member_routes_test.rs @@ -0,0 +1,101 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv::cluster::{group::PxGroup, PxLocalReplica, PxLocalReplicaRole, PxRemoteReplica}; +use crowdb_kv_server::recovery::member_routes::{apply, configure}; +use crowdb_protocol::mgmt::{ + node::recovery::{MemberEndpoint, MemberEndpoints, RecoveryGroup, RecoveryGroups, RecoveryReplica}, + SystemBootstrapIdentity, +}; +use crowdb_test_harness::test_dirs::tempdir_in_test_data; +use std::{ + os::unix::fs::PermissionsExt, + path::Path, + time::{SystemTime, UNIX_EPOCH}, +}; + +fn write(path: &Path, value: &impl serde::Serialize) { + std::fs::write(path, serde_json::to_vec(value).unwrap()).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).unwrap(); +} + +#[test] +fn native_routes_require_live_observer_proof_and_keep_canonical_membership() { + let dir = tempdir_in_test_data("native-member-routes"); + let root = dir.path(); + let config = root.join("conf"); + let identity = SystemBootstrapIdentity { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + configuration_digest: "a".repeat(64), + }; + let local = MemberEndpoint { + discovery_id: uuid::Uuid::new_v4().to_string(), + node_id: 1, + boot_id: uuid::Uuid::new_v4().to_string(), + addresses: vec!["192.0.2.1".parse().unwrap()], + }; + write(&root.join("node-endpoint.json"), &local); + write( + &root.join("accepted-node.json"), + &serde_json::json!({"node_id":1,"bootstrap":identity}), + ); + configure(&config, root).unwrap(); + write( + &config.join("system-bootstrap.json"), + &serde_json::json!({"identity":identity,"replica_id":1}), + ); + write( + &root.join("recovery-groups.json"), + &RecoveryGroups { + bootstrap: identity.clone(), + groups: vec![RecoveryGroup { + store_id: 0, + group_id: 0, + deployment: None, + members: vec![ + RecoveryReplica { + replica_id: 1, + node_id: 1, + }, + RecoveryReplica { + replica_id: 2, + node_id: 2, + }, + ], + }], + }, + ); + let mut group = PxGroup::new(0, PxLocalReplica::new(1, PxLocalReplicaRole::Follower)); + group.set_remote_replicas(vec![PxRemoteReplica::new(2, "192.0.2.2:10100".into())]); + apply(&group, &config, 0).unwrap(); + assert!(group.status().remotes[0].endpoint.is_empty()); + let mut peer = local.clone(); + peer.node_id = 2; + peer.discovery_id = uuid::Uuid::new_v4().to_string(); + peer.addresses = vec!["192.0.2.3".parse().unwrap()]; + let mut snapshot = MemberEndpoints { + version: 1, + bootstrap: identity, + observer_boot_id: local.boot_id.clone(), + observed_at_unix_ms: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_millis() + .try_into() + .unwrap(), + members: vec![local, peer], + conflicts: vec![], + }; + write(&root.join("member-endpoints.json"), &snapshot); + apply(&group, &config, 0).unwrap(); + assert_eq!(group.status().remotes[0].endpoint, "192.0.2.3:10100"); + assert_eq!(group.remote_replica_info(), vec![(2, "192.0.2.2:10100", true)]); + snapshot.observer_boot_id = uuid::Uuid::new_v4().to_string(); + write(&root.join("member-endpoints.json"), &snapshot); + apply(&group, &config, 0).unwrap(); + assert!(group.status().remotes[0].endpoint.is_empty()); + std::fs::write(config.join("member-route-source.json"), "corrupt").unwrap(); + assert!(apply(&group, &config, 0).is_err()); + assert!(group.status().remotes[0].endpoint.is_empty()); +} diff --git a/app/crowdb-web/Cargo.toml b/app/crowdb-web/Cargo.toml index 35864edc8..67c848a14 100644 --- a/app/crowdb-web/Cargo.toml +++ b/app/crowdb-web/Cargo.toml @@ -13,6 +13,9 @@ workspace = true name = "crowdb-web" path = "src/main.rs" +[features] +test-util = [] + [dependencies] flatbuffers = { workspace = true } arc-swap = "1.7" @@ -40,11 +43,13 @@ sha2 = "0.10" percent-encoding = "2" chrono = { version = "0.4", default-features = false, features = ["std"] } hex = "0.4" +quick-xml = "0.38" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "stream"] } futures = "0.3" uuid = { version = "1", features = ["v4"] } [dev-dependencies] +crowdb-web = { path = ".", features = ["test-util"] } crowdb-chunk-kv-client = { path = "../../lib/crowdb-chunk-kv-client" } crowdb-access-s3 = { path = "../../lib/crowdb-access-s3" } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "net"] } diff --git a/app/crowdb-web/src/access.rs b/app/crowdb-web/src/access.rs index aae8f2b14..ce5704db9 100644 --- a/app/crowdb-web/src/access.rs +++ b/app/crowdb-web/src/access.rs @@ -26,6 +26,13 @@ mod signing; const BODY_LIMIT: usize = 16 * 1024 * 1024; type ApiError = (StatusCode, Json); +pub(crate) fn sign_local_s3_read(state: &AppState, url: &reqwest::Url) -> Result { + let credentials = credentials::local_s3(state)?; + let mut headers = HeaderMap::new(); + signing::sign(&credentials, &axum::http::Method::GET, url, &[], &mut headers)?; + Ok(headers) +} + pub(crate) fn read_router() -> Router { Router::new() .route("/api/access/connections", get(connections)) @@ -65,6 +72,11 @@ fn validate_origin(origin: &str) -> Result { fn origin(state: &AppState, protocol: &str) -> Result, ApiError> { let environment = protocol_environment(protocol).ok_or_else(|| err_400("Unknown Access protocol"))?; + if state.node_monitor_url.is_some() { + if let Some(value) = crate::services::access::origin(state, protocol) { + return validate_origin(&value).map(Some); + } + } if let Ok(value) = std::env::var(environment) { return validate_origin(&value).map(Some); } @@ -112,6 +124,10 @@ pub(crate) async fn client_credentials(State(state): State) -> Result< "iceberg": { "uri": value("ICEBERG_URI"), "token": value("ICEBERG_TOKEN") + }, + "dataset": { + "uri": value("DATASET_URI"), + "token": value("DATASET_TOKEN") } }))) } diff --git a/app/crowdb-web/src/access/credentials.rs b/app/crowdb-web/src/access/credentials.rs index 520e883dc..d5afd7926 100644 --- a/app/crowdb-web/src/access/credentials.rs +++ b/app/crowdb-web/src/access/credentials.rs @@ -40,12 +40,7 @@ pub(super) fn dataset(state: &AppState) -> Result { .map_err(|_| "Dataset credential is not configured on the Console server".into()) } -pub(super) struct S3Credentials { - pub access: String, - pub secret: String, - pub region: String, - pub session: Option, -} +pub(super) use crowdb_console_shared::ops::s3::S3Credentials; pub(super) fn s3(state: &AppState, target: &str) -> Result { if let (Ok(access), Ok(secret)) = ( @@ -67,6 +62,11 @@ pub(super) fn s3(state: &AppState, target: &str) -> Result Result { + let get = |name| value(body, name).ok_or_else(|| "Cluster S3 credentials are incomplete".to_owned()); Ok(S3Credentials { access: get("AWS_ACCESS_KEY_ID")?, secret: get("AWS_SECRET_ACCESS_KEY")?, @@ -75,6 +75,12 @@ pub(super) fn s3(state: &AppState, target: &str) -> Result Result { + let body = crowdb_monitor::show_client_credentials(&root(state)) + .map_err(|_| "Retained local S3 credentials are unavailable")?; + from_client_env(&body) +} + pub(super) fn manager(state: &AppState) -> Result { if let Ok(token) = std::env::var("CROWDB_ICEBERG_MANAGE_TOKEN") { return Ok(token); diff --git a/app/crowdb-web/src/access/signing.rs b/app/crowdb-web/src/access/signing.rs index 738a132ab..ec3633e86 100644 --- a/app/crowdb-web/src/access/signing.rs +++ b/app/crowdb-web/src/access/signing.rs @@ -1,98 +1,6 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::credentials::S3Credentials; -use axum::http::{HeaderMap, Method}; -use hmac::{Hmac, Mac}; -use sha2::{Digest, Sha256}; -use std::fmt::Write as _; +//! Shared S3 signing for console forwarding and attached CLI clients. -fn encode(value: &str, slash: bool) -> String { - let mut result = String::new(); - for byte in percent_encoding::percent_decode_str(value) { - if byte.is_ascii_alphanumeric() || b"-._~".contains(&byte) || (slash && byte == b'/') { - result.push(char::from(byte)); - } else { - write!(result, "%{byte:02X}").expect("String writing cannot fail"); - } - } - result -} - -fn mac(key: &[u8], value: &str) -> Vec { - let mut mac = Hmac::::new_from_slice(key).expect("HMAC accepts arbitrary key lengths"); - mac.update(value.as_bytes()); - mac.finalize().into_bytes().to_vec() -} - -pub(super) fn sign( - credentials: &S3Credentials, - method: &Method, - url: &reqwest::Url, - body: &[u8], - headers: &mut HeaderMap, -) -> Result<(), String> { - let timestamp = chrono::DateTime::::from(std::time::SystemTime::now()) - .format("%Y%m%dT%H%M%SZ") - .to_string(); - let date = ×tamp[..8]; - let payload = hex::encode(Sha256::digest(body)); - let origin = url.origin().ascii_serialization(); - let host = origin.split_once("://").map_or(origin.as_str(), |(_, host)| host); - let mut signed = std::collections::BTreeMap::from([ - ("host", host.to_owned()), - ("x-amz-content-sha256", payload.clone()), - ("x-amz-date", timestamp.clone()), - ]); - if let Some(token) = &credentials.session { - signed.insert("x-amz-security-token", token.clone()); - } - let names = signed.keys().copied().collect::>().join(";"); - let mut canonical_headers = String::new(); - for (name, value) in &signed { - writeln!(canonical_headers, "{name}:{}", value.trim()).expect("String writing cannot fail"); - } - let mut query: Vec<_> = url - .query() - .unwrap_or("") - .split('&') - .filter(|part| !part.is_empty()) - .map(|part| { - let (key, value) = part.split_once('=').unwrap_or((part, "")); - (encode(key, false), encode(value, false)) - }) - .collect(); - query.sort(); - let query = query - .iter() - .map(|(key, value)| format!("{key}={value}")) - .collect::>() - .join("&"); - let canonical = format!( - "{method}\n{}\n{query}\n{canonical_headers}\n{names}\n{payload}", - encode(url.path(), true) - ); - let scope = format!("{date}/{}/s3/aws4_request", credentials.region); - let mut key = format!("AWS4{}", credentials.secret).into_bytes(); - for part in [date, &credentials.region, "s3", "aws4_request"] { - key = mac(&key, part); - } - let signature = hex::encode(mac( - &key, - &format!( - "AWS4-HMAC-SHA256\n{timestamp}\n{scope}\n{}", - hex::encode(Sha256::digest(canonical.as_bytes())) - ), - )); - signed.insert( - "authorization", - format!( - "AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={names}, Signature={signature}", - credentials.access - ), - ); - for (name, value) in signed { - headers.insert(name, value.parse().map_err(|_| "Invalid cluster S3 credential")?); - } - Ok(()) -} +pub(super) use crowdb_console_shared::ops::s3::sign_request as sign; diff --git a/app/crowdb-web/src/chunk_kv/runtime.rs b/app/crowdb-web/src/chunk_kv/runtime.rs index cc1ceceaa..575712286 100644 --- a/app/crowdb-web/src/chunk_kv/runtime.rs +++ b/app/crowdb-web/src/chunk_kv/runtime.rs @@ -48,8 +48,10 @@ pub(crate) async fn runtime( if query.id.len() != 32 || !query.id.bytes().all(|byte| byte.is_ascii_hexdigit()) { return Err(err_400("Invalid partition identity")); } - if query.stream_offset > 0 && query.stream_generation.is_none() { - return Err(err_400("Extent continuation requires a stream generation")); + if query.stream_offset != 0 { + return Err(err_400( + "Tail stream chunk record observation requires offset zero", + )); } if query .page_path @@ -121,10 +123,9 @@ async fn inspect(state: &AppState, query: &RuntimeQuery) -> Result, if query .stream_generation .is_some_and(|generation| integer(&observation["journal"]["generation"]) != Some(generation)) - || (query.stream_offset > 0 - && observation["journal"]["offset"].as_u64() != Some(query.stream_offset as u64)) + || (observation["journal"].is_object() && observation["journal"]["offset"].as_u64() != Some(0)) { - return Err(err_409("Stream manifest changed during observation")); + return Err(err_409("Stream chunk record changed during observation")); } let kv = state.kv_client().await; let current: ChunkKvRangeCatalogHead = diff --git a/app/crowdb-web/src/diskdb/lifecycle.rs b/app/crowdb-web/src/diskdb/lifecycle.rs index 1b5b6239d..02a135b6c 100644 --- a/app/crowdb-web/src/diskdb/lifecycle.rs +++ b/app/crowdb-web/src/diskdb/lifecycle.rs @@ -164,7 +164,7 @@ async fn deploy_diskdb( .map_err(|e| err_502(format!("diskdb deploy: {e}")))?; let entry = crowdb_console_shared::config::ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: format!("diskdb-{node_id}"), // DiskDB has no public HTTP management URL; retain its RPC endpoint // as the service URL and endpoint for persisted topology metadata. @@ -291,7 +291,7 @@ async fn restart_diskdb( .map_err(|e| err_502(format!("diskdb redeploy (restart): {e}")))?; let new_entry = crowdb_console_shared::config::ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: entry.id.clone(), // DiskDB has no public HTTP management URL; retain its RPC endpoint // as the service URL and endpoint for persisted topology metadata. diff --git a/app/crowdb-web/src/diskdb/remote.rs b/app/crowdb-web/src/diskdb/remote.rs index 4c9c1ac47..29cc6a10f 100644 --- a/app/crowdb-web/src/diskdb/remote.rs +++ b/app/crowdb-web/src/diskdb/remote.rs @@ -69,6 +69,9 @@ pub(crate) fn configuration( writeln!(configuration, "\n[heartbeat]\ninterval_secs = {interval}") .expect("writing configuration to a String cannot fail"); } + // Managed setup advances as ownership and provisional disks are published. + // Their change notifications avoid a full polling interval at each step. + configuration.push_str("\n[notify]\nnotify_enabled = true\n"); Ok(configuration) } diff --git a/app/crowdb-web/src/error.rs b/app/crowdb-web/src/error.rs index 4e51cc551..eaeb2382c 100644 --- a/app/crowdb-web/src/error.rs +++ b/app/crowdb-web/src/error.rs @@ -54,6 +54,7 @@ pub fn map_err(e: Error) -> (StatusCode, Json) { pub fn map_config_err(e: Error) -> (StatusCode, Json) { let msg = format!("{e}"); match e { + Error::DependenciesNotReady(_) => err_409(msg), Error::Conflict { .. } => (StatusCode::CONFLICT, Json(ErrorBody { error: msg })), Error::UpstreamRpc { ref status, .. } if status.contains("HTTP 409") => { (StatusCode::CONFLICT, Json(ErrorBody { error: msg })) diff --git a/app/crowdb-web/src/lib.rs b/app/crowdb-web/src/lib.rs index 548d9018e..8cdd235ef 100644 --- a/app/crowdb-web/src/lib.rs +++ b/app/crowdb-web/src/lib.rs @@ -32,8 +32,21 @@ pub mod spa; mod standalone; pub mod state; +pub use node::observation::{start_node_observer, NodeObserver}; pub use state::AppState; +#[cfg(feature = "test-util")] +pub use node::observation::{node_observation_time_for_tests, start_node_observer_for_tests}; + +#[cfg(feature = "test-util")] +pub use node::observation::protocol_probe_for_tests; +#[cfg(feature = "test-util")] +pub use node::observation::s3_protocol_probe_for_tests; +#[cfg(feature = "test-util")] +pub use node::observation::sample_node_services_for_tests; +#[cfg(feature = "test-util")] +pub use node::observation::service_detail_for_tests; + /// Run standard-node automatic admission independently of browser connections. #[must_use] pub fn start_node_auto_join(state: AppState) -> tokio::task::JoinHandle<()> { @@ -65,6 +78,9 @@ pub fn router(state: AppState) -> axum::Router { ) .route("/api/node/admissions", get(node::admissions)) .route("/api/node/status", get(node::status)) + .route("/api/cluster/target", get(node::target::identity)) + .route("/api/cluster/node-observation", get(node::observation::read)) + .route("/api/cluster/status", get(node::cluster_status::read)) .route("/api/authority", get(managed::authority)) .route("/api/preview", get(managed::snapshot)) .route("/api/chunk-kv/catalog", get(chunk_kv::catalog)) @@ -257,6 +273,9 @@ pub fn router(state: AppState) -> axum::Router { .route("/api/node/cleanup", post(node::cleanup)) .route("/api/node/cancel", post(node::cancel)) .route("/api/node/status", get(node::status)) + .route("/api/cluster/target", get(node::target::identity)) + .route("/api/cluster/node-observation", get(node::observation::read)) + .route("/api/cluster/status", get(node::cluster_status::read)) .route( "/api/node/auto-join", get(node::auto_join::status).post(node::auto_join::update), diff --git a/app/crowdb-web/src/main.rs b/app/crowdb-web/src/main.rs index aa9f11a99..781346ffd 100644 --- a/app/crowdb-web/src/main.rs +++ b/app/crowdb-web/src/main.rs @@ -159,6 +159,7 @@ async fn main() -> Result<(), Box> { } let listener = tokio::net::TcpListener::bind(addr).await?; + let _node_observer = crowdb_web::start_node_observer(&state)?; let auto_join = crowdb_web::start_node_auto_join(state.clone()); if args.test_mode { let cleanup_state = state.clone(); diff --git a/app/crowdb-web/src/mgmt/topology.rs b/app/crowdb-web/src/mgmt/topology.rs index 65f9522a9..86b66a190 100644 --- a/app/crowdb-web/src/mgmt/topology.rs +++ b/app/crowdb-web/src/mgmt/topology.rs @@ -180,6 +180,7 @@ async fn ensure_group_local( replica_id, initial_role: Some(initial_role), start_election, + deployment: None, }, ) .await @@ -197,6 +198,7 @@ async fn ensure_group_local( replica_id, initial_role: Some(initial_role), start_election, + deployment: None, }, ) .await diff --git a/app/crowdb-web/src/node.rs b/app/crowdb-web/src/node.rs index 5683f1ccb..8772d5669 100644 --- a/app/crowdb-web/src/node.rs +++ b/app/crowdb-web/src/node.rs @@ -10,10 +10,13 @@ pub(crate) mod setup; mod update; pub(crate) use update::update; mod authority; +pub(crate) mod cluster_status; mod hardware; pub(crate) mod initialization; +pub(crate) mod observation; mod preparation; pub(crate) mod storage; +pub(crate) mod target; pub(crate) use admission::binding as cluster_binding; pub(crate) use admission::{admissions, admit, control_socket, node_key_path, records, save}; pub(crate) use authority::{guard, status}; diff --git a/app/crowdb-web/src/node/admission.rs b/app/crowdb-web/src/node/admission.rs index 142a1252f..a1b79985a 100644 --- a/app/crowdb-web/src/node/admission.rs +++ b/app/crowdb-web/src/node/admission.rs @@ -19,6 +19,7 @@ use crate::{ state::AppState, }; type Failure = (StatusCode, Json); +mod recovery; #[derive(Deserialize)] #[serde(deny_unknown_fields)] @@ -32,6 +33,9 @@ pub(crate) struct AdmitRequest { pub(super) preflight_operation_id: Option, #[serde(default)] pub(super) setup: Option, + /// Automatic callers must confirm discovery hints over authenticated control. + #[serde(skip)] + pub(super) expected_formation: Option, } pub(crate) async fn admit( @@ -164,7 +168,7 @@ async fn admit_observed( if target.ssh_password.is_none() { target.ssh_key = Some(key.to_string_lossy().into_owned()); } - admission::prepare( + let identity = admission::prepare( &control_socket(), &key, &record.operation_id, @@ -174,7 +178,18 @@ async fn admit_observed( ) .await .map_err(map_config_err)?; - super::setup::accept(state, &record, &target, body.setup).await?; + record.pin_host_key(&identity.host_key).map_err(map_config_err)?; + if let Some(binding) = &binding { + record = registry::admit( + state.kv_client().await.as_ref(), + &binding.bootstrap.cluster_id, + record, + ) + .await + .map_err(map_config_err)?; + } + save(&path, &record)?; + super::setup::accept(state, &record, &target, body.setup, body.expected_formation).await?; target.ssh_password = None; target.ssh_key = None; target.ssh_credential_ref = Some("id_ed25519".into()); @@ -390,6 +405,7 @@ async fn confirm_admission( ) .await .map_err(map_config_err)?; + recovery::distribute(&ctx, binding, key).await?; Ok(()) } @@ -420,6 +436,7 @@ fn initial_record( ssh_port: body.ssh_port, ssh_user: body.ssh_user.clone(), operation_id: uuid::Uuid::new_v4().to_string(), + host_key: None, confirmed: false, cancelled: false, } diff --git a/app/crowdb-web/src/node/admission/recovery.rs b/app/crowdb-web/src/node/admission/recovery.rs new file mode 100644 index 000000000..9f3dbd26f --- /dev/null +++ b/app/crowdb-web/src/node/admission/recovery.rs @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Distribute confirmed contact pins without changing the accepted voting manifest. + +use crate::error::{err_409, err_502, map_config_err}; +use crowdb_console_shared::{ + config::NodeEntry, + deployment::{ + admission, + recovery::{ConfirmedMemberTrust, MemberTrust}, + registry, PreparedBootstrap, + }, + ops::OpContext, +}; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeControl}; +use std::path::Path; + +pub(super) async fn distribute( + ctx: &OpContext, + binding: &NodeBinding, + key: &Path, +) -> Result<(), super::Failure> { + let GetOutcome::Found { value, .. } = ctx + .kv() + .get( + 0, + 0, + crowdb_console_shared::deployment::CLUSTER_KEY, + ReadMode::Linearizable, + None, + ) + .await + .map_err(|error| map_config_err(error.into()))? + else { + return Err(err_409("Confirmed bootstrap is absent")); + }; + let operation: PreparedBootstrap = + serde_json::from_slice(&value).map_err(|error| err_502(error.to_string()))?; + if operation.identity != binding.bootstrap { + return Err(err_409("Confirmed bootstrap identity differs")); + } + let (registry, _) = registry::read(ctx.kv()) + .await + .map_err(map_config_err)? + .ok_or_else(|| err_409("Confirmed node registry is absent"))?; + let initial = MemberTrust::from_bootstrap(&operation).map_err(map_config_err)?; + let snapshot = ConfirmedMemberTrust::new(initial, registry).map_err(map_config_err)?; + snapshot.validate(&operation).map_err(map_config_err)?; + let command = NodeControl::AcceptConfirmedMemberTrust { + trust: serde_json::to_value(&snapshot).map_err(|error| err_502(error.to_string()))?, + }; + let replies = futures::future::join_all(snapshot.current.members.iter().map(|member| { + let command = &command; + let node = snapshot + .registry + .nodes + .iter() + .find(|node| node.node_id == member.node_id); + async move { + let node = node.ok_or_else(|| err_409("Confirmed recovery destination is absent"))?; + let target = NodeEntry { + id: node.node_id, + rack_id: node.rack_id, + host: node.host.clone(), + ssh_port: node.ssh_port, + ssh_user: node.ssh_user.clone(), + ssh_key: Some(key.to_string_lossy().into_owned()), + ssh_password: None, + ssh_credential_ref: None, + }; + admission::remote_control_pinned(&target, member, command) + .await + .map_err(map_config_err)?; + Ok::<_, super::Failure>(()) + } + })) + .await; + for reply in replies { + reply?; + } + Ok(()) +} diff --git a/app/crowdb-web/src/node/authority.rs b/app/crowdb-web/src/node/authority.rs index 4ae2afd68..28016e27f 100644 --- a/app/crowdb-web/src/node/authority.rs +++ b/app/crowdb-web/src/node/authority.rs @@ -87,6 +87,9 @@ async fn bootstrap_status(operation: &PreparedBootstrap) -> Value { } pub(crate) async fn guard(State(state): State, request: Request, next: Next) -> Response { + if observation_request(&request) { + return next.run(request).await; + } if state.node_monitor_url.is_none() { return next.run(request).await; } @@ -184,6 +187,14 @@ pub(crate) async fn guard(State(state): State, request: Request, next: next.run(request).await } +fn observation_request(request: &Request) -> bool { + *request.method() == Method::GET + && matches!( + request.uri().path(), + "/api/cluster/target" | "/api/cluster/node-observation" | "/api/cluster/status" + ) +} + fn reconcile_cleanup(state: &AppState) -> Result<(), Failure> { if super::admission::binding(state)?.is_some() || state.runtime_root.join("cleanup-progress.json").exists() @@ -204,6 +215,7 @@ fn reconcile_cleanup(state: &AppState) -> Result<(), Failure> { if acknowledged.exists() { return Ok(()); } + super::cleanup::retire_credentials(state, &bootstrap)?; for name in [ "prepared-bootstrap.json", "confirmed-cluster.json", diff --git a/app/crowdb-web/src/node/auto_join.rs b/app/crowdb-web/src/node/auto_join.rs index 25cd612e3..9e7bfd3b9 100644 --- a/app/crowdb-web/src/node/auto_join.rs +++ b/app/crowdb-web/src/node/auto_join.rs @@ -3,6 +3,7 @@ //! Workspace policy schedules admissions; accepted bootstrap belongs to monitors. +mod formation; mod runner; use std::fs; @@ -44,7 +45,13 @@ fn read(state: &AppState) -> Result { let path = state.runtime_root.join("auto-join.json"); let metadata = match fs::symlink_metadata(&path) { Ok(metadata) => metadata, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Policy::default()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Policy { + enabled: deployment(state)?.mode + == crowdb_protocol::mgmt::node::deployment::DeploymentMode::Local, + ..Policy::default() + }) + } Err(error) => return Err(err_502(error.to_string())), }; if !metadata.is_file() || metadata.len() > 4096 { @@ -58,6 +65,18 @@ fn read(state: &AppState) -> Result { Ok(policy) } +fn deployment( + state: &AppState, +) -> Result { + let root = state + .runtime_root + .parent() + .ok_or_else(|| err_409("Node policy root is absent"))?; + crowdb_monitor::read_node_policy(root) + .map_err(|error| err_502(error.to_string()))? + .ok_or_else(|| err_409("Accepted node deployment policy is absent")) +} + fn supported(state: &AppState) -> Result<(), Failure> { if state.node_monitor_url.is_none() || state.managed_mode { return Err(err_409("Auto join requires a standard node workspace")); @@ -68,6 +87,7 @@ fn supported(state: &AppState) -> Result<(), Failure> { pub(crate) async fn status(State(state): State) -> Result, Failure> { supported(&state)?; let policy = read(&state)?; + let deployment = deployment(&state)?; let path = state.runtime_root.join("auto-join-status.json"); match fs::symlink_metadata(&path) { Ok(metadata) if !metadata.is_file() || metadata.len() > 65536 => { @@ -86,7 +106,7 @@ pub(crate) async fn status(State(state): State) -> Result, }; Ok(Json( json!({"enabled": policy.enabled, "target_cluster_id": policy.target_cluster_id, - "mode": "standard", "required_voters": 3, "observation": observation}), + "mode": "standard", "deployment_mode": deployment.mode, "cluster": deployment.cluster, "required_voters": deployment.initial_voters, "observation": observation}), )) } diff --git a/app/crowdb-web/src/node/auto_join/formation.rs b/app/crowdb-web/src/node/auto_join/formation.rs new file mode 100644 index 000000000..218a24fe0 --- /dev/null +++ b/app/crowdb-web/src/node/auto_join/formation.rs @@ -0,0 +1,160 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Candidate policy checks precede local credential use and admission effects. + +use axum::{extract::State, Json}; +use crowdb_protocol::mgmt::node::{ + deployment::{DeploymentMode, NodeDeploymentPolicy}, + CandidateNode, CandidateState, +}; +use std::{io::Read, os::unix::fs::PermissionsExt}; + +use super::Failure; +use crate::{ + error::{err_409, err_502}, + node::NodeCandidates, + state::AppState, +}; + +pub(super) fn candidates<'a>( + snapshot: &'a NodeCandidates, + policy: &NodeDeploymentPolicy, +) -> Result, Failure> { + let expected = policy.formation(); + if snapshot.local.advertisement.formation.as_ref() != Some(&expected) { + return Err(err_409( + "Local monitor formation policy differs from accepted startup inputs", + )); + } + let local_id = &snapshot.local.advertisement.discovery_id; + if snapshot.candidates.iter().any(|peer| { + peer.state == CandidateState::IdentityConflict + && (peer.advertisement.discovery_id == *local_id + || peer.advertisement.formation.as_ref() == Some(&expected)) + }) { + return Err(err_409( + "Conflicting live node identities prevent automatic formation", + )); + } + let mut candidates: Vec<_> = snapshot + .candidates + .iter() + .filter(|peer| { + peer.state == CandidateState::Unbound + && peer.advertisement.formation.as_ref() == Some(&expected) + && (policy.initial_voters != 1 || peer.advertisement.discovery_id == *local_id) + }) + .collect(); + candidates.sort_by(|left, right| { + left.advertisement + .discovery_id + .cmp(&right.advertisement.discovery_id) + }); + if candidates.len() > usize::from(policy.initial_voters) { + return Err(err_409( + "Too many fresh nodes in the cluster scope; select an unambiguous formation", + )); + } + if policy.initial_voters == 3 + && snapshot.candidates.iter().any(|peer| { + peer.state == CandidateState::ForeignCluster + && peer.advertisement.formation.as_ref() == Some(&expected) + }) + { + return Err(err_409( + "Cluster scope already has bound members; use explicit admission to join it", + )); + } + Ok(candidates) +} + +pub(super) async fn prepare_local_rack( + state: &AppState, + policy: &NodeDeploymentPolicy, +) -> Result<(), Failure> { + let has_rack = !state + .config + .read() + .map_err(|error| err_502(error.to_string()))? + .racks + .is_empty(); + if policy.mode == DeploymentMode::Local && !has_rack { + let request = serde_json::from_value(serde_json::json!({"id":1,"name":"local cluster"})) + .map_err(|error| err_502(error.to_string()))?; + let _ = crate::lifecycle::http_add_rack(State(state.clone()), Json(request)).await?; + } + Ok(()) +} + +pub(super) fn local_password(state: &AppState) -> Result { + let root = state + .runtime_root + .parent() + .ok_or_else(|| err_409("Node data root is absent"))?; + let path = root.join("ssh/local-admission-password"); + let metadata = std::fs::symlink_metadata(&path).map_err(|_| { + err_409( + "Local admission credential is unavailable; prepare the node through its container entrypoint", + ) + })?; + if !metadata.file_type().is_file() + || metadata.permissions().mode() & 0o777 != 0o600 + || metadata.len() == 0 + || metadata.len() > 4096 + { + return Err(err_409( + "Local admission credential must be a bounded private regular file", + )); + } + let mut password = String::new(); + std::fs::File::open(path) + .map_err(|_| err_502("Cannot read local admission credential"))? + .take(4097) + .read_to_string(&mut password) + .map_err(|_| err_502("Cannot read local admission credential"))?; + if password.is_empty() || password.len() > 4096 { + return Err(err_409("Invalid local admission credential length")); + } + Ok(password) +} + +pub(super) fn waiting_for_peers( + state: &AppState, + snapshot: &NodeCandidates, + selected: &[&CandidateNode], + policy: &NodeDeploymentPolicy, +) -> Result, Failure> { + if selected.len() < usize::from(policy.initial_voters) { + let records = crate::node::records(state)?; + let config = state.config.read().map_err(|error| err_502(error.to_string()))?; + let admitted: Vec<_> = selected + .iter() + .filter_map(|candidate| { + records + .iter() + .find(|record| { + !record.cancelled + && record.discovery_id == candidate.advertisement.discovery_id + && config.node(record.node_id).is_some() + }) + .map(|record| record.node_id) + }) + .collect(); + return Ok(Some( + serde_json::json!({"phase":"waiting","required_voters":policy.initial_voters, + "members":admitted,"observed_voters":selected.len(),"reason":"Waiting for the complete initial formation before admission"}), + )); + } + if let Some(coordinator) = selected + .first() + .filter(|candidate| candidate.advertisement.discovery_id != snapshot.local.advertisement.discovery_id) + { + return Ok(Some( + serde_json::json!({"phase":"waiting","required_voters":policy.initial_voters, + "coordinator":coordinator.advertisement.discovery_id,"members":[], + "reason":"Waiting for the UUID-ordered formation coordinator"}), + )); + } + Ok(None) +} diff --git a/app/crowdb-web/src/node/auto_join/runner.rs b/app/crowdb-web/src/node/auto_join/runner.rs index b117c1245..f83841a06 100644 --- a/app/crowdb-web/src/node/auto_join/runner.rs +++ b/app/crowdb-web/src/node/auto_join/runner.rs @@ -8,10 +8,10 @@ use std::{ use axum::{extract::State, Json}; use crowdb_console_shared::deployment::PreparedBootstrap; -use crowdb_protocol::mgmt::node::CandidateState; +use crowdb_protocol::mgmt::node::deployment::DeploymentMode; use serde_json::{json, Value}; -use super::{Failure, Policy}; +use super::Failure; use crate::{ error::{err_409, err_502, map_config_err}, node, @@ -54,7 +54,7 @@ async fn step( if !state.runtime_root.join("auto-join.json").exists() { let _operation = crate::services::Operation::claim(state, vec!["node/auto-join".into()])?; if !state.runtime_root.join("auto-join.json").exists() { - node::save(&state.runtime_root.join("auto-join.json"), &Policy::default())?; + node::save(&state.runtime_root.join("auto-join.json"), &policy)?; } } if !policy.enabled { @@ -93,6 +93,10 @@ async fn admit_candidates( attempts: &mut BTreeMap, bound: bool, ) -> Result { + if bound { + return finish(state, true, Vec::new(), Vec::new()).await; + } + let deployment = super::deployment(state)?; let snapshot = node::candidates(State(state.clone())) .await .map_err(|status| { @@ -104,6 +108,11 @@ async fn admit_candidates( ) })? .0; + let selected = super::formation::candidates(&snapshot, &deployment)?; + if let Some(waiting) = super::formation::waiting_for_peers(state, &snapshot, &selected, &deployment)? { + return Ok(waiting); + } + super::formation::prepare_local_rack(state, &deployment).await?; let config = state .config .read() @@ -114,41 +123,22 @@ async fn admit_candidates( let Some(rack) = config.racks.first() else { return Ok(json!({"phase": "needs-rack"})); }; - let mut candidates: Vec<_> = snapshot - .candidates - .iter() + let candidates: Vec<_> = selected + .into_iter() .filter(|candidate| { - candidate.state == CandidateState::Unbound - && (!prepared_access.contains(&candidate.advertisement.discovery_id) - || records.iter().any(|record| { - record.discovery_id == candidate.advertisement.discovery_id - && !record.cancelled - && config.node(record.node_id).is_some() - })) + !prepared_access.contains(&candidate.advertisement.discovery_id) + || records.iter().any(|record| { + record.discovery_id == candidate.advertisement.discovery_id + && !record.cancelled + && config.node(record.node_id).is_some() + }) }) .collect(); - candidates.sort_by(|left, right| { - left.advertisement - .discovery_id - .cmp(&right.advertisement.discovery_id) - }); attempts.retain(|id, _| { candidates .iter() .any(|candidate| candidate.advertisement.discovery_id == *id) }); - if !bound && config.nodes.is_empty() && !super::read(state)?.local_draft_selected { - let clusters: std::collections::BTreeSet<_> = snapshot - .candidates - .iter() - .filter_map(|candidate| candidate.advertisement.cluster_id.as_ref()) - .collect(); - if clusters.len() > 1 { - return Ok( - json!({"phase": "needs-target", "error": "Open the intended cluster Console or select nodes for this local draft"}), - ); - } - } let mut eligible = Vec::new(); let mut needs_access = Vec::new(); for candidate in candidates { @@ -159,7 +149,7 @@ async fn admit_candidates( } if let Some(record) = record.filter(|record| config.node(record.node_id).is_some() && !bound) { eligible.push(record.node_id); - if eligible.len() == 3 { + if eligible.len() == usize::from(deployment.initial_voters) { break; } continue; @@ -185,9 +175,10 @@ async fn finish( return Ok(json!({"phase": "ready", "needs_access": needs_access, "members": operation.intent.members(), "operation_id": operation.identity.operation_id})); } - if eligible.len() < 3 { + let required = super::deployment(state)?.initial_voters; + if eligible.len() < usize::from(required) { return Ok( - json!({"phase": "waiting", "members": eligible, "required_voters": 3, "needs_access": needs_access}), + json!({"phase": "waiting", "members": eligible, "required_voters": required, "needs_access": needs_access}), ); } if !super::read(state)?.enabled { @@ -214,15 +205,22 @@ async fn admit_candidate( { return Ok(None); } - // Existing node keys must authenticate admission; the worker never supplies passwords. + let deployment = super::deployment(state)?; + // Only compatible local formation can use the documented test credential. + let password = if deployment.mode == DeploymentMode::Local { + Some(super::formation::local_password(state)?) + } else { + None + }; let body = node::admission::AdmitRequest { discovery_id: id.clone(), rack_id: record.map_or(default_rack, |record| record.rack_id), ssh_user: record.map_or_else(|| "crowdb".into(), |record| record.ssh_user.clone()), ssh_port: record.map_or(2222, |record| record.ssh_port), - ssh_password: None, + ssh_password: password, preflight_operation_id: None, setup: None, + expected_formation: Some(deployment.formation()), }; if !super::read(state)?.enabled { return Ok(Some(json!({"phase": "disabled"}))); diff --git a/app/crowdb-web/src/node/cleanup.rs b/app/crowdb-web/src/node/cleanup.rs index 39ff243b1..203f4c3ca 100644 --- a/app/crowdb-web/src/node/cleanup.rs +++ b/app/crowdb-web/src/node/cleanup.rs @@ -72,6 +72,7 @@ pub(crate) async fn cleanup( )), &operation, )?; + retire_credentials(&state, &operation.identity)?; for entry in std::fs::read_dir(state.runtime_root.as_ref()).map_err(|error| err_502(error.to_string()))? { @@ -103,6 +104,37 @@ pub(crate) async fn cleanup( )) } +pub(super) fn retire_credentials( + state: &AppState, + bootstrap: &crowdb_protocol::mgmt::SystemBootstrapIdentity, +) -> Result<(), (StatusCode, Json)> { + let source = state.runtime_root.join("secrets"); + match std::fs::symlink_metadata(&source) { + Ok(metadata) if metadata.is_dir() => {} + Ok(_) => return Err(err_409("Invalid Console credential directory")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(err_502(error.to_string())), + } + let operation = + uuid::Uuid::parse_str(&bootstrap.operation_id).map_err(|error| err_409(error.to_string()))?; + let retired = state.runtime_root.join(format!("retired-secrets-{operation}")); + match std::fs::symlink_metadata(&retired) { + Ok(_) => return Err(err_409("Retired Console credential directory already exists")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(err_502(error.to_string())), + } + match std::fs::rename(&source, &retired) { + Ok(()) => {} + Err(error) + if error.kind() == std::io::ErrorKind::NotFound + && std::fs::symlink_metadata(&retired).is_ok_and(|metadata| metadata.is_dir()) => {} + Err(error) => return Err(err_502(error.to_string())), + } + std::fs::File::open(state.runtime_root.as_ref()) + .and_then(|directory| directory.sync_all()) + .map_err(|error| err_502(error.to_string())) +} + fn cleanup_config( state: &AppState, operation: &PreparedBootstrap, diff --git a/app/crowdb-web/src/node/cluster_status.rs b/app/crowdb-web/src/node/cluster_status.rs new file mode 100644 index 000000000..3d8c19ed3 --- /dev/null +++ b/app/crowdb-web/src/node/cluster_status.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Application readiness remains observable while initialization is unavailable. + +use crate::{ + error::{err_409, ErrorBody}, + state::AppState, +}; +use axum::{extract::State, http::StatusCode, Json}; +use crowdb_console_shared::{ + ops::{cluster::readiness, OpContext}, + ConsoleConfig, +}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::mgmt::{ + cluster::{ClusterStatus, DependencyKind, DependencyObservation, DependencyState, ReadinessReason}, + node::observation::NodeRuntimeObservation, +}; +use std::{ + sync::Arc, + time::{SystemTime, UNIX_EPOCH}, +}; + +type Failure = (StatusCode, Json); + +pub(crate) async fn read(State(state): State) -> Result, Failure> { + if state.node_monitor_url.is_none() { + return Err(err_409("Cluster observation requires an owning node monitor")); + } + let (local, failure) = match super::observation::read(State(state.clone())).await { + Ok(Json(local)) => (local, None), + Err(error) if error.0 == StatusCode::BAD_GATEWAY => (retained(&state)?, Some(error.1 .0.error)), + Err(error) => return Err(error), + }; + let seeds = local + .binding + .as_ref() + .map_or_else(Vec::new, |binding| binding.management_seeds.clone()); + let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); + let context = OpContext::with_shared_client_preserving_hint(client, &seeds, ConsoleConfig::default()); + let id = local.binding.as_ref().map_or_else( + || "local-node/observation".into(), + |binding| format!("node/{}/observation", binding.node_id), + ); + let mut report = readiness::collect(&context, local) + .await + .map_err(|_| err_409("Cluster observation identity or policy is invalid"))?; + if let Some(failure) = failure { + let detail = format!("Local monitor/runtime observation failed: {failure}; retained identity is available, but current execution and protocol health could not be observed"); + report.observation.dependencies.push(DependencyObservation { + id: id.clone(), + kind: DependencyKind::Member, + required: true, + state: DependencyState::Unknown, + detail: detail.clone(), + observed_at_unix_ms: report.observation.observed_at_unix_ms, + max_age_ms: 3000, + revision: None, + operation_id: None, + current_operation_id: None, + reason: Some(ReadinessReason { + dependency: id, + code: "node-observation-unavailable".into(), + detail, + }), + }); + let observed_at = report.observation.observed_at_unix_ms; + report = readiness::reduce(report.observation, observed_at); + } + Ok(Json(report)) +} + +fn retained(state: &AppState) -> Result { + let root = state + .runtime_root + .parent() + .ok_or_else(|| err_409("Node state root is absent"))?; + let identity = crowdb_monitor::NodeIdentity::load_existing(root) + .map_err(|_| err_409("Retained node identity is unavailable"))?; + let policy = crowdb_monitor::read_node_policy(root) + .map_err(|_| err_409("Retained node policy is invalid"))? + .ok_or_else(|| err_409("Retained node policy is absent"))?; + Ok(NodeRuntimeObservation { + version: 1, + discovery_id: identity.uuid().to_string(), + policy, + binding: super::cluster_binding(state)?, + setup: None, + observed_at_unix_ms: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |now| u64::try_from(now.as_millis()).unwrap_or(u64::MAX)), + services: Vec::new(), + protocols: Vec::new(), + }) +} diff --git a/app/crowdb-web/src/node/observation.rs b/app/crowdb-web/src/node/observation.rs new file mode 100644 index 000000000..0282c2a55 --- /dev/null +++ b/app/crowdb-web/src/node/observation.rs @@ -0,0 +1,183 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded local evidence for aggregate cluster observation. + +mod protocols; +mod services; +mod worker; +pub(crate) use worker::NodeObservations; +pub use worker::{start_node_observer, NodeObserver}; + +use crate::{ + error::{err_409, err_502}, + state::AppState, +}; +use axum::{extract::State, Json}; +use crowdb_console_shared::deployment::{admission::local_control, registry, services as service_intents}; +use crowdb_protocol::mgmt::node::{observation::NodeRuntimeObservation, NodeControl}; +use std::time::Duration; + +type Failure = (axum::http::StatusCode, Json); + +pub(crate) async fn read(State(state): State) -> Result, Failure> { + if state.node_monitor_url.is_none() { + return Err(err_409("Node observation requires an owning monitor")); + } + let observed = state + .node_observations + .latest + .load_full() + .ok_or_else(|| err_502("Backend node observation is pending"))?; + if observed.started.elapsed() > Duration::from_secs(3) { + return Err(err_502( + "Backend node observation expired; current health is unknown", + )); + } + match &observed.result { + Ok(value) => Ok(Json(value.clone())), + Err((status, error)) => Err((*status, Json(crate::error::ErrorBody { error: error.clone() }))), + } +} + +async fn observe(state: &AppState) -> Result { + let value = local_control(&super::control_socket(), &NodeControl::RuntimeObservation) + .await + .map_err(|_| err_502("Local monitor observation is unavailable"))?; + let mut observation: NodeRuntimeObservation = + serde_json::from_value(value).map_err(|_| err_502("Local monitor observation is invalid"))?; + observation + .policy + .validate() + .map_err(|_| err_409("Accepted node observation policy is invalid"))?; + if observation.version != 1 || uuid::Uuid::parse_str(&observation.discovery_id).is_err() { + return Err(err_409("Local monitor observation identity is invalid")); + } + if let Some(binding) = &observation.binding { + if !super::cluster_binding(state)?.as_ref().is_some_and(|current| { + current.node_id == binding.node_id && current.bootstrap == binding.bootstrap + }) { + return Err(err_409("Bound node and monitor observation identities differ")); + } + enrich(state, &mut observation).await; + } + Ok(observation) +} + +async fn enrich(state: &AppState, observation: &mut NodeRuntimeObservation) { + if let Some(binding) = &observation.binding { + let current = tokio::time::timeout(Duration::from_secs(1), async { + let kv = state.kv_client().await; + kv.set_mgmt_seeds(binding.management_seeds.clone()); + let (intents, registry) = + tokio::try_join!(service_intents::list(kv.as_ref()), registry::read(&kv))?; + let host = registry + .filter(|(registry, _)| registry.cluster_id == binding.bootstrap.cluster_id) + .and_then(|(registry, _)| { + registry.nodes.into_iter().find(|node| { + node.node_id == binding.node_id + && node.discovery_id == observation.discovery_id + && node.confirmed + && !node.cancelled + }) + }) + .map(|node| node.host) + .ok_or_else(|| { + crowdb_console_shared::error::Error::Config( + "Current confirmed local address is unavailable".into(), + ) + })?; + Ok::<_, crowdb_console_shared::error::Error>((intents, host)) + }) + .await + .ok() + .and_then(Result::ok); + let intents = current.as_ref().map(|(intents, _)| intents.as_slice()); + let own_host = current.as_ref().map(|(_, host)| host.as_str()); + let (protocols, service_details) = tokio::join!( + protocols::observe(state, observation, intents, own_host), + services::observe(observation, intents, own_host), + ); + observation.protocols = protocols; + for (id, detail) in service_details { + if let Some(service) = observation + .services + .iter_mut() + .find(|service| service.service_id == id) + { + service.detail = format!("{}; {detail}", service.detail); + } + } + } +} + +#[cfg(feature = "test-util")] +pub async fn protocol_probe_for_tests(protocol: &str, url: &str, token: Option<&str>) -> Option { + protocols::request_probe(protocol, url, token).await +} + +/// Execute the same signed local S3 probe used by node observation. +/// +/// # Errors +/// Returns the credential, transport, HTTP or response diagnostic. +#[cfg(feature = "test-util")] +pub async fn s3_protocol_probe_for_tests( + state: &AppState, + url: &str, +) -> Result { + protocols::signed_s3_probe(state, url).await +} + +/// Read bounded local Chunk-KV range evidence using a committed service config. +/// +/// # Errors +/// Returns a listener, identity, transport or response diagnostic. +#[cfg(feature = "test-util")] +pub async fn service_detail_for_tests( + state: &AppState, + intent: &crowdb_protocol::mgmt::node::NodeServiceIntent, +) -> Result { + let host = state + .config + .read() + .map_err(|_| "Invalid test configuration")? + .node(intent.node_id) + .map(|node| node.host.clone()); + services::read(intent, host.as_deref()).await +} + +/// Sample actual committed local intents and addresses without UI traffic. +/// Uses the production authority deadline, identity fences and HTTP probes. +#[cfg(feature = "test-util")] +pub async fn sample_node_services_for_tests(state: &AppState, observation: &mut NodeRuntimeObservation) { + enrich(state, observation).await; +} + +/// Start the production sampling loop with an injected observation source. +/// +/// # Errors +/// Rejects a duplicate worker or a state without an owning node monitor. +#[cfg(feature = "test-util")] +pub fn start_node_observer_for_tests( + state: &AppState, + source: F, +) -> Result +where + F: Fn() -> Fut + Send + 'static, + Fut: + std::future::Future> + Send, +{ + worker::start(state, source) +} + +#[cfg(feature = "test-util")] +#[must_use] +pub fn node_observation_time_for_tests(state: &AppState) -> Option { + state + .node_observations + .latest + .load() + .as_ref() + .and_then(|observed| observed.result.as_ref().ok()) + .map(|value| value.observed_at_unix_ms) +} diff --git a/app/crowdb-web/src/node/observation/protocols.rs b/app/crowdb-web/src/node/observation/protocols.rs new file mode 100644 index 000000000..f772508cb --- /dev/null +++ b/app/crowdb-web/src/node/observation/protocols.rs @@ -0,0 +1,305 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Authenticated, non-mutating observations on each required protocol listener. + +use crate::state::AppState; +use crowdb_protocol::mgmt::{ + cluster::{DependencyKind, DependencyObservation, DependencyState, ReadinessReason}, + node::{ + observation::NodeRuntimeObservation, setup::SetupServiceKind, NodeServiceAction, NodeServiceIntent, + }, +}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +mod response; +use response::list_buckets_body; + +const DATASET_PROBE: &str = "crowdb_readiness_probe"; +type ProbeResult = Result; + +pub(super) async fn observe( + state: &AppState, + node: &NodeRuntimeObservation, + intents: Option<&[NodeServiceIntent]>, + own_host: Option<&str>, +) -> Vec { + let Some(binding) = &node.binding else { + return Vec::new(); + }; + if !node + .policy + .setup + .services + .iter() + .any(|service| service.kind == SetupServiceKind::AccessServer && service.enabled) + { + return Vec::new(); + } + let id = format!("access-{}", binding.node_id); + let current = intents.and_then(|intents| { + intents.iter().find(|intent| { + intent.service_id == id + && intent.node_id == binding.node_id + && intent.cluster_id == binding.bootstrap.cluster_id + }) + }); + let runtime_ready = current.is_some_and(|current| { + node.services.iter().any(|service| { + service.service_id == current.service_id + && service.operation_id.as_ref() == Some(¤t.operation_id) + && service.state == DependencyState::Ready + }) + }); + futures::future::join_all( + ["s3", "iceberg", "dataset"] + .into_iter() + .map(|protocol| async move { + let result = if let Some(current) = current { + if matches!( + current.action, + NodeServiceAction::Stop | NodeServiceAction::Delete + ) { + Err(( + "operator-stopped-service", + "Access service is explicitly stopped or deleted".into(), + )) + } else if runtime_ready { + request(state, protocol, current, own_host).await + } else { + Err(( + "access-service-not-ready", + format!( + "Access service {} has no ready observation for current operation {}", + current.service_id, current.operation_id + ), + )) + } + } else { + Err(( + "access-intent-unavailable", + "Current committed Access service intent is unavailable".into(), + )) + }; + let status = match &result { + Ok(_) => DependencyState::Ready, + Err(("operator-stopped-service", _)) => DependencyState::Stopped, + Err(("access-service-not-ready", _)) => DependencyState::Initializing, + Err(_) => DependencyState::Unknown, + }; + let detail = match &result { + Ok(detail) | Err((_, detail)) => detail.clone(), + }; + let dependency = format!("node/{}/protocol/{protocol}", binding.node_id); + DependencyObservation { + id: dependency.clone(), + kind: DependencyKind::Protocol, + required: true, + state: status, + detail: detail.clone(), + observed_at_unix_ms: now_ms(), + max_age_ms: 3000, + revision: None, + operation_id: current.map(|intent| intent.operation_id.clone()), + current_operation_id: current.map(|intent| intent.operation_id.clone()), + reason: result.err().map(|(code, _)| ReadinessReason { + dependency, + code: code.into(), + detail, + }), + } + }), + ) + .await +} + +async fn request( + state: &AppState, + protocol: &str, + intent: &NodeServiceIntent, + own_host: Option<&str>, +) -> ProbeResult { + let invalid = || { + ( + "protocol-configuration-invalid", + format!("{protocol} has no valid local listener in the committed Access configuration"), + ) + }; + let config: toml::Value = toml::from_str(&intent.configuration).map_err(|_| invalid())?; + let address = config + .get(protocol) + .and_then(|value| value.get("listen")) + .and_then(toml::Value::as_str) + .ok_or_else(invalid)?; + let origin = internal_origin(own_host, address).ok_or_else(invalid)?; + if protocol == "s3" { + return signed_s3_probe(state, &format!("{origin}/")).await; + } + let path = match protocol { + "iceberg" => "/v1/config".into(), + "dataset" => format!("/api/dataset/v1/namespaces/dataset-ns/datasets/{DATASET_PROBE}"), + _ => return Err(invalid()), + }; + let credentials = + crowdb_monitor::ServerCredentials::load_existing(state.runtime_root.as_ref()).map_err(|_| { + ( + "protocol-credentials-unavailable", + format!("Retained {protocol} credentials are unavailable"), + ) + })?; + let token = if protocol == "iceberg" { + credentials.iceberg_read_token().to_owned() + } else { + credentials.dataset_token() + }; + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::AUTHORIZATION, + format!("Bearer {token}").parse().map_err(|_| { + ( + "protocol-credentials-invalid", + "Retained bearer credential is invalid".into(), + ) + })?, + ); + execute_probe(protocol, &format!("{origin}{path}"), headers).await +} + +pub(super) async fn signed_s3_probe(state: &AppState, url: &str) -> ProbeResult { + let parsed = reqwest::Url::parse(url).map_err(|_| { + ( + "protocol-configuration-invalid", + "Invalid local S3 listener".into(), + ) + })?; + let headers = crate::access::sign_local_s3_read(state, &parsed).map_err(|_| { + ( + "protocol-credentials-unavailable", + "Cannot sign S3 ListBuckets with retained local credentials".into(), + ) + })?; + execute_probe("s3", url, headers).await +} + +#[cfg(feature = "test-util")] +pub(super) async fn request_probe(protocol: &str, url: &str, token: Option<&str>) -> Option { + let mut headers = reqwest::header::HeaderMap::new(); + if let Some(token) = token { + headers.insert( + reqwest::header::AUTHORIZATION, + format!("Bearer {token}").parse().ok()?, + ); + } + match execute_probe(protocol, url, headers).await { + Ok(_) => Some(true), + Err(("protocol-http-status", _)) => Some(false), + Err(_) => None, + } +} + +async fn execute_probe(protocol: &str, url: &str, headers: reqwest::header::HeaderMap) -> ProbeResult { + let transport = || { + ( + "protocol-transport-unavailable", + format!("{protocol} read at {url} failed or exceeded its one-second deadline"), + ) + }; + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(1)) + .build() + .map_err(|_| transport())?; + let mut response = client + .get(url) + .headers(headers) + .send() + .await + .map_err(|_| transport())?; + let status = response.status().as_u16(); + if status != 200 && !(protocol == "dataset" && status == 404) { + return Err(( + "protocol-http-status", + format!( + "{protocol} read at {url} returned HTTP {status}; expected {}", + if protocol == "dataset" { + "200 or authenticated not-found 404" + } else { + "200" + } + ), + )); + } + let mut bytes = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(|_| transport())? { + if bytes.len() + chunk.len() > 65536 { + return Err(( + "protocol-response-too-large", + format!("{protocol} response at {url} exceeds 65536 bytes"), + )); + } + bytes.extend_from_slice(&chunk); + } + let invalid = || { + ( + "protocol-response-invalid", + format!("{protocol} read at {url} returned HTTP {status} with an unexpected protocol response"), + ) + }; + if protocol == "s3" { + return if list_buckets_body(&bytes) { + Ok(format!( + "S3 signed ListBuckets at {url}: HTTP 200, valid ListAllMyBucketsResult" + )) + } else { + Err(invalid()) + }; + } + let body: serde_json::Value = serde_json::from_slice(&bytes).map_err(|_| invalid())?; + let valid = match protocol { + "iceberg" => status == 200 && body["defaults"].is_object() && body["overrides"].is_object(), + "dataset" => { + (status == 404 && body["code"] == "not_found") + || (status == 200 && body["name"] == DATASET_PROBE && body["id"].is_string()) + } + _ => false, + }; + if valid { + Ok(format!( + "{protocol} authenticated read at {url}: HTTP {status}, valid protocol response" + )) + } else { + Err(invalid()) + } +} + +pub(super) fn internal_origin(own_host: Option<&str>, address: &str) -> Option { + let url = reqwest::Url::parse(&format!("http://{address}")).ok()?; + let host = url.host_str()?; + let own_host = own_host?; + if !["127.0.0.1", "localhost", "0.0.0.0", "[::]", "[::1]"].contains(&host) && host != own_host { + return None; + } + if !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + || url.path() != "/" + { + return None; + } + let host = match host { + "0.0.0.0" => "127.0.0.1", + "[::]" => "[::1]", + host => host, + }; + Some(format!("http://{host}:{}", url.port()?)) +} + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .and_then(|time| time.as_millis().try_into().ok()) + .unwrap_or(0) +} diff --git a/app/crowdb-web/src/node/observation/protocols/response.rs b/app/crowdb-web/src/node/observation/protocols/response.rs new file mode 100644 index 000000000..fd4a500e1 --- /dev/null +++ b/app/crowdb-web/src/node/observation/protocols/response.rs @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Validate the bounded S3 `ListBuckets` XML response. + +pub(super) fn list_buckets_body(bytes: &[u8]) -> bool { + use quick_xml::events::Event; + let mut reader = quick_xml::Reader::from_reader(bytes); + let mut depth = 0usize; + let mut root = false; + let mut buckets = false; + loop { + match reader.read_event() { + Ok(Event::Start(element)) => { + if depth == 0 { + if root || element.local_name().as_ref() != b"ListAllMyBucketsResult" { + return false; + } + root = true; + } + if depth == 1 && element.local_name().as_ref() == b"Buckets" { + buckets = true; + } + depth += 1; + } + Ok(Event::Empty(element)) => { + if depth == 1 && element.local_name().as_ref() == b"Buckets" { + buckets = true; + } else if depth == 0 { + return false; + } + } + Ok(Event::End(_)) => { + let Some(next) = depth.checked_sub(1) else { + return false; + }; + depth = next; + } + Ok(Event::Eof) => return root && buckets && depth == 0, + Ok(Event::DocType(_)) | Err(_) => return false, + _ => {} + } + } +} diff --git a/app/crowdb-web/src/node/observation/services.rs b/app/crowdb-web/src/node/observation/services.rs new file mode 100644 index 000000000..b8d29b8f2 --- /dev/null +++ b/app/crowdb-web/src/node/observation/services.rs @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Enrich supervised operation evidence with bounded local range diagnostics. + +mod access; + +use crowdb_protocol::mgmt::node::{ + observation::NodeRuntimeObservation, NodeServiceAction, NodeServiceIntent, +}; +use std::time::Duration; + +pub(super) async fn observe( + node: &NodeRuntimeObservation, + intents: Option<&[NodeServiceIntent]>, + own_host: Option<&str>, +) -> Vec<(String, String)> { + let Some(binding) = &node.binding else { + return Vec::new(); + }; + futures::future::join_all( + intents + .into_iter() + .flatten() + .filter(|intent| { + matches!(intent.kind.as_str(), "chunk-kv" | "access") + && intent.node_id == binding.node_id + && intent.cluster_id == binding.bootstrap.cluster_id + && !matches!(intent.action, NodeServiceAction::Stop | NodeServiceAction::Delete) + }) + .map(|intent| async move { + let detail = read(intent, own_host).await.unwrap_or_else(|detail| detail); + (intent.service_id.clone(), detail) + }), + ) + .await +} + +pub(super) async fn read(intent: &NodeServiceIntent, own_host: Option<&str>) -> Result { + if intent.kind == "access" { + return access::read(intent, own_host).await; + } + let invalid = || "Chunk-KV current intent has no valid local management listener".to_owned(); + let config: toml::Value = toml::from_str(&intent.configuration).map_err(|_| invalid())?; + let address = config + .get("http_listen_addr") + .and_then(toml::Value::as_str) + .ok_or_else(invalid)?; + let origin = super::protocols::internal_origin(own_host, address).ok_or_else(invalid)?; + let expected = config + .get("instance_id") + .and_then(toml::Value::as_integer) + .ok_or_else(invalid)?; + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(1)) + .build() + .map_err(|_| invalid())?; + let mut response = client.get(format!("{origin}/health")).send().await.map_err(|_| { + format!("Chunk-KV health at {origin} is unreachable or exceeded its one-second deadline") + })?; + if !response.status().is_success() { + return Err(format!( + "Chunk-KV health at {origin} returned HTTP {}", + response.status() + )); + } + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| "Chunk-KV health body is unavailable".to_owned())? + { + if bytes.len() + chunk.len() > 65_536 { + return Err("Chunk-KV health body exceeds 65536 bytes".into()); + } + bytes.extend_from_slice(&chunk); + } + let body: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|_| "Chunk-KV health body is invalid JSON".to_owned())?; + if body["instance_id"].as_i64() != Some(expected) { + return Err("Chunk-KV health instance differs from current committed intent".into()); + } + let detail = body["detail"] + .as_str() + .ok_or("Chunk-KV health has no detailed readiness evidence")?; + let ranges = body["ranges"] + .as_array() + .ok_or("Chunk-KV health has no range ownership evidence")?; + Ok(format!( + "{detail}; ranges: {}", + serde_json::to_string(ranges).map_err(|_| "Chunk-KV range evidence cannot be encoded")? + )) +} diff --git a/app/crowdb-web/src/node/observation/services/access.rs b/app/crowdb-web/src/node/observation/services/access.rs new file mode 100644 index 000000000..3a5ab56e2 --- /dev/null +++ b/app/crowdb-web/src/node/observation/services/access.rs @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Exact Access readiness diagnostics even while the process is not ready. + +use crowdb_protocol::mgmt::node::NodeServiceIntent; +use std::time::Duration; + +pub(super) async fn read(intent: &NodeServiceIntent, own_host: Option<&str>) -> Result { + let invalid = || "Access current intent has no valid local health listener".to_owned(); + let config: toml::Value = toml::from_str(&intent.configuration).map_err(|_| invalid())?; + let address = config + .get("health") + .and_then(|health| health.get("listen")) + .and_then(toml::Value::as_str) + .ok_or_else(invalid)?; + let origin = super::super::protocols::internal_origin(own_host, address).ok_or_else(invalid)?; + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(1)) + .build() + .map_err(|_| invalid())?; + let mut response = client + .get(format!("{origin}/_crowdb/health/ready")) + .send() + .await + .map_err(|_| "Access readiness is unreachable or exceeded its one-second deadline".to_owned())?; + let status = response.status(); + if status.as_u16() != 200 && status.as_u16() != 503 { + return Err(format!("Access readiness returned HTTP {status}")); + } + let mut bytes = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|_| "Access readiness body is unavailable")? + { + if bytes.len() + chunk.len() > 65_536 { + return Err("Access readiness body exceeds 65536 bytes".into()); + } + bytes.extend_from_slice(&chunk); + } + let body: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|_| "Access readiness body is invalid JSON")?; + if body["ready"].as_bool() != Some(status.as_u16() == 200) { + return Err("Access readiness HTTP status and ready flag disagree".into()); + } + let fields = [ + "listener", + "metadata", + "chunks", + "native_pool", + "cleanup", + "authentication", + ]; + let flags = fields + .into_iter() + .map(|name| { + let value = body[name] + .as_str() + .filter(|value| ["ready", "busy", "unavailable", "unknown"].contains(value)) + .ok_or_else(|| format!("Access readiness has no valid {name} state"))?; + Ok(format!("{name}={value}")) + }) + .collect::, String>>()?; + Ok(format!( + "Access readiness HTTP {status}: {}; details: {}; observation: {}; cleanup backlog: {}/{}", + flags.join(", "), + body["details"], + body["observation"], + body["cleanup_backlog"], + body["cleanup_backlog_limit"] + )) +} diff --git a/app/crowdb-web/src/node/observation/worker.rs b/app/crowdb-web/src/node/observation/worker.rs new file mode 100644 index 000000000..57f913eeb --- /dev/null +++ b/app/crowdb-web/src/node/observation/worker.rs @@ -0,0 +1,103 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Node protocol and service probes run independently of status readers. + +use crate::state::AppState; +use arc_swap::ArcSwapOption; +use axum::http::StatusCode; +use crowdb_protocol::mgmt::node::observation::NodeRuntimeObservation; +use std::{ + future::Future, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; +use tokio::time::{Instant, MissedTickBehavior}; + +type ObservationResult = Result; + +#[derive(Default)] +pub(crate) struct NodeObservations { + pub(super) latest: ArcSwapOption, + running: AtomicBool, +} + +pub(super) struct CachedObservation { + pub result: ObservationResult, + pub started: Instant, +} + +/// Cancels the node's background sampler when the Web listener exits. +pub struct NodeObserver(tokio::task::JoinHandle<()>); +impl Drop for NodeObserver { + fn drop(&mut self) { + self.0.abort(); + } +} + +struct SamplingLease(Arc); +impl Drop for SamplingLease { + fn drop(&mut self) { + self.0.running.store(false, Ordering::Release); + } +} + +/// Start independent protocol and service observations for a monitor-owned Web. +/// +/// # Errors +/// Rejects another active sampling worker. +pub fn start_node_observer(state: &AppState) -> Result, &'static str> { + if state.node_monitor_url.is_none() { + return Ok(None); + } + let sample_state = state.clone(); + start(state, move || { + let state = sample_state.clone(); + async move { + super::observe(&state) + .await + .map_err(|(status, body)| (status, body.0.error)) + } + }) + .map(Some) +} + +pub(super) fn start(state: &AppState, source: F) -> Result +where + F: Fn() -> Fut + Send + 'static, + Fut: Future + Send, +{ + if state.node_monitor_url.is_none() { + return Err("Node sampling requires an owning monitor"); + } + state + .node_observations + .running + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .map_err(|_| "Node observation sampler is already running")?; + let observations = Arc::clone(&state.node_observations); + let lease = SamplingLease(Arc::clone(&observations)); + Ok(NodeObserver(tokio::spawn(async move { + let _lease = lease; + let mut interval = tokio::time::interval(Duration::from_secs(1)); + interval.set_missed_tick_behavior(MissedTickBehavior::Skip); + loop { + interval.tick().await; + let started = Instant::now(); + let result = tokio::time::timeout(Duration::from_secs(3), source()) + .await + .unwrap_or_else(|_| { + Err(( + StatusCode::BAD_GATEWAY, + "Backend node observation exceeded its three-second deadline".into(), + )) + }); + observations + .latest + .store(Some(Arc::new(CachedObservation { result, started }))); + } + }))) +} diff --git a/app/crowdb-web/src/node/preflight.rs b/app/crowdb-web/src/node/preflight.rs index 52ba57e4a..32a7269fc 100644 --- a/app/crowdb-web/src/node/preflight.rs +++ b/app/crowdb-web/src/node/preflight.rs @@ -24,6 +24,7 @@ pub(crate) struct PreparedAccess { operation_id: String, inventory: DiskInventory, accepted_choices: Option, + policy_choices: Option, } #[derive(Debug, Serialize)] @@ -120,6 +121,7 @@ pub(crate) async fn prepare( ssh_user: body.ssh_user.clone(), ssh_port: body.ssh_port, operation_id: uuid::Uuid::new_v4().to_string(), + host_key: None, confirmed: false, cancelled: false, } @@ -138,6 +140,7 @@ pub(crate) async fn prepare( &[ NodeControl::Identity, NodeControl::DiskInventory, + NodeControl::DeploymentPolicy, NodeControl::NodeSetupIntent, ], ) @@ -159,6 +162,16 @@ pub(crate) async fn prepare( "Candidate has another accepted setup; resume its original admission", )); } + let policy: Option = + serde_json::from_value( + observed + .pop() + .ok_or_else(|| err_502("Missing target startup policy"))?, + ) + .map_err(|error| err_502(error.to_string()))?; + if let Some(policy) = &policy { + policy.validate().map_err(err_400)?; + } let inventory = serde_json::from_value( observed .pop() @@ -169,6 +182,7 @@ pub(crate) async fn prepare( operation_id: record.operation_id, inventory, accepted_choices: accepted.map(|request| request.choices), + policy_choices: policy.map(|policy| policy.setup), })) } diff --git a/app/crowdb-web/src/node/setup.rs b/app/crowdb-web/src/node/setup.rs index faa2f4cb4..919590ef4 100644 --- a/app/crowdb-web/src/node/setup.rs +++ b/app/crowdb-web/src/node/setup.rs @@ -27,6 +27,7 @@ pub(super) async fn accept( record: &NodeRecord, target: &NodeEntry, choices: Option, + expected: Option, ) -> Result<(), Failure> { let mut target = target.clone(); target.ssh_password = None; @@ -37,13 +38,43 @@ pub(super) async fn accept( .map_err(map_config_err)?, ) .map_err(|error| err_502(error.to_string()))?; - let choices = choices - .or_else(|| { - existing - .filter(|request| request.operation_id == record.operation_id) - .map(|request| request.choices) - }) - .unwrap_or_default(); + let policy: Option = + serde_json::from_value( + remote_control(&target, &NodeControl::DeploymentPolicy) + .await + .map_err(map_config_err)?, + ) + .map_err(|_| err_502("Target deployment policy is invalid"))?; + let choices = if let Some(policy) = policy { + policy.validate().map_err(crate::error::err_400)?; + if expected + .as_ref() + .is_some_and(|expected| *expected != policy.formation()) + { + return Err(err_409( + "Authenticated target formation policy differs from discovery", + )); + } + if choices.as_ref().is_some_and(|choices| *choices != policy.setup) { + return Err(err_409( + "Setup overrides differ from the target's accepted deployment policy", + )); + } + policy.setup + } else { + if expected.is_some() { + return Err(err_409( + "Automatic formation requires an accepted target deployment policy", + )); + } + choices + .or_else(|| { + existing + .filter(|request| request.operation_id == record.operation_id) + .map(|request| request.choices) + }) + .unwrap_or_default() + }; let request = NodeSetupRequest { version: 1, discovery_id: record.discovery_id.clone(), diff --git a/app/crowdb-web/src/node/target.rs b/app/crowdb-web/src/node/target.rs new file mode 100644 index 000000000..952b40eea --- /dev/null +++ b/app/crowdb-web/src/node/target.rs @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Confirmed remote attach identity, without granting runtime ownership. + +use axum::{extract::State, http::StatusCode, Json}; +use crowdb_console_shared::deployment::{registry, PreparedBootstrap, CLUSTER_KEY}; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_protocol::mgmt::cluster::ClusterTargetIdentity; + +use crate::{ + error::{err_409, err_502, map_config_err}, + state::AppState, +}; + +type Failure = (StatusCode, Json); + +pub(crate) async fn identity(State(state): State) -> Result, Failure> { + tokio::time::timeout(std::time::Duration::from_secs(3), observe(&state)) + .await + .map_err(|_| err_502("Confirmed cluster target observation timed out"))? + .map(Json) +} + +async fn observe(state: &AppState) -> Result { + let kv = state.kv_client().await; + let GetOutcome::Found { value, .. } = kv + .get(0, 0, CLUSTER_KEY, ReadMode::Linearizable, None) + .await + .map_err(|_| err_502("Confirmed cluster target is unavailable"))? + else { + return Err(err_409("Cluster identity has not been committed")); + }; + if value.len() > 1024 * 1024 { + return Err(err_409("Committed cluster identity exceeds its size bound")); + } + let prepared: PreparedBootstrap = + serde_json::from_slice(&value).map_err(|_| err_409("Committed cluster identity is invalid"))?; + prepared.validate().map_err(map_config_err)?; + if let Some(binding) = super::cluster_binding(state)? { + if binding.bootstrap != prepared.identity { + return Err(err_409("Bound and committed cluster identities differ")); + } + } + let config = prepared.intent.to_config(); + let current = registry::read(&kv).await.map_err(map_config_err)?; + if current + .as_ref() + .is_some_and(|(registry, _)| registry.cluster_id != prepared.identity.cluster_id) + { + return Err(err_409("Confirmed node registry belongs to another cluster")); + } + let mut management = Vec::new(); + let mut rpc = Vec::new(); + for member in prepared.intent.members() { + let server = config + .server_for_node(*member) + .ok_or_else(|| err_409("Committed cluster member has no management endpoint"))?; + let rpc_url = server + .rpc_url + .as_deref() + .ok_or_else(|| err_409("Committed cluster member has no native RPC endpoint"))?; + if let Some((registry, _)) = ¤t { + let node = registry + .nodes + .iter() + .find(|node| node.node_id == *member && node.confirmed && !node.cancelled) + .ok_or_else(|| err_409("Confirmed bootstrap member is unavailable"))?; + if prepared + .nodes + .iter() + .find(|expected| expected.node_id == *member) + .is_some_and(|expected| expected.discovery_id != node.discovery_id) + { + return Err(err_409("Confirmed member discovery identity differs")); + } + management.push(with_host(&server.url, &node.host)?); + rpc.push( + with_host(rpc_url, &node.host)? + .trim_start_matches("http://") + .to_owned(), + ); + } else { + management.push(server.url.clone()); + rpc.push(rpc_url.to_owned()); + } + } + Ok(ClusterTargetIdentity { + version: 1, + cluster_id: prepared.identity.cluster_id, + group0_management_endpoints: management, + group0_rpc_endpoints: rpc, + }) +} + +fn with_host(endpoint: &str, host: &str) -> Result { + let endpoint = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { + endpoint.to_owned() + } else { + format!("http://{endpoint}") + }; + let mut url = reqwest::Url::parse(&endpoint).map_err(|_| err_409("Invalid committed member endpoint"))?; + let host = if host.contains(':') && !host.starts_with('[') { + format!("[{host}]") + } else { + host.to_owned() + }; + url.set_host(Some(&host)) + .map_err(|_| err_409("Invalid confirmed member host"))?; + Ok(url.as_str().trim_end_matches('/').to_owned()) +} diff --git a/app/crowdb-web/src/services/access.rs b/app/crowdb-web/src/services/access.rs index ec72e3cf2..dd7d0e42e 100644 --- a/app/crowdb-web/src/services/access.rs +++ b/app/crowdb-web/src/services/access.rs @@ -20,12 +20,16 @@ pub(crate) fn origin(state: &AppState, protocol: &str) -> Option { if local.is_some() { return local; } + if let Some(origin) = config + .servers + .iter() + .filter(|service| service.service_type == ServiceType::AccessServer) + .find_map(|service| service.access_origins.get(protocol).cloned()) + { + return Some(origin); + } if protocol == "dataset" { - return config - .servers - .iter() - .filter(|service| service.service_type == ServiceType::AccessServer) - .find_map(|service| service.dataset_origin.clone()); + return None; } // Remote node launches have no local process specification. Their normal // provisioning persists the client origins alongside cluster credentials. diff --git a/app/crowdb-web/src/services/defaults.rs b/app/crowdb-web/src/services/defaults.rs index 81c3d5f80..976320edd 100644 --- a/app/crowdb-web/src/services/defaults.rs +++ b/app/crowdb-web/src/services/defaults.rs @@ -26,7 +26,7 @@ fn occupied(config: &ConsoleConfig) -> HashSet { ports.extend(port(&server.url)); ports.extend(server.rpc_url.as_deref().and_then(port)); ports.extend(server.rest_port); - ports.extend(server.dataset_origin.as_deref().and_then(port)); + ports.extend(server.access_origins.values().filter_map(|origin| port(origin))); ports.extend(server.rpc_port); if server.service_type == ServiceType::Diskdb { if let Some(rpc) = server.rpc_url.as_deref().and_then(port) { diff --git a/app/crowdb-web/src/services/dependencies.rs b/app/crowdb-web/src/services/dependencies.rs index ee7894402..d438c5c4a 100644 --- a/app/crowdb-web/src/services/dependencies.rs +++ b/app/crowdb-web/src/services/dependencies.rs @@ -49,7 +49,7 @@ pub(super) async fn storage_wait_reason(state: &AppState) -> Option { let Ok(instances) = registry.read_all_diskio_instances().await else { return Some("Waiting: live DiskIO ownership registry is unavailable".into()); }; - let hardware = HardwareClient::from_shared(client); + let hardware = HardwareClient::from_shared(client.clone()); let Ok(groups) = hardware.list_disk_groups().await else { return Some("Waiting: disk-group hardware is unavailable".into()); }; @@ -77,6 +77,9 @@ pub(super) async fn storage_wait_reason(state: &AppState) -> Option { if groups.iter().any(|group| !owned.contains(&group.dg_id)) { return Some("Waiting: every configured disk group needs live DiskIO ownership".into()); } + if let Err(error) = crowdb_console_shared::deployment::services::diskdb_storage_ready(client).await { + return Some(format!("Waiting: {error}")); + } None } diff --git a/app/crowdb-web/src/services/deployment.rs b/app/crowdb-web/src/services/deployment.rs index 07c1305f9..0829e2c1a 100644 --- a/app/crowdb-web/src/services/deployment.rs +++ b/app/crowdb-web/src/services/deployment.rs @@ -21,6 +21,7 @@ mod credentials; mod geometry; mod launch; pub(super) mod planned; +mod policy; mod remote; #[derive(Clone, Copy, Deserialize, Serialize)] @@ -78,9 +79,18 @@ pub(super) struct Deploy { #[serde(default)] stream_chunk_capacity_bytes: Option, #[serde(default)] - stream_extent_page_entries: Option, + local_deployment: Option, #[serde(default)] - test_single_node: bool, + local_mirror_copies: Option, +} + +impl Deploy { + fn local(&self) -> bool { + self.local_deployment.unwrap_or(false) + } + fn local_copies(&self) -> u32 { + self.local_mirror_copies.unwrap_or(1) + } } fn deserialize_id<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { @@ -111,8 +121,9 @@ pub(super) async fn deploy( async fn deploy_inner( state: AppState, node_id: u64, - body: Deploy, + mut body: Deploy, ) -> Result<(StatusCode, Json), Failure> { + policy::apply(&state, &mut body)?; validate(&body)?; if state.node_monitor_url.is_some() { return remote::deploy(&state, node_id, &body).await; @@ -140,7 +151,7 @@ async fn deploy_inner( .collect(); let port_claim = super::defaults::claim_ports(&state, &ports)?; let (node, seeds) = inputs(&state, node_id, &id, &body)?; - if matches!(body.kind, Kind::Chunkdb | Kind::ChunkKv) && !body.test_single_node { + if matches!(body.kind, Kind::Chunkdb | Kind::ChunkKv) && !body.local() { if let Some(reason) = super::dependencies::storage_wait_reason(&state).await { return Err(err_409(reason)); } @@ -162,6 +173,14 @@ async fn deploy_inner( } fn validate(body: &Deploy) -> Result<(), Failure> { + if body + .local_mirror_copies + .is_some_and(|copies| !body.local() || !matches!(copies, 1 | 2)) + { + return Err(err_400( + "Local mirror copies must be one or two and require local deployment", + )); + } if body.instance_id == 0 || body.instance_id > (u64::MAX >> 1) { return Err(err_400("Instance ID must be in 1..=9223372036854775807")); } @@ -194,8 +213,7 @@ fn validate(body: &Deploy) -> Result<(), Failure> { || (!matches!(body.kind, Kind::ChunkKv) && (body.metadata_store_id.is_some() || body.bootstrap_group_id.is_some() - || body.stream_chunk_capacity_bytes.is_some() - || body.stream_extent_page_entries.is_some())) + || body.stream_chunk_capacity_bytes.is_some())) || (!matches!(body.kind, Kind::AccessServer) && body.s3_port.is_some()) || (!matches!(body.kind, Kind::AccessServer) && body.dataset_port.is_some()) || (!matches!(body.kind, Kind::AccessServer) && body.health_port.is_some()) @@ -266,7 +284,7 @@ fn inputs( }); let origins = std::iter::once(service.url.as_str()) .chain(service.rpc_url.as_deref()) - .chain(service.dataset_origin.as_deref()) + .chain(service.access_origins.values().map(String::as_str)) .chain(public); for origin in origins { let normalized = if origin.contains("://") { @@ -309,9 +327,7 @@ fn inputs( .iter() .filter(|disk| disk.node_id == node_id && disk.disk_group_id == group) .collect(); - if !body.test_single_node - && (disks.is_empty() || disks.iter().any(|disk| disk.device_path.is_empty())) - { + if !body.local() && (disks.is_empty() || disks.iter().any(|disk| disk.device_path.is_empty())) { return Err(err_400( "Production DiskIO requires configured disks with device paths; configure them in Capacity", )); diff --git a/app/crowdb-web/src/services/deployment/credentials.rs b/app/crowdb-web/src/services/deployment/credentials.rs index 7be817928..fe3963958 100644 --- a/app/crowdb-web/src/services/deployment/credentials.rs +++ b/app/crowdb-web/src/services/deployment/credentials.rs @@ -76,6 +76,7 @@ pub(super) async fn prepare( server .persist_client(&ClientCredentials { s3_endpoint, + dataset_endpoint: spec.env.get("CROWDB_DATASET_PUBLIC_URI").cloned(), iceberg_endpoint, region: "us-east-1".into(), access_key_id: get("AWS_ACCESS_KEY_ID=")?, diff --git a/app/crowdb-web/src/services/deployment/launch.rs b/app/crowdb-web/src/services/deployment/launch.rs index 9750298f2..c5282e5bf 100644 --- a/app/crowdb-web/src/services/deployment/launch.rs +++ b/app/crowdb-web/src/services/deployment/launch.rs @@ -42,13 +42,14 @@ pub(super) async fn launch( err_502("Chunk slot initialization timed out; reconcile the layout before retrying") })??; let request = ChunkdbDeployRequest { + local_deployment: body.local(), dynamic_ownership: body.dynamic_ownership, server_id: id.into(), instance_id: body.instance_id, http_port: body.http_port.unwrap(), rpc_port: body.rpc_port.unwrap(), kv_server_mgmt_seeds: seeds.to_vec(), - allow_unsafe_ec: body.test_single_node, + allow_unsafe_ec: false, rpc_workers: None, kv_connections: None, kv_client_rpc_workers: None, @@ -179,14 +180,14 @@ pub(super) async fn chunk_kv_config( "rpc_advertise_addr":format!("{}:{}",node.host,body.rpc_port.unwrap()), "http_listen_addr":format!("{}:{}",node.host,body.http_port.unwrap()), "http_advertise_addr":format!("{}:{}",node.host,body.http_port.unwrap()), - "group0_mgmt_seeds":seeds, "storage":{"metadata_store_id":store,"stream_mirror_copies":if body.test_single_node {1} else {2}}, + "group0_mgmt_seeds":seeds, "storage":{"metadata_store_id":store,"stream_mirror_copies":if body.local() {body.local_copies()} else {2}}, }); + if body.local() { + config["storage"]["tree_strip_capacity_bytes"] = json!(1024 * 1024); + } if let Some(bytes) = body.stream_chunk_capacity_bytes { config["storage"]["stream_chunk_capacity_bytes"] = json!(bytes); } - if let Some(entries) = body.stream_extent_page_entries { - config["storage"]["stream_extent_page_entries"] = json!(entries); - } if let Some(group) = body.bootstrap_group_id { let client = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new(seeds.to_vec())); @@ -221,14 +222,15 @@ pub(super) fn access_config(body: &Deploy, seeds: &[String], node: &NodeEntry) - if let Some(port) = body.dataset_port { config["dataset"] = json!({"listen":format!("{}:{port}",node.host),"request_timeout_seconds":60}); } - if body.test_single_node { - config["deployment"] = json!({"mode":"test_single_node","max_node_failures":0}); - config["small_write"] = json!({"mirror_copies":1,"conversion_enabled":false,"ec_data":2,"ec_code":1}); + if body.local() { + config["deployment"] = json!({"mode":"local","max_node_failures":0}); + config["small_write"] = + json!({"mirror_copies":body.local_copies(),"conversion_enabled":false,"ec_data":2,"ec_code":1}); for name in ["s3", "iceberg"] { - config[name]["large_mirror_copies"] = json!(1); + config[name]["large_mirror_copies"] = json!(body.local_copies()); } config["s3"]["small_write"] = - json!({"mirror_copies":1,"conversion_enabled":false,"ec_data":2,"ec_code":1}); + json!({"mirror_copies":body.local_copies(),"conversion_enabled":false,"ec_data":2,"ec_code":1}); } config } diff --git a/app/crowdb-web/src/services/deployment/planned.rs b/app/crowdb-web/src/services/deployment/planned.rs index 9015f07bb..c47679e1e 100644 --- a/app/crowdb-web/src/services/deployment/planned.rs +++ b/app/crowdb-web/src/services/deployment/planned.rs @@ -24,6 +24,8 @@ use crowdb_protocol::mgmt::node::{ use serde::Deserialize; use serde_json::{json, Value}; +mod catalog; + #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct Prepare { @@ -169,10 +171,11 @@ async fn build( .remove("dynamic_ownership"); } if selected.kind == SetupServiceKind::ChunkKv { - metadata(state, &mut choices).await?; + metadata(state, binding, &mut choices).await?; } - let body: super::Deploy = + let mut body: super::Deploy = serde_json::from_value(choices).map_err(|error| err_400(error.to_string()))?; + super::policy::apply(state, &mut body)?; super::validate(&body)?; if matches!( selected.kind, @@ -191,9 +194,16 @@ async fn build( if selected.kind == SetupServiceKind::AccessServer { super::remote::prepare_credentials(state, binding, &node, &body).await?; } + let configuration = + super::remote::configuration(state, &node, &id, &binding.management_seeds, &body).await?; + let configuration = if selected.kind == SetupServiceKind::ChunkKv { + catalog::configure(state, binding, configuration).await? + } else { + configuration + }; ( - super::remote::configuration(state, &node, &id, &binding.management_seeds, &body).await?, - super::remote::environment(&body, &node), + configuration, + super::remote::owned_environment(state, &body, &node)?, ) }; Ok(NodeServiceIntent { @@ -219,20 +229,11 @@ fn service_name(kind: SetupServiceKind) -> &'static str { } } -async fn metadata(state: &AppState, choices: &mut Value) -> Result<(), Failure> { - let group = state - .op_context() - .await - .map_err(map_config_err)? - .sysmd() - .list_all_groups() +async fn metadata(state: &AppState, binding: &NodeBinding, choices: &mut Value) -> Result<(), Failure> { + let ctx = state.op_context().await.map_err(map_config_err)?; + let group = crowdb_console_shared::deployment::preparation::metadata_binding(&ctx, &binding.bootstrap) .await - .map_err(|error| map_config_err(error.into()))? - .into_iter() - .filter(|group| group.group_id != 0) - .min_by_key(|group| (group.store_id, group.group_id)) - .ok_or_else(|| err_409("Select an existing ordinary metadata group before starting Chunk-KV"))?; + .map_err(map_config_err)?; choices["metadata_store_id"] = json!(group.store_id); - choices["bootstrap_group_id"] = json!(group.group_id); Ok(()) } diff --git a/app/crowdb-web/src/services/deployment/planned/catalog.rs b/app/crowdb-web/src/services/deployment/planned/catalog.rs new file mode 100644 index 000000000..2851f584a --- /dev/null +++ b/app/crowdb-web/src/services/deployment/planned/catalog.rs @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Only the accepted initial owner may initialize the shared Chunk-KV catalog. + +use crowdb_protocol::mgmt::node::NodeBinding; + +use crate::{ + error::{err_502, map_config_err}, + services::Failure, + state::AppState, +}; + +pub(super) async fn configure( + state: &AppState, + binding: &NodeBinding, + configuration: String, +) -> Result { + let ctx = state.op_context().await.map_err(map_config_err)?; + let accepted = crowdb_console_shared::deployment::preparation::chunk_kv_binding(&ctx, &binding.bootstrap) + .await + .map_err(map_config_err)?; + let mut config: toml::Value = + toml::from_str(&configuration).map_err(|error| err_502(error.to_string()))?; + if accepted.node_id == binding.node_id { + let mut bootstrap = toml::Value::try_from(accepted).map_err(|error| err_502(error.to_string()))?; + bootstrap + .as_table_mut() + .expect("bootstrap table") + .remove("node_id"); + config + .as_table_mut() + .ok_or_else(|| err_502("Chunk-KV configuration must be a table"))? + .insert("bootstrap_partition".into(), bootstrap); + } + toml::to_string_pretty(&config).map_err(|error| err_502(error.to_string())) +} diff --git a/app/crowdb-web/src/services/deployment/policy.rs b/app/crowdb-web/src/services/deployment/policy.rs new file mode 100644 index 000000000..f61c77fee --- /dev/null +++ b/app/crowdb-web/src/services/deployment/policy.rs @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::mgmt::node::deployment::DeploymentMode; + +use super::Deploy; +use crate::{ + error::{err_409, err_502}, + services::Failure, + state::AppState, +}; + +pub(super) fn apply(state: &AppState, body: &mut Deploy) -> Result<(), Failure> { + if state.node_monitor_url.is_some() { + let root = state + .runtime_root + .parent() + .ok_or_else(|| err_409("Node deployment policy root is absent"))?; + if let Some(policy) = + crowdb_monitor::read_node_policy(root).map_err(|error| err_502(error.to_string()))? + { + let local = policy.mode == DeploymentMode::Local; + if body.local() && !local { + return Err(err_409( + "Production deployment cannot use local service placement", + )); + } + body.local_deployment = Some(body.local_deployment.unwrap_or(local)); + if body.local() && body.local_mirror_copies.is_none() { + body.local_mirror_copies = Some(if policy.initial_voters == 1 { 1 } else { 2 }); + } + } + } + Ok(()) +} diff --git a/app/crowdb-web/src/services/deployment/remote.rs b/app/crowdb-web/src/services/deployment/remote.rs index 935b38921..cf0d6d7e6 100644 --- a/app/crowdb-web/src/services/deployment/remote.rs +++ b/app/crowdb-web/src/services/deployment/remote.rs @@ -33,7 +33,7 @@ pub(super) async fn deploy( let id = format!("{}-{}", body.kind.name(), body.instance_id); let _operation = crate::services::Operation::claim(state, vec![format!("node/{node_id}"), format!("service/{id}")])?; - if matches!(body.kind, Kind::Chunkdb | Kind::ChunkKv) && !body.test_single_node { + if matches!(body.kind, Kind::Chunkdb | Kind::ChunkKv) && !body.local() { super::geometry::validate(state).await?; if let Some(reason) = crate::services::dependencies::storage_wait_reason(state).await { return Err(err_409(reason)); @@ -60,7 +60,7 @@ pub(super) async fn deploy( }, action: NodeServiceAction::Start, configuration, - environment: environment(body, &node), + environment: owned_environment(state, body, &node)?, }; let result = match crowdb_console_shared::deployment::services::execute( state.kv_client().await.as_ref(), @@ -94,7 +94,15 @@ pub(super) async fn deploy( entry.rpc_url = body.rpc_port.map(|port| format!("{host}:{port}")); entry.rest_port = body.http_port; if matches!(body.kind, Kind::AccessServer) { - entry.dataset_origin = body.dataset_port.map(|port| format!("http://{host}:{port}")); + entry.access_origins.insert("iceberg".into(), entry.url.clone()); + entry + .access_origins + .insert("s3".into(), format!("http://{host}:{}", body.s3_port.unwrap())); + if let Some(port) = body.dataset_port { + entry + .access_origins + .insert("dataset".into(), format!("http://{host}:{port}")); + } } entry.rpc_port = body.rpc_port; entry.auto_start = true; @@ -135,7 +143,7 @@ pub(super) async fn prepare_credentials( program: "/opt/crowdb/bin/crowdb-access-server".into(), args: Vec::new(), workdir: state.runtime_root.to_string_lossy().into_owned(), - env: environment(body, node), + env: owned_environment(state, body, node)?, env_file: None, readiness_url: None, }; @@ -160,12 +168,13 @@ pub(super) async fn configuration( super::chunk_slots::prepare(state, body.instance_id, body.dynamic_ownership).await?; Ok(lifecycle::chunkdb_config( &ChunkdbDeployRequest { + local_deployment: body.local(), server_id: id.into(), instance_id: body.instance_id, http_port: body.http_port.unwrap(), rpc_port: body.rpc_port.unwrap(), kv_server_mgmt_seeds: seeds.to_vec(), - allow_unsafe_ec: body.test_single_node, + allow_unsafe_ec: false, dynamic_ownership: body.dynamic_ownership, rpc_workers: None, kv_connections: None, @@ -205,6 +214,28 @@ fn encode(value: Value) -> Result { toml::to_string_pretty(&value).map_err(|error| err_502(error.to_string())) } +pub(super) fn owned_environment( + state: &AppState, + body: &Deploy, + node: &NodeEntry, +) -> Result, Failure> { + let mut env = environment(body, node); + if matches!(body.kind, Kind::AccessServer) + && crate::node::cluster_binding(state)?.is_some_and(|binding| binding.node_id == node.id) + { + let root = state + .runtime_root + .parent() + .ok_or_else(|| err_409("Node root is absent"))?; + if let Some(policy) = + crowdb_monitor::read_node_policy(root).map_err(|error| err_502(error.to_string()))? + { + env.extend(policy.public_access.environment()); + } + } + Ok(env) +} + pub(super) fn environment(body: &Deploy, node: &NodeEntry) -> BTreeMap { let mut env = BTreeMap::new(); if matches!(body.kind, Kind::Chunkdb) { diff --git a/app/crowdb-web/src/services/observation.rs b/app/crowdb-web/src/services/observation.rs index 4fbfa94f0..6cbb729c6 100644 --- a/app/crowdb-web/src/services/observation.rs +++ b/app/crowdb-web/src/services/observation.rs @@ -27,6 +27,8 @@ pub struct ServerSummary { /// Live pid if the console currently tracks the process. #[serde(skip_serializing_if = "Option::is_none")] pub pid: Option, + /// Accepted start awaits an owned PID observation and has not exhausted its launch budget. + pub start_pending: bool, /// Node process namespace for a remote PID; never a Web-host PID. #[serde(skip_serializing_if = "Option::is_none")] pub pid_node_id: Option, @@ -43,16 +45,27 @@ pub struct ServerSummary { /// # Panics /// Panics if the `RwLock` is poisoned. pub async fn http_list_servers(State(state): State) -> Json> { - if state.node_monitor_url.is_some() && crate::node::cluster_binding(&state).ok().flatten().is_some() { - let _ = tokio::time::timeout(std::time::Duration::from_secs(3), super::remote::refresh(&state)).await; - } + let intents = if state.node_monitor_url.is_some() + && crate::node::cluster_binding(&state).ok().flatten().is_some() + { + tokio::time::timeout( + std::time::Duration::from_secs(3), + super::remote::refresh_with_intents(&state), + ) + .await + .ok() + .and_then(Result::ok) + .unwrap_or_default() + } else { + Vec::new() + }; let snap = state.monitor_cache.snapshot().await; let cfg = state.config.read().unwrap().clone(); let remote = state.node_monitor_url.is_some(); let remote_pids = if remote { - super::remote::processes(&state, &cfg).await + super::remote::processes(&state, &cfg, &intents).await } else { - std::collections::BTreeMap::new() + super::remote::ProcessObservations::default() }; let rows = futures::future::join_all(cfg.servers.iter().map(|s| { let state = &state; @@ -76,7 +89,7 @@ pub async fn http_list_servers(State(state): State) -> Json) -> Json, + exhausted: std::collections::BTreeSet<(u64, String)>, +} + +impl ProcessObservations { + fn record(&mut self, node: u64, value: &Value, intents: &[NodeServiceIntent]) { + if let Some(processes) = value["processes"].as_object() { + for (id, value) in processes { + if let Some(pid) = value + .as_u64() + .and_then(|pid| u32::try_from(pid).ok()) + .filter(|pid| *pid > 0) + { + self.pids.insert((node, id.clone()), pid); + } + } + } + for intent in intents.iter().filter(|intent| intent.node_id == node) { + let failure = &value["service_failures"][&intent.service_id]; + if failure["operation_id"] == intent.operation_id && failure["exhausted"] == true { + self.exhausted.insert((node, intent.service_id.clone())); + } + } + } + + pub fn start_pending(&self, entry: &ServerEntry, remote: bool, pid: Option) -> bool { + remote + && entry.auto_start + && pid.is_none() + && !entry + .node_id + .is_some_and(|node| self.exhausted.contains(&(node, entry.id.clone()))) + } +} + pub(super) async fn processes( state: &AppState, config: &crowdb_console_shared::ConsoleConfig, -) -> std::collections::BTreeMap<(u64, String), u32> { + intents: &[NodeServiceIntent], +) -> ProcessObservations { let Some(binding) = crate::node::cluster_binding(state).ok().flatten() else { - return std::collections::BTreeMap::new(); + return ProcessObservations::default(); }; let cluster = &binding.bootstrap.cluster_id; let observations = futures::future::join_all(config.nodes.iter().map(|node| async move { @@ -44,21 +82,11 @@ pub(super) async fn processes( Some((node.id, result)) })) .await; - let mut pids = std::collections::BTreeMap::new(); + let mut processes = ProcessObservations::default(); for (node, observation) in observations.into_iter().flatten() { - if let Some(services) = observation["processes"].as_object() { - for (id, value) in services { - if let Some(pid) = value - .as_u64() - .and_then(|pid| u32::try_from(pid).ok()) - .filter(|pid| *pid > 0) - { - pids.insert((node, id.clone()), pid); - } - } - } + processes.record(node, &observation, intents); } - pids + processes } pub(crate) async fn act( @@ -97,12 +125,18 @@ pub(crate) async fn act( } pub(crate) async fn refresh(state: &AppState) -> crowdb_console_shared::error::Result<()> { + refresh_with_intents(state).await.map(|_| ()) +} + +pub(super) async fn refresh_with_intents( + state: &AppState, +) -> crowdb_console_shared::error::Result> { let intents = services::list(state.kv_client().await.as_ref()).await?; let mut config = state .config .write() .map_err(|error| crowdb_console_shared::error::Error::Config(error.to_string()))?; - for intent in intents { + for intent in &intents { // KV endpoints come from admitted node bootstrap, not auxiliary // service configuration. Its lifecycle intent must not erase them. if intent.kind == "kv" { @@ -112,11 +146,11 @@ pub(crate) async fn refresh(state: &AppState) -> crowdb_console_shared::error::R if matches!(intent.action, NodeServiceAction::Delete) { continue; } - if let Some(entry) = project(&intent) { + if let Some(entry) = project(intent) { config.servers.push(entry); } } - Ok(()) + Ok(intents) } fn project(intent: &NodeServiceIntent) -> Option { @@ -165,7 +199,13 @@ fn project(intent: &NodeServiceIntent) -> Option { ); entry.node_id = Some(intent.node_id); if intent.kind == "access" { - entry.dataset_origin = intent.environment.get("CROWDB_DATASET_PUBLIC_URI").cloned(); + for protocol in ["s3", "iceberg", "dataset"] { + if let Some(address) = lookup(&value, &[protocol, "listen"]) { + entry + .access_origins + .insert(protocol.into(), format!("http://{address}")); + } + } } entry.service_type = kind; entry.rpc_url = diskio_address @@ -223,3 +263,7 @@ pub(crate) async fn kv_action( .await .map_err(map_config_err) } + +#[cfg(test)] +#[path = "../../tests/internal/service_start_pending_test.rs"] +mod tests; diff --git a/app/crowdb-web/src/standalone/recovery.rs b/app/crowdb-web/src/standalone/recovery.rs index 1d789195f..c3fda27ab 100644 --- a/app/crowdb-web/src/standalone/recovery.rs +++ b/app/crowdb-web/src/standalone/recovery.rs @@ -7,11 +7,11 @@ use std::path::Path; use crowdb_console_shared::clients::http::ServerClient; use crowdb_console_shared::config::{ - GroupEntry, NodeEntry, ReplicaEntry, ServerEntry, ServiceType, StoreEntry, + DiskEntry, DiskGroupEntry, GroupEntry, NodeEntry, ReplicaEntry, ServerEntry, ServiceType, StoreEntry, }; use crowdb_console_shared::error::{Error, Result}; use crowdb_console_shared::lifecycle::{self, DeployRequest}; -use crowdb_console_shared::ops::hardware; +use crowdb_console_shared::ops::{hardware, OpContext}; use crate::state::AppState; @@ -192,8 +192,11 @@ impl AppState { pub(crate) async fn reload_group0(&self) -> Result<()> { let ctx = self.op_context().await?; - let racks = hardware::list_racks_from_group0(&ctx).await?; - let mut nodes = hardware::list_nodes_from_group0(&ctx, None).await?; + let (racks, mut nodes, (stores, groups)) = tokio::try_join!( + hardware::list_racks_from_group0(&ctx), + hardware::list_nodes_from_group0(&ctx, None), + logical_topology(&ctx), + )?; // Private SSH material remains local; Group 0 owns connection identity. let previous = ctx.config().nodes.clone(); for node in &mut nodes { @@ -205,39 +208,7 @@ impl AppState { node.ssh_password.clone_from(&local.ssh_password); } } - let sysmd = ctx.sysmd(); - let mut stores = Vec::new(); - let mut groups = Vec::new(); - for store in sysmd.list_stores().await? { - for group in sysmd.list_groups_in_store(store.store_id).await? { - let replicas = sysmd - .list_replicas_in_group(store.store_id, group.group_id) - .await? - .into_iter() - .map(|replica| ReplicaEntry { - replica_id: replica.replica_id, - node_id: replica.node_id, - }) - .collect(); - groups.push(GroupEntry { - store_id: store.store_id, - group_id: group.group_id, - replicas, - }); - } - stores.push(StoreEntry { - store_id: store.store_id, - nodes: store.node_ids, - }); - } - let mut disk_groups = Vec::new(); - let mut disks = Vec::new(); - for node in &nodes { - for group in hardware::list_disk_groups_from_group0(&ctx, node.id).await? { - disks.extend(hardware::list_disks_from_group0(&ctx, node.id, group.id).await?); - disk_groups.push(group); - } - } + let (disk_groups, disks) = disk_topology(&ctx, &nodes).await?; if self.node_monitor_url.is_some() { crate::services::remote::refresh(self).await?; } @@ -254,3 +225,65 @@ impl AppState { Ok(()) } } + +async fn logical_topology(ctx: &OpContext) -> Result<(Vec, Vec)> { + let sysmd = ctx.sysmd(); + let raw_stores = sysmd.list_stores().await?; + let groups = futures::future::try_join_all(raw_stores.iter().map(|store| { + let sysmd = &sysmd; + async move { + let groups = sysmd.list_groups_in_store(store.store_id).await?; + futures::future::try_join_all(groups.into_iter().map(|group| async move { + let replicas = sysmd + .list_replicas_in_group(store.store_id, group.group_id) + .await? + .into_iter() + .map(|replica| ReplicaEntry { + replica_id: replica.replica_id, + node_id: replica.node_id, + }) + .collect(); + Ok::<_, Error>(GroupEntry { + store_id: store.store_id, + group_id: group.group_id, + replicas, + }) + })) + .await + } + })) + .await? + .into_iter() + .flatten() + .collect(); + let stores = raw_stores + .into_iter() + .map(|store| StoreEntry { + store_id: store.store_id, + nodes: store.node_ids, + }) + .collect(); + Ok((stores, groups)) +} + +async fn disk_topology( + ctx: &OpContext, + nodes: &[NodeEntry], +) -> Result<(Vec, Vec)> { + let groups = futures::future::try_join_all(nodes.iter().map(|node| async move { + let groups = hardware::list_disk_groups_from_group0(ctx, node.id).await?; + futures::future::try_join_all(groups.into_iter().map(|group| async move { + let disks = hardware::list_disks_from_group0(ctx, node.id, group.id).await?; + Ok::<_, Error>((group, disks)) + })) + .await + })) + .await?; + let mut disk_groups = Vec::new(); + let mut disks = Vec::new(); + for (group, group_disks) in groups.into_iter().flatten() { + disk_groups.push(group); + disks.extend(group_disks); + } + Ok((disk_groups, disks)) +} diff --git a/app/crowdb-web/src/state.rs b/app/crowdb-web/src/state.rs index 608bd1124..1385083ea 100644 --- a/app/crowdb-web/src/state.rs +++ b/app/crowdb-web/src/state.rs @@ -21,6 +21,7 @@ use crowdb_console_shared::{config::ServerEntry, ConsoleConfig}; /// request (the service registry may not be ready at console startup). #[derive(Clone)] pub struct AppState { + pub(crate) node_observations: Arc, pub(crate) service_operations: Arc>>, pub config: Arc>, pub(crate) config_path: Option, @@ -98,6 +99,7 @@ impl AppState { #[must_use] pub fn with_runtime_root(config: ConsoleConfig, runtime_root: PathBuf) -> Self { Self { + node_observations: Arc::default(), service_operations: Arc::new(arc_swap::ArcSwap::from_pointee(std::collections::HashSet::new())), config: Arc::new(RwLock::new(config)), config_path: None, @@ -722,7 +724,7 @@ mod tests { fn resolve_group0_endpoint_no_store0_returns_none() { let mut config = ConsoleConfig::default(); let _ = config.add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "s1".into(), url: "http://127.0.0.1:10000".into(), node_id: Some(1), @@ -746,7 +748,7 @@ mod tests { fn resolve_group0_endpoint_prefers_store0_rpc_url() { let mut config = ConsoleConfig::default(); let _ = config.add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "s1".into(), url: "http://127.0.0.1:10000".into(), node_id: Some(1), @@ -781,7 +783,7 @@ mod tests { async fn op_context_shares_cached_kv_client() { let mut config = ConsoleConfig::default(); let _ = config.add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "s1".into(), url: "http://127.0.0.1:10000".into(), node_id: Some(1), @@ -837,7 +839,7 @@ mod tests { async fn commit_op_context_writes_back_config() { let mut config = ConsoleConfig::default(); let _ = config.add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "s1".into(), url: "http://127.0.0.1:10000".into(), node_id: Some(1), diff --git a/app/crowdb-web/tests/access_routes_test.rs b/app/crowdb-web/tests/access_routes_test.rs index 4f5c30ac9..d3f2e19f1 100644 --- a/app/crowdb-web/tests/access_routes_test.rs +++ b/app/crowdb-web/tests/access_routes_test.rs @@ -10,6 +10,71 @@ use crowdb_console_shared::ConsoleConfig; use crowdb_web::{router, AppState}; use tower::ServiceExt; +#[tokio::test] +async fn node_proxy_uses_internal_listeners_while_exporting_separate_public_client_origins() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("split-origins"); + let credentials = crowdb_monitor::ServerCredentials::load_or_create(root.path()).unwrap(); + credentials + .persist_client(&crowdb_monitor::ClientCredentials { + s3_endpoint: "http://public.example.test:19091".into(), + iceberg_endpoint: "http://public.example.test:19092".into(), + dataset_endpoint: Some("http://public.example.test:19093".into()), + region: "us-east-1".into(), + access_key_id: "AKIDEXAMPLE".into(), + secret_access_key: "server-secret".into(), + }) + .unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let internal = format!("http://{}", listener.local_addr().unwrap()); + let upstream = axum::Router::new() + .route("/", any(|| async { "internal-service" })) + .route("/*path", any(|| async { "internal-service" })); + let task = tokio::spawn(async move { + axum::serve(listener, upstream).await.unwrap(); + }); + let mut config = ConsoleConfig::default(); + let mut entry = ServerEntry::new("access-1", internal.clone()); + entry.node_id = Some(1); + entry.service_type = ServiceType::AccessServer; + for protocol in ["s3", "iceberg", "dataset"] { + entry.access_origins.insert(protocol.into(), internal.clone()); + } + config.servers.push(entry); + let state = AppState::with_runtime_root(config, root.path().to_owned()) + .with_node_monitor("http://127.0.0.1:1") + .unwrap(); + let app = router(state); + for path in ["/api/access/s3", "/api/access/iceberg/v1/namespaces"] { + let response = app + .clone() + .oneshot(Request::builder().uri(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{path}"); + assert_eq!( + axum::body::to_bytes(response.into_body(), 100).await.unwrap(), + "internal-service" + ); + } + let response = app + .oneshot( + Request::builder() + .uri("/api/access/credentials") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let bytes = axum::body::to_bytes(response.into_body(), 4096).await.unwrap(); + let clients: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(clients["s3"]["endpoint"], "http://public.example.test:19091"); + assert_eq!(clients["iceberg"]["uri"], "http://public.example.test:19092"); + assert_eq!(clients["dataset"]["uri"], "http://public.example.test:19093"); + assert_eq!(clients["dataset"]["token"], credentials.dataset_token()); + assert!(!String::from_utf8_lossy(&bytes).contains(credentials.s3_master_key())); + task.abort(); +} + #[tokio::test] async fn native_deployment_resolves_cluster_origins_and_private_reader() { let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("native-access-origin"); @@ -96,6 +161,7 @@ fn configured(origin: &str) -> axum::Router { .persist_client(&crowdb_monitor::ClientCredentials { s3_endpoint: origin.into(), iceberg_endpoint: origin.into(), + dataset_endpoint: None, region: "us-east-1".into(), access_key_id: "AKIDEXAMPLE".into(), secret_access_key: "server-secret".into(), @@ -116,6 +182,7 @@ async fn remote_deployment_resolves_persisted_origins_without_local_launches() { .persist_client(&crowdb_monitor::ClientCredentials { s3_endpoint: "http://192.0.2.10:9001".into(), iceberg_endpoint: "http://192.0.2.10:9000".into(), + dataset_endpoint: None, region: "us-east-1".into(), access_key_id: "test-client".into(), secret_access_key: "private-test-secret".into(), diff --git a/app/crowdb-web/tests/auto_formation_test.rs b/app/crowdb-web/tests/auto_formation_test.rs new file mode 100644 index 000000000..ecc7d568b --- /dev/null +++ b/app/crowdb-web/tests/auto_formation_test.rs @@ -0,0 +1,129 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/auto_formation.rs"] +mod auto_formation; + +use auto_formation::{policy, TestFormationFixture}; +use crowdb_protocol::mgmt::node::{deployment::DeploymentMode, CandidateState}; + +#[tokio::test] +async fn incomplete_three_node_scope_waits_without_admitting_partial_members() { + for count in [0, 1] { + let policy = policy(3, DeploymentMode::Local); + let peers = vec![(policy.formation(), CandidateState::Unbound); count]; + let fixture = TestFormationFixture::start(policy, peers, 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "waiting", "{report}"); + assert_eq!(report["required_voters"], 3); + assert_eq!(report["observed_voters"], count + 1); + fixture.assert_no_admission(); + } +} + +#[tokio::test] +async fn wrong_scope_count_or_mode_never_reaches_local_credentials() { + let expected = policy(3, DeploymentMode::Local); + for foreign in [ + policy(1, DeploymentMode::Local).formation(), + policy(3, DeploymentMode::Production).formation(), + crowdb_protocol::mgmt::node::deployment::FormationPolicy { + cluster: "foreign".into(), + ..expected.formation() + }, + ] { + let fixture = + TestFormationFixture::start(expected.clone(), vec![(foreign, CandidateState::Unbound)], 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "waiting", "{report}"); + assert_eq!(report["observed_voters"], 1); + fixture.assert_no_admission(); + } +} + +#[tokio::test] +async fn overfull_fresh_scope_fails_before_admission_or_default_rack_creation() { + let policy = policy(3, DeploymentMode::Local); + let peers = vec![(policy.formation(), CandidateState::Unbound); 3]; + let fixture = TestFormationFixture::start(policy, peers, 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "failed"); + assert!(report["error"].as_str().unwrap().contains("Too many fresh nodes")); + fixture.assert_no_admission(); +} + +#[tokio::test] +async fn only_uuid_ordered_coordinator_can_begin_fresh_admission() { + let policy = policy(3, DeploymentMode::Local); + let peers = vec![(policy.formation(), CandidateState::Unbound); 2]; + let fixture = TestFormationFixture::start(policy, peers, 99).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "waiting"); + assert_eq!(report["coordinator"], "00000000-0000-4000-8000-000000000002"); + fixture.assert_no_admission(); +} + +#[tokio::test] +async fn known_bound_scope_and_duplicate_identity_require_explicit_attention() { + for state in [CandidateState::ForeignCluster, CandidateState::IdentityConflict] { + let policy = policy(3, DeploymentMode::Local); + let peers = vec![(policy.formation(), state)]; + let fixture = TestFormationFixture::start(policy, peers, 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "failed", "{report}"); + fixture.assert_no_admission(); + } +} + +#[tokio::test] +async fn production_does_not_enable_local_convenience_admission() { + let fixture = TestFormationFixture::start(policy(3, DeploymentMode::Production), vec![], 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "disabled"); + fixture.assert_no_admission(); +} + +#[tokio::test] +async fn unrelated_bound_clusters_do_not_block_fresh_scoped_formation() { + for voters in [1, 3] { + let expected = policy(voters, DeploymentMode::Local); + let mut peers: Vec<_> = ["foreign-a", "foreign-b"] + .map(|cluster| { + let mut foreign = expected.formation(); + foreign.cluster = cluster.into(); + (foreign, CandidateState::ForeignCluster) + }) + .into(); + peers.extend((1..voters).map(|_| (expected.formation(), CandidateState::Unbound))); + let fixture = TestFormationFixture::start(expected, peers, 1).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "failed", "{report}"); + assert!( + report["error"].as_str().unwrap().contains("credential"), + "{report}" + ); + assert!(!fixture.root.join("accepted-node.json").exists()); + assert!(fixture.state.config.read().unwrap().nodes.is_empty()); + } +} + +#[tokio::test] +async fn missing_exposed_empty_or_corrupt_local_credentials_never_fall_back_to_a_password() { + for password in [ + None, + Some((b"never-emit-secret".as_slice(), 0o644)), + Some((b"".as_slice(), 0o600)), + Some(([0xff].as_slice(), 0o600)), + ] { + let fixture = + TestFormationFixture::with_password(policy(1, DeploymentMode::Local), vec![], 1, password).await; + let report = fixture.observation().await; + assert_eq!(report["phase"], "failed", "{report}"); + assert!(!report.to_string().contains("never-emit-secret")); + assert!(!fixture + .root + .join("console/admission-00000000-0000-4000-8000-000000000001.json") + .exists()); + assert!(!fixture.root.join("accepted-node.json").exists()); + } +} diff --git a/app/crowdb-web/tests/auto_join_test.rs b/app/crowdb-web/tests/auto_join_test.rs index 8f22567e4..128da7d21 100644 --- a/app/crowdb-web/tests/auto_join_test.rs +++ b/app/crowdb-web/tests/auto_join_test.rs @@ -15,7 +15,20 @@ use tower::ServiceExt; struct TestWorkspace(std::path::PathBuf); impl TestWorkspace { fn new() -> Self { + Self::with_voters(1) + } + fn with_voters(voters: u8) -> Self { let root = std::env::temp_dir().join(format!("crowdb-auto-join-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root).unwrap(); + crowdb_monitor::accept_node_policy( + &root, + &crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides::default(), + &crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides { + initial_voters: Some(voters), + ..Default::default() + }, + ) + .unwrap(); std::fs::create_dir_all(root.join("console")).unwrap(); Self(root) } @@ -24,6 +37,12 @@ impl TestWorkspace { .with_node_monitor("http://127.0.0.1:1") .unwrap() } + fn formation(&self) -> crowdb_protocol::mgmt::node::deployment::FormationPolicy { + crowdb_monitor::read_node_policy(&self.0) + .unwrap() + .unwrap() + .formation() + } } impl Drop for TestWorkspace { fn drop(&mut self) { @@ -52,7 +71,7 @@ async fn default_enabled_policy_survives_disable_and_restart_without_touching_ac let (status, policy) = request(router(workspace.state()), None).await; assert_eq!(status, StatusCode::OK); assert_eq!(policy["enabled"], true); - assert_eq!(policy["required_voters"], 3); + assert_eq!(policy["required_voters"], 1); let (status, _) = request(router(workspace.state()), Some(json!({"enabled": false}))).await; assert_eq!(status, StatusCode::OK); let state = workspace.state(); @@ -145,7 +164,8 @@ async fn two_authenticated_nodes_wait_in_uuid_order_without_initializing() { CandidateNode, CandidateSnapshot, CandidateState, NodeAdvertisement, NodeHandshake, NodeHardware, NODE_PROTOCOL_VERSION, }; - let workspace = TestWorkspace::new(); + let workspace = TestWorkspace::with_voters(3); + let formation = workspace.formation(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = format!("http://{}", listener.local_addr().unwrap()); let mut config = ConsoleConfig::default(); @@ -178,6 +198,7 @@ async fn two_authenticated_nodes_wait_in_uuid_order_without_initializing() { ssh_user: "crowdb".into(), operation_id: uuid::Uuid::new_v4().to_string(), confirmed: false, + host_key: None, cancelled: false, }; std::fs::write( @@ -191,6 +212,7 @@ async fn two_authenticated_nodes_wait_in_uuid_order_without_initializing() { protocol_version: NODE_PROTOCOL_VERSION, monitor_endpoints: vec![endpoint.clone()], cluster_id: None, + formation: Some(formation.clone()), }, state: CandidateState::Unbound, }); @@ -270,3 +292,42 @@ async fn choosing_a_new_draft_requires_matching_completed_cleanup_for_a_pinned_t assert!(body["target_cluster_id"].is_null()); assert_eq!(body["enabled"], true); } + +#[tokio::test] +async fn completed_cleanup_retires_console_credentials_before_another_cluster_is_imported() { + let workspace = TestWorkspace::new(); + let console = workspace.0.join("console"); + let old = crowdb_monitor::ServerCredentials::load_or_create(&console).unwrap(); + let old_body = std::fs::read(console.join("secrets/server.env")).unwrap(); + let operation = uuid::Uuid::new_v4().to_string(); + let cleaned = workspace.0.join("node-cleaned.json"); + std::fs::write( + &cleaned, + json!({ + "cluster_id":uuid::Uuid::new_v4().to_string(),"operation_id":operation, + "configuration_digest":"a".repeat(64) + }) + .to_string(), + ) + .unwrap(); + std::fs::set_permissions(&cleaned, std::fs::Permissions::from_mode(0o600)).unwrap(); + let (status, body) = request(router(workspace.state()), None).await; + assert_eq!(status, StatusCode::OK, "{body}"); + assert!( + !console.join("secrets").exists(), + "completed cleanup retained the previous cluster's Console credentials" + ); + let archived = console.join(format!("retired-secrets-{operation}/server.env")); + assert!(std::fs::read(archived).unwrap().eq(&old_body)); + let current = crowdb_monitor::ServerCredentials::load_or_create(&workspace.0).unwrap(); + assert!(!current.s3_master_key().eq(old.s3_master_key())); + crowdb_monitor::ServerCredentials::import(&console, ¤t.server_env()).unwrap(); + assert_eq!(request(router(workspace.state()), None).await.0, StatusCode::OK); + assert!( + crowdb_monitor::ServerCredentials::load_existing(&console) + .unwrap() + .server_env() + == current.server_env(), + "observing an acknowledged cleanup must preserve the new cluster's credentials" + ); +} diff --git a/app/crowdb-web/tests/chunk_kv_catalog_test.rs b/app/crowdb-web/tests/chunk_kv_catalog_test.rs index ad7d5b647..0c66f7dbf 100644 --- a/app/crowdb-web/tests/chunk_kv_catalog_test.rs +++ b/app/crowdb-web/tests/chunk_kv_catalog_test.rs @@ -115,16 +115,24 @@ async fn catalog_windows_pin_generation_validate_checksums_and_keep_exact_ids() } async fn runtime_get(app: axum::Router, query: &str) -> StatusCode { - app.oneshot( - Request::get(format!( + let response = app + .oneshot( + Request::get(format!( "/api/chunk-kv/runtime?page=0&offset=0&generation=9&id=ffffffffffffffff0000000000000001&{query}" )) - .body(Body::empty()) - .unwrap(), - ) - .await - .unwrap() - .status() + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + if status.is_server_error() { + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + eprintln!("runtime observation {status}: {}", String::from_utf8_lossy(&body)); + } + status } async fn verify_runtime(state: &AppState, app: &axum::Router) { @@ -146,12 +154,12 @@ async fn verify_runtime(state: &AppState, app: &axum::Router) { let mut value = serde_json::json!({ "partition_id":"ffffffffffffffff0000000000000001", "catalog_generation":"9", "owner_epoch":u64::MAX.to_string(), "instance_id":u64::MAX.to_string(), "tree_id":"1", "stream_id":"ffffffffffffffff0000000000000001", - "journal":{"generation":"17", "offset":query.get("stream_offset").unwrap().parse::().unwrap(), "extent_pages":[]}}); + "journal":{"generation":"17", "offset":query.get("stream_offset").unwrap().parse::().unwrap(), "chunk_record":{"chunk_id":"00000000000000000000000000000001","state":"Active","logical_start":"0","logical_end":"0","mapping_count":"0","mapping_bytes":"0"}}}); match mode.load(Ordering::Acquire) { 1 => value["instance_id"] = "wrong-owner".into(), 2 => value["stream_id"] = "wrong-stream".into(), 3 => value["oversized"] = "x".repeat(65_537).into(), - 4 => value["journal"]["offset"] = 0.into(), + 4 => value["journal"]["offset"] = 100.into(), _ => (), } Json(value) @@ -168,7 +176,7 @@ async fn verify_runtime(state: &AppState, app: &axum::Router) { entry.rpc_url = Some("127.0.0.1:41000".into()); state.config.write().unwrap().add_server(entry).unwrap(); assert_eq!(runtime_get(app.clone(), &query).await, StatusCode::OK); - let continuation = format!("{query}&stream_generation=17&stream_offset=100"); + let continuation = format!("{query}&stream_generation=17&stream_offset=0"); assert_eq!(runtime_get(app.clone(), &continuation).await, StatusCode::OK); assert_eq!( runtime_get(app.clone(), &format!("{query}&stream_offset=100")).await, @@ -177,7 +185,7 @@ async fn verify_runtime(state: &AppState, app: &axum::Router) { assert_eq!( runtime_get( app.clone(), - &format!("{query}&stream_generation=16&stream_offset=100") + &format!("{query}&stream_generation=16&stream_offset=0") ) .await, StatusCode::CONFLICT @@ -210,8 +218,21 @@ async fn verify_discovered_runtime(state: &AppState, app: &axum::Router, query: .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_millis(); - let base = serde_json::json!({ "instance_id":u64::MAX, "rpc_endpoint":"127.0.0.1:41000", "last_heartbeat_ms":u64::try_from(now).unwrap(), - "extra":{"chunk_kv":{"node_id":7,"http_endpoint":origin,"capacity_bytes":0,"durable_bytes":0,"request_rate":0,"hosted":[]}} }); + let base = serde_json::to_value(crowdb_protocol::common::InstanceValue { + instance_id: u64::MAX, + rpc_endpoint: "127.0.0.1:41000".into(), + last_heartbeat_ms: u64::try_from(now).unwrap(), + extra: Some(crowdb_protocol::common::ServiceExtra { + chunk_kv: Some(crowdb_protocol::common::ChunkKvExtra { + process_epoch: 1, + node_id: Some(7), + http_endpoint: Some(origin.into()), + ..Default::default() + }), + ..Default::default() + }), + }) + .unwrap(); for (field, changed, expected) in [ ("", Value::Null, StatusCode::OK), ("rpc_endpoint", "127.0.0.1:41001".into(), StatusCode::CONFLICT), diff --git a/app/crowdb-web/tests/chunk_kv_deployment_configuration_test.rs b/app/crowdb-web/tests/chunk_kv_deployment_configuration_test.rs index e0d52e728..6ae321be4 100644 --- a/app/crowdb-web/tests/chunk_kv_deployment_configuration_test.rs +++ b/app/crowdb-web/tests/chunk_kv_deployment_configuration_test.rs @@ -39,11 +39,6 @@ async fn invalid_stream_geometry_is_rejected_before_execution() { assert_eq!(status, StatusCode::BAD_REQUEST); assert!(message.contains("Stream chunk capacity")); } - for entries in [json!(0), json!(-1), json!(4_294_967_296u64), json!("1")] { - let mut request = body(); - request["stream_extent_page_entries"] = entries; - assert_eq!(submit(request).await.0, StatusCode::UNPROCESSABLE_ENTITY); - } } #[tokio::test] @@ -52,13 +47,12 @@ async fn stream_geometry_is_scoped_to_chunk_kv_and_requires_authority() { let mut request = body(); if let Some(bytes) = bytes { request["stream_chunk_capacity_bytes"] = json!(bytes); - request["stream_extent_page_entries"] = json!(1); } let (status, message) = submit(request).await; assert_eq!(status, StatusCode::CONFLICT); assert!(message.contains("Group 0 is not ready")); } - for field in ["stream_chunk_capacity_bytes", "stream_extent_page_entries"] { + for field in ["stream_chunk_capacity_bytes"] { let mut request = json!({"kind":"chunkdb","instance_id":1,"http_port":7300,"rpc_port":7301}); request[field] = if field == "stream_chunk_capacity_bytes" { json!(1024 * 1024) diff --git a/app/crowdb-web/tests/cluster_target_test.rs b/app/crowdb-web/tests/cluster_target_test.rs new file mode 100644 index 000000000..13c5e66b0 --- /dev/null +++ b/app/crowdb-web/tests/cluster_target_test.rs @@ -0,0 +1,235 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/cluster_target.rs"] +mod cluster_target; +#[path = "common/memory_kv.rs"] +mod memory_kv; + +use axum::{ + body::Body, + http::{Request, StatusCode}, +}; +use cluster_target::TestClusterTarget; +use crowdb_console_shared::deployment::CLUSTER_KEY; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use serde_json::Value; +use tower::ServiceExt; + +async fn get(state: crowdb_web::AppState) -> (StatusCode, Value) { + let response = crowdb_web::router(state) + .oneshot( + Request::builder() + .uri("/api/cluster/target") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + let body = axum::body::to_bytes(response.into_body(), 65536).await.unwrap(); + (status, serde_json::from_slice(&body).unwrap()) +} + +async fn committed(fixture: &TestClusterTarget) -> (Vec, u64) { + match fixture + .kv + .get(0, 0, CLUSTER_KEY, ReadMode::Linearizable, None) + .await + .unwrap() + { + GetOutcome::Found { value, revision, .. } => (value.to_vec(), revision), + GetOutcome::NotFound => panic!("missing cluster identity"), + } +} + +#[tokio::test] +async fn readonly_target_observes_committed_identity_without_publishing_or_repairing_state() { + let fixture = TestClusterTarget::start().await; + let before = committed(&fixture).await; + let (status, value) = get(fixture.state.clone()).await; + assert_eq!(status, StatusCode::OK, "{value}"); + assert_eq!(value["cluster_id"], fixture.prepared.identity.cluster_id); + assert!(value["group0_management_endpoints"][0] + .as_str() + .unwrap() + .starts_with("http://")); + assert!(value["group0_rpc_endpoints"][0].as_str().unwrap().contains(':')); + assert_eq!(committed(&fixture).await, before); + assert_eq!( + std::fs::read_dir(fixture.root.path().join("console")) + .unwrap() + .count(), + 0 + ); + assert!(value.get("runtime").is_none()); + let mut target = crowdb_console_shared::deployment::target::ResolvedClusterTarget { + identity: serde_json::from_value(value).unwrap(), + management_endpoints: vec![], + runtime: None, + }; + target.verify_native_authority(&fixture.kv).await.unwrap(); + target.identity.cluster_id = uuid::Uuid::new_v4().to_string(); + assert!(matches!( + target.verify_native_authority(&fixture.kv).await, + Err(crowdb_console_shared::deployment::target::TargetError::IdentityConflict) + )); + assert_eq!(committed(&fixture).await, before); +} + +#[tokio::test] +async fn conflicting_binding_fails_without_rebinding_or_overwriting_authority() { + let mut fixture = TestClusterTarget::start().await; + fixture.binding.bootstrap.cluster_id = uuid::Uuid::new_v4().to_string(); + let path = fixture.root.path().join("node-binding.json"); + let bytes = serde_json::to_vec(&fixture.binding).unwrap(); + std::fs::write(&path, &bytes).unwrap(); + let before = committed(&fixture).await; + let (status, error) = get(fixture.state.clone()).await; + assert_eq!(status, StatusCode::CONFLICT, "{error}"); + assert_eq!(std::fs::read(path).unwrap(), bytes); + assert_eq!(committed(&fixture).await, before); +} + +#[tokio::test] +async fn absent_or_corrupt_publication_cannot_report_an_accepted_local_uuid_as_confirmed() { + let fixture = TestClusterTarget::start().await; + for bytes in [b"{}".as_slice(), b"not-json".as_slice()] { + fixture.kv.put(0, 0, CLUSTER_KEY, bytes, None).await.unwrap(); + let before = committed(&fixture).await; + let (status, error) = get(fixture.state.clone()).await; + assert_eq!(status, StatusCode::CONFLICT, "{error}"); + assert_eq!(committed(&fixture).await, before); + } + fixture.kv.delete(0, 0, CLUSTER_KEY, None).await.unwrap(); + let (status, error) = get(fixture.state.clone()).await; + assert_eq!(status, StatusCode::CONFLICT, "{error}"); +} + +#[tokio::test] +async fn node_observation_does_not_recover_or_copy_a_pending_deployment() { + let fixture = TestClusterTarget::start().await; + std::fs::write( + fixture.root.path().join("prepared-bootstrap.json"), + serde_json::to_vec(&fixture.prepared).unwrap(), + ) + .unwrap(); + crowdb_monitor::ServerCredentials::load_or_create(fixture.root.path()).unwrap(); + let before = committed(&fixture).await; + let response = crowdb_web::router(fixture.state.clone()) + .oneshot( + Request::builder() + .uri("/api/cluster/node-observation") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + assert_eq!(committed(&fixture).await, before); + assert_eq!( + std::fs::read_dir(fixture.root.path().join("console")) + .unwrap() + .count(), + 0 + ); +} + +#[tokio::test] +async fn target_uses_confirmed_current_addresses_without_rewriting_the_manifest() { + use crowdb_console_shared::deployment::registry::{NodeRecord, NodeRegistry, NODES_KEY}; + let fixture = TestClusterTarget::start().await; + let before = committed(&fixture).await; + let registry = NodeRegistry { + cluster_id: fixture.prepared.identity.cluster_id.clone(), + nodes: vec![NodeRecord { + discovery_id: uuid::Uuid::new_v4().to_string(), + node_id: 2, + physical_host_id: "test-host".into(), + rack_id: 1, + host: "192.0.2.99".into(), + ssh_port: 22, + ssh_user: "operator".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + confirmed: true, + host_key: None, + cancelled: false, + }], + }; + fixture + .kv + .put(0, 0, NODES_KEY, &serde_json::to_vec(®istry).unwrap(), None) + .await + .unwrap(); + let (status, value) = get(fixture.state.clone()).await; + assert_eq!(status, StatusCode::OK, "{value}"); + assert!(value["group0_management_endpoints"][0] + .as_str() + .unwrap() + .starts_with("http://192.0.2.99:")); + assert!(value["group0_rpc_endpoints"][0] + .as_str() + .unwrap() + .starts_with("192.0.2.99:")); + assert_eq!(committed(&fixture).await, before); +} + +#[tokio::test] +async fn cluster_status_is_observational_when_the_monitor_control_socket_is_unavailable() { + use crowdb_protocol::mgmt::{ + cluster::{ClusterPhase, ClusterStatus, DependencyKind, DependencyState}, + node::deployment::NodeDeploymentOverrides, + }; + let fixture = TestClusterTarget::start().await; + let node_root = fixture.root.path().join("observed-node"); + std::fs::create_dir(&node_root).unwrap(); + crowdb_monitor::accept_node_policy( + &node_root, + &NodeDeploymentOverrides::default(), + &NodeDeploymentOverrides::default(), + ) + .unwrap(); + crowdb_monitor::NodeIdentity::load_or_create(&node_root).unwrap(); + std::fs::write( + node_root.join("prepared-bootstrap.json"), + serde_json::to_vec(&fixture.prepared).unwrap(), + ) + .unwrap(); + std::fs::write( + node_root.join("node-binding.json"), + serde_json::to_vec(&fixture.binding).unwrap(), + ) + .unwrap(); + std::fs::create_dir(node_root.join("console")).unwrap(); + let mut state = fixture.state.clone(); + state.runtime_root = node_root.join("console").into(); + let before = committed(&fixture).await; + let response = crowdb_web::router(state) + .oneshot( + Request::builder() + .uri("/api/cluster/status") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + let report: ClusterStatus = serde_json::from_slice(&body).unwrap(); + assert_ne!(report.phase, ClusterPhase::Ready); + assert!(report + .reasons + .iter() + .any(|reason| reason.code == "node-observation-unavailable" + && reason.detail.contains("current execution and protocol health"))); + assert!(report + .observation + .dependencies + .iter() + .any(|fact| fact.kind == DependencyKind::Authority && fact.state == DependencyState::Ready)); + assert_eq!(committed(&fixture).await, before); + assert_eq!(std::fs::read_dir(node_root.join("console")).unwrap().count(), 0); + assert!(!node_root.join("node-setup.json").exists()); +} diff --git a/app/crowdb-web/tests/common/auto_formation.rs b/app/crowdb-web/tests/common/auto_formation.rs new file mode 100644 index 000000000..31320a211 --- /dev/null +++ b/app/crowdb-web/tests/common/auto_formation.rs @@ -0,0 +1,162 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{routing::get, Json, Router}; +use crowdb_protocol::mgmt::node::{ + deployment::{DeploymentMode, FormationPolicy, NodeDeploymentOverrides, NodeDeploymentPolicy}, + CandidateNode, CandidateSnapshot, CandidateState, NodeAdvertisement, NodeHandshake, NodeHardware, + NODE_PROTOCOL_VERSION, +}; +use crowdb_web::{start_node_auto_join, AppState}; + +pub struct TestFormationFixture { + pub root: std::path::PathBuf, + pub state: AppState, + monitor: tokio::task::JoinHandle<()>, + runner: tokio::task::JoinHandle<()>, +} + +pub fn policy(voters: u8, mode: DeploymentMode) -> NodeDeploymentPolicy { + NodeDeploymentOverrides { + initial_voters: Some(voters), + mode: Some(mode), + ..Default::default() + } + .resolve(&NodeDeploymentOverrides::default(), None) + .unwrap() +} + +impl TestFormationFixture { + pub async fn start( + policy: NodeDeploymentPolicy, + peers: Vec<(FormationPolicy, CandidateState)>, + local_id: u64, + ) -> Self { + Self::with_password(policy, peers, local_id, None).await + } + + pub async fn with_password( + policy: NodeDeploymentPolicy, + peers: Vec<(FormationPolicy, CandidateState)>, + local_id: u64, + password: Option<(&[u8], u32)>, + ) -> Self { + use std::os::unix::fs::PermissionsExt; + let root = std::env::temp_dir().join(format!("crowdb-formation-{}", uuid::Uuid::new_v4())); + std::fs::create_dir(&root).unwrap(); + let inputs = NodeDeploymentOverrides { + mode: Some(policy.mode), + cluster: Some(policy.cluster.clone()), + initial_voters: Some(policy.initial_voters), + storage: Some(policy.setup.storage.clone()), + services: Some(policy.setup.services.clone()), + ..Default::default() + }; + crowdb_monitor::accept_node_policy(&root, &NodeDeploymentOverrides::default(), &inputs).unwrap(); + if let Some((bytes, mode)) = password { + std::fs::create_dir(root.join("ssh")).unwrap(); + let path = root.join("ssh/local-admission-password"); + std::fs::write(&path, bytes).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + std::fs::create_dir(root.join("console")).unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let local = NodeHandshake { + advertisement: NodeAdvertisement { + discovery_id: format!("00000000-0000-4000-8000-{local_id:012x}"), + protocol_version: NODE_PROTOCOL_VERSION, + monitor_endpoints: vec![endpoint.clone()], + cluster_id: None, + formation: Some(policy.formation()), + }, + physical_host_id: "one-host".into(), + rack_hint: None, + hardware: NodeHardware::default(), + }; + let peers: Vec<_> = peers + .into_iter() + .enumerate() + .map(|(index, (formation, state))| CandidateNode { + advertisement: NodeAdvertisement { + discovery_id: format!("00000000-0000-4000-8000-{:012x}", index + 2), + formation: Some(formation), + cluster_id: if state == CandidateState::ForeignCluster { + Some(uuid::Uuid::new_v4().to_string()) + } else { + None + }, + ..local.advertisement.clone() + }, + state, + }) + .collect(); + let server = Router::new() + .route("/node", get(move || async move { Json(local) })) + .route( + "/candidates", + get(move || async move { + Json(CandidateSnapshot { + nodes: peers, + discovery_diagnostics: vec![], + }) + }), + ); + let monitor = tokio::spawn(async move { + axum::serve(listener, server).await.unwrap(); + }); + let state = AppState::with_runtime_root( + crowdb_console_shared::ConsoleConfig::default(), + root.join("console"), + ) + .with_node_monitor(&endpoint) + .unwrap(); + let runner = start_node_auto_join(state.clone()); + Self { + root, + state, + monitor, + runner, + } + } + + pub async fn observation(&self) -> serde_json::Value { + let path = self.state.runtime_root.join("auto-join-status.json"); + tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if let Ok(bytes) = std::fs::read(&path) { + break serde_json::from_slice(&bytes).unwrap(); + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap() + } + + pub fn assert_no_admission(&self) { + assert!(!self.state.runtime_root.join("prepared-bootstrap.json").exists()); + assert!(!std::fs::read_dir(self.state.runtime_root.as_ref()) + .unwrap() + .any(|entry| { + entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("admission-") + })); + assert!(self.state.config.read().unwrap().nodes.is_empty()); + assert!(self.state.config.read().unwrap().racks.is_empty()); + assert!(!self.root.join("disks").exists()); + } +} + +impl Drop for TestFormationFixture { + fn drop(&mut self) { + self.runner.abort(); + self.monitor.abort(); + if !std::thread::panicking() { + std::fs::remove_dir_all(&self.root).unwrap(); + } + } +} diff --git a/app/crowdb-web/tests/common/cluster_target.rs b/app/crowdb-web/tests/common/cluster_target.rs new file mode 100644 index 000000000..5ffdee81e --- /dev/null +++ b/app/crowdb-web/tests/common/cluster_target.rs @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::memory_kv::TestKvCluster; +use crowdb_console_shared::{ + config::{NodeEntry, RackEntry, ServerEntry}, + deployment::{PreparedBootstrap, CLUSTER_KEY}, + ConsoleConfig, +}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::mgmt::node::NodeBinding; +use crowdb_web::AppState; +use std::sync::Arc; + +pub struct TestClusterTarget { + pub state: AppState, + pub prepared: PreparedBootstrap, + pub binding: NodeBinding, + pub kv: Arc, + pub root: crowdb_test_harness::test_dirs::TestDir, + _cluster: TestKvCluster, +} + +impl TestClusterTarget { + pub async fn start() -> Self { + let cluster = TestKvCluster::start().await; + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + cluster.mgmt_endpoints.clone(), + ))); + kv.seed_leader(0, 0, cluster.group0_leader_endpoint.clone()); + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("cluster-target-api"); + let mut server = ServerEntry::new("kv-2", &cluster.mgmt_endpoints[0]); + server.node_id = Some(2); + server.rpc_url = Some(cluster.group0_leader_endpoint.clone()); + let config = ConsoleConfig { + servers: vec![server], + racks: vec![RackEntry { + id: 1, + name: "local".into(), + }], + nodes: vec![NodeEntry { + id: 2, + rack_id: 1, + host: "127.0.0.1".into(), + ssh_port: 22, + ssh_user: String::new(), + ssh_credential_ref: None, + ssh_key: None, + ssh_password: None, + }], + ..Default::default() + }; + let prepared = PreparedBootstrap::open(&root.path().join("prepared.json"), &config, &[2]).unwrap(); + kv.put(0, 0, CLUSTER_KEY, &serde_json::to_vec(&prepared).unwrap(), None) + .await + .unwrap(); + let binding = NodeBinding { + node_id: 2, + bootstrap: prepared.identity.clone(), + management_seeds: cluster.mgmt_endpoints.clone(), + }; + std::fs::write( + root.path().join("node-binding.json"), + serde_json::to_vec(&binding).unwrap(), + ) + .unwrap(); + std::fs::create_dir(root.path().join("console")).unwrap(); + let mut state = AppState::with_runtime_root(config, root.path().join("console")) + .with_node_monitor("http://127.0.0.1:1") + .unwrap(); + state.readonly = true; + *state.kv_client.write().await = Some(Arc::clone(&kv)); + Self { + state, + prepared, + binding, + kv, + root, + _cluster: cluster, + } + } +} diff --git a/app/crowdb-web/tests/common/node_health.rs b/app/crowdb-web/tests/common/node_health.rs new file mode 100644 index 000000000..6f6838797 --- /dev/null +++ b/app/crowdb-web/tests/common/node_health.rs @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{ + body::{to_bytes, Body}, + http::{Request, StatusCode}, +}; +use crowdb_protocol::mgmt::node::{ + deployment::{DeploymentMode, NodeDeploymentPolicy, PublicAccessUris}, + observation::NodeRuntimeObservation, + setup::NodeSetupChoices, +}; +use crowdb_web::{AppState, NodeObserver}; +use std::{ + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tower::ServiceExt; + +pub struct TestNodeHealth { + pub state: AppState, + pub calls: Arc, + pub fail: Arc, + pub observer: Option, + app: axum::Router, +} +impl TestNodeHealth { + pub fn new() -> Self { + let state = AppState::default() + .with_node_monitor("http://127.0.0.1:1") + .unwrap(); + let app = crowdb_web::router(state.clone()); + Self { + state, + calls: Arc::new(AtomicUsize::new(0)), + fail: Arc::new(AtomicBool::new(false)), + observer: None, + app, + } + } + pub fn start(&mut self) { + let calls = self.calls.clone(); + let fail = self.fail.clone(); + self.observer = Some( + crowdb_web::start_node_observer_for_tests(&self.state, move || { + let calls = calls.clone(); + let fail = fail.clone(); + async move { + calls.fetch_add(1, Ordering::Relaxed); + if fail.load(Ordering::Relaxed) { + return Err((StatusCode::BAD_GATEWAY, "backend dependency unavailable".into())); + } + Ok(NodeRuntimeObservation { + version: 1, + discovery_id: "00000000-0000-4000-8000-000000000001".into(), + policy: NodeDeploymentPolicy { + version: 1, + mode: DeploymentMode::Local, + cluster: "cached-health".into(), + initial_voters: 1, + setup: NodeSetupChoices::default(), + public_access: PublicAccessUris::default(), + }, + binding: None, + setup: None, + observed_at_unix_ms: u64::try_from( + SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis(), + ) + .unwrap(), + services: Vec::new(), + protocols: Vec::new(), + }) + } + }) + .unwrap(), + ); + } + pub async fn after(&self, calls: usize) { + tokio::time::timeout(Duration::from_secs(3), async { + while self.calls.load(Ordering::Relaxed) <= calls { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("node sampler did not advance without reader traffic"); + } + pub async fn read(&self, status: u16) -> serde_json::Value { + let response = self + .app + .clone() + .oneshot( + Request::builder() + .uri("/api/cluster/node-observation") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status().as_u16(), status); + serde_json::from_slice(&to_bytes(response.into_body(), 256 * 1024).await.unwrap()).unwrap() + } +} diff --git a/app/crowdb-web/tests/common/node_listener_recovery.rs b/app/crowdb-web/tests/common/node_listener_recovery.rs new file mode 100644 index 000000000..5bb248998 --- /dev/null +++ b/app/crowdb-web/tests/common/node_listener_recovery.rs @@ -0,0 +1,122 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::cluster_target::TestClusterTarget; +use axum::{routing::get, Json, Router}; +use crowdb_console_shared::deployment::registry::{NodeRegistry, NODES_KEY}; +use crowdb_protocol::mgmt::{ + cluster::DependencyState, + node::{ + deployment::{DeploymentMode, NodeDeploymentPolicy, PublicAccessUris}, + observation::{NodeRuntimeObservation, NodeServiceObservation}, + setup::NodeSetupChoices, + NodeServiceIntent, + }, +}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +pub fn observation( + fixture: &TestClusterTarget, + discovery: &str, + intent: &NodeServiceIntent, +) -> NodeRuntimeObservation { + NodeRuntimeObservation { + version: 1, + discovery_id: discovery.into(), + binding: Some(fixture.binding.clone()), + setup: None, + policy: NodeDeploymentPolicy { + version: 1, + mode: DeploymentMode::Local, + cluster: "listener-recovery".into(), + initial_voters: 1, + setup: NodeSetupChoices::default(), + public_access: PublicAccessUris::default(), + }, + observed_at_unix_ms: 1, + services: vec![NodeServiceObservation { + service_id: intent.service_id.clone(), + kind: "access".into(), + operation_id: Some(intent.operation_id.clone()), + state: DependencyState::Ready, + detail: String::new(), + observed_at_unix_ms: 1, + max_age_ms: 3000, + reason_code: None, + }], + protocols: Vec::new(), + } +} + +pub async fn publish(fixture: &TestClusterTarget, registry: &NodeRegistry) { + fixture + .kv + .put(0, 0, NODES_KEY, &serde_json::to_vec(registry).unwrap(), None) + .await + .unwrap(); +} + +pub async fn listener() -> ( + std::net::SocketAddr, + Arc, + tokio::task::JoinHandle<()>, +) { + let calls = Arc::new(AtomicUsize::new(0)); + let listener = tokio::net::TcpListener::bind("127.0.0.2:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let counted = calls.clone(); + let app = Router::new() + .route( + "/", + get(|| async { "" }), + ) + .route( + "/v1/config", + get(|| async { Json(serde_json::json!({"defaults":{},"overrides":{}})) }), + ) + .route( + "/api/dataset/v1/namespaces/dataset-ns/datasets/crowdb_readiness_probe", + get(|| async { + ( + axum::http::StatusCode::NOT_FOUND, + Json(serde_json::json!({"code":"not_found"})), + ) + }), + ) + .route( + "/_crowdb/health/ready", + get(|| async { + Json(serde_json::json!({"ready":true, + "listener":"ready","metadata":"ready","chunks":"ready","native_pool":"ready", + "cleanup":"ready","authentication":"ready"})) + }), + ) + .layer(axum::middleware::from_fn( + move |request: axum::extract::Request, next: axum::middleware::Next| { + let calls = counted.clone(); + async move { + calls.fetch_add(1, Ordering::Relaxed); + next.run(request).await + } + }, + )); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + (address, calls, task) +} + +pub fn credentials(root: &std::path::Path) { + let credentials = crowdb_monitor::ServerCredentials::load_or_create(root).unwrap(); + credentials + .persist_client(&crowdb_monitor::ClientCredentials { + s3_endpoint: "http://public.example.test:9091".into(), + iceberg_endpoint: "http://public.example.test:9092".into(), + dataset_endpoint: None, + region: "us-east-1".into(), + access_key_id: "test-access".into(), + secret_access_key: "test-secret".into(), + }) + .unwrap(); +} diff --git a/app/crowdb-web/tests/common/protocol_observation.rs b/app/crowdb-web/tests/common/protocol_observation.rs new file mode 100644 index 000000000..761c7efd3 --- /dev/null +++ b/app/crowdb-web/tests/common/protocol_observation.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{body::Body, http::Request, response::Response, routing::any, Router}; +use std::time::Duration; + +pub struct TestListener { + pub url: String, + task: tokio::task::JoinHandle<()>, +} + +impl TestListener { + pub async fn serve(status: u16, body: String, delay: Duration, authenticated: bool) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/probe", listener.local_addr().unwrap()); + let app = Router::new().route( + "/probe", + any(move |request: Request| { + let body = body.clone(); + async move { + assert_eq!(request.method(), "GET", "Readiness must not mutate data"); + assert_eq!( + request + .headers() + .get("authorization") + .map(|v| v.to_str().unwrap()), + authenticated.then_some("Bearer fixture-token") + ); + tokio::time::sleep(delay).await; + Response::builder() + .status(status) + .header("location", "/must-not-follow") + .body(Body::from(body)) + .unwrap() + } + }), + ); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + Self { url, task } + } +} + +impl Drop for TestListener { + fn drop(&mut self) { + self.task.abort(); + } +} diff --git a/app/crowdb-web/tests/common/storage_readiness.rs b/app/crowdb-web/tests/common/storage_readiness.rs new file mode 100644 index 000000000..2c3d490ca --- /dev/null +++ b/app/crowdb-web/tests/common/storage_readiness.rs @@ -0,0 +1,45 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv_client::HardwareClient; +use crowdb_protocol::{ + common::{DiskId, HwStatus, NodeValue, RackValue}, + diskdb::rpc::DiskValue, +}; + +pub async fn publish_disk(hardware: &HardwareClient) { + hardware + .add_rack( + 1, + &RackValue { + node_ids: vec![1], + ..Default::default() + }, + ) + .await + .unwrap(); + hardware + .add_node( + 1, + 1, + &NodeValue { + disk_group_ids: vec![11, 12], + ..Default::default() + }, + ) + .await + .unwrap(); + hardware + .add_disk( + 1, + 1, + 11, + &DiskId { high: 0, low: 11 }, + &DiskValue { + status: HwStatus::Up as i32, + ..Default::default() + }, + ) + .await + .unwrap(); +} diff --git a/app/crowdb-web/tests/dataset_proxy_test.rs b/app/crowdb-web/tests/dataset_proxy_test.rs index 7bdc3b9f8..a5f721e73 100644 --- a/app/crowdb-web/tests/dataset_proxy_test.rs +++ b/app/crowdb-web/tests/dataset_proxy_test.rs @@ -40,7 +40,7 @@ async fn fixture(readonly: bool) -> (axum::Router, tokio::task::JoinHandle<()>) let mut config = ConsoleConfig::default(); let mut entry = ServerEntry::new("access-1", "http://127.0.0.1:1"); entry.service_type = ServiceType::AccessServer; - entry.dataset_origin = Some(origin); + entry.access_origins.insert("dataset".into(), origin); config.servers.push(entry); let mut state = AppState::with_runtime_root(config, root); state.readonly = readonly; diff --git a/app/crowdb-web/tests/diskdb_auto_start_test.rs b/app/crowdb-web/tests/diskdb_auto_start_test.rs index 0233b1d23..8e18353d2 100644 --- a/app/crowdb-web/tests/diskdb_auto_start_test.rs +++ b/app/crowdb-web/tests/diskdb_auto_start_test.rs @@ -28,7 +28,7 @@ async fn startup_does_not_replay_local_diskdb_launch_policy_without_group0() { .unwrap(); config .add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "diskdb-7777".into(), url: "http://127.0.0.1:1".into(), node_id: Some(7777), diff --git a/app/crowdb-web/tests/internal/service_start_pending_test.rs b/app/crowdb-web/tests/internal/service_start_pending_test.rs new file mode 100644 index 000000000..390aeedea --- /dev/null +++ b/app/crowdb-web/tests/internal/service_start_pending_test.rs @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{NodeServiceAction, NodeServiceIntent, ProcessObservations, ServerEntry}; +use serde_json::json; + +#[test] +fn accepted_start_waits_until_owned_pid_or_current_operation_exhaustion() { + let intent = NodeServiceIntent { + node_id: 2, + service_id: "chunkdb-7".into(), + kind: "chunkdb".into(), + cluster_id: "cluster".into(), + operation_id: "current".into(), + action: NodeServiceAction::Start, + configuration: String::new(), + environment: std::collections::BTreeMap::new(), + }; + let mut entry = ServerEntry::new("chunkdb-7", "http://127.0.0.1:7300"); + entry.node_id = Some(2); + entry.auto_start = true; + for (operation, exhausted, pending) in [ + ("old", true, true), + ("current", false, true), + ("current", true, false), + ] { + let mut observation = ProcessObservations::default(); + observation.record( + 2, + &json!({"processes":{}, "service_failures": { + "chunkdb-7": {"operation_id": operation, "exhausted": exhausted} + }}), + std::slice::from_ref(&intent), + ); + assert_eq!(observation.start_pending(&entry, true, None), pending); + assert!(!observation.start_pending(&entry, true, Some(42))); + assert!(!observation.start_pending(&entry, false, None)); + } + entry.auto_start = false; + assert!(!ProcessObservations::default().start_pending(&entry, true, None)); +} diff --git a/app/crowdb-web/tests/kv_routes_test.rs b/app/crowdb-web/tests/kv_routes_test.rs index 1842eabaf..02b8894f7 100644 --- a/app/crowdb-web/tests/kv_routes_test.rs +++ b/app/crowdb-web/tests/kv_routes_test.rs @@ -44,7 +44,7 @@ async fn kv_get_returns_502_when_leader_unreachable() { }); // The node has a configured rpc_url, but the port is dead. cfg.add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "n1".to_string(), url: format!("http://127.0.0.1:{dead_port}"), node_id: Some(1), diff --git a/app/crowdb-web/tests/managed_mode_test.rs b/app/crowdb-web/tests/managed_mode_test.rs index cd2d056db..150446462 100644 --- a/app/crowdb-web/tests/managed_mode_test.rs +++ b/app/crowdb-web/tests/managed_mode_test.rs @@ -235,6 +235,7 @@ async fn managed_snapshot_uses_group0_and_monitor_without_local_fallback() { status.services.insert( "diskio".into(), ServiceStatus { + operation_id: None, pid: Some(123), generation: 2, healthy: true, @@ -313,6 +314,7 @@ async fn verify_chunk_kv_placement(app: &axum::Router, sysmd: &CrowdbSysmdClient "127.0.0.1:15201", &ServiceExtra { chunk_kv: Some(crowdb_protocol::common::ChunkKvExtra { + process_epoch: 1, node_id: Some(1), http_endpoint: Some("http://127.0.0.1:15101".into()), ..Default::default() diff --git a/app/crowdb-web/tests/node_candidates_test.rs b/app/crowdb-web/tests/node_candidates_test.rs index d19b15fc9..1707357b8 100644 --- a/app/crowdb-web/tests/node_candidates_test.rs +++ b/app/crowdb-web/tests/node_candidates_test.rs @@ -23,6 +23,7 @@ async fn local_candidate_is_shown_before_any_kv_server_exists() { protocol_version: NODE_PROTOCOL_VERSION, monitor_endpoints: vec![endpoint.clone()], cluster_id: None, + formation: None, }, physical_host_id: "one-host".into(), rack_hint: None, diff --git a/app/crowdb-web/tests/node_listener_recovery_test.rs b/app/crowdb-web/tests/node_listener_recovery_test.rs new file mode 100644 index 000000000..bc0657501 --- /dev/null +++ b/app/crowdb-web/tests/node_listener_recovery_test.rs @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/cluster_target.rs"] +mod cluster_target; +#[path = "common/memory_kv.rs"] +mod memory_kv; +#[path = "common/node_listener_recovery.rs"] +mod node_listener_recovery; +use cluster_target::TestClusterTarget; +use crowdb_console_shared::deployment::registry::{NodeRecord, NodeRegistry}; +use crowdb_protocol::mgmt::{ + cluster::DependencyState, + node::{NodeServiceAction, NodeServiceIntent}, +}; +use node_listener_recovery::{credentials, listener, observation, publish}; +use std::sync::atomic::Ordering; + +#[tokio::test] +async fn background_protocol_reads_follow_confirmed_addresses_without_ui_refresh_or_repair() { + let fixture = TestClusterTarget::start().await; + let sealed = std::fs::read(fixture.root.path().join("prepared.json")).unwrap(); + assert_eq!(fixture.prepared.identity, fixture.binding.bootstrap); + credentials(&fixture.root.path().join("console")); + let (address, calls, task) = listener().await; + let discovery = uuid::Uuid::new_v4().to_string(); + let registry = NodeRegistry { + cluster_id: fixture.binding.bootstrap.cluster_id.clone(), + nodes: vec![NodeRecord { + discovery_id: discovery.clone(), + node_id: 2, + physical_host_id: "retained-host".into(), + rack_id: 1, + host: "127.0.0.2".into(), + ssh_port: 2222, + ssh_user: "crowdb".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + confirmed: true, + host_key: None, + cancelled: false, + }], + }; + let intent = NodeServiceIntent { + cluster_id: registry.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: 2, + service_id: "access-2".into(), + kind: "access".into(), + action: NodeServiceAction::Start, + configuration: ["s3", "iceberg", "dataset", "health"] + .map(|kind| format!("[{kind}]\nlisten = '{address}'\n")) + .join(""), + environment: std::collections::BTreeMap::new(), + }; + fixture + .kv + .put( + 0, + 0, + b"/deployment/services/2/access-2", + &serde_json::to_vec(&intent).unwrap(), + None, + ) + .await + .unwrap(); + publish(&fixture, ®istry).await; + let baseline = observation(&fixture, &discovery, &intent); + let mut sampled = baseline.clone(); + crowdb_web::sample_node_services_for_tests(&fixture.state, &mut sampled).await; + assert_eq!(sampled.protocols.len(), 3); + assert!( + sampled + .protocols + .iter() + .all(|value| value.state == DependencyState::Ready), + "{:?}", + sampled.protocols + ); + assert!(sampled.services[0].detail.contains("Access readiness HTTP 200")); + assert_eq!(calls.load(Ordering::Relaxed), 4); + assert_eq!( + fixture.state.config.read().unwrap().node(2).unwrap().host, + "127.0.0.1" + ); + assert_eq!( + std::fs::read(fixture.root.path().join("prepared.json")).unwrap(), + sealed + ); + for case in 0..5 { + let mut changed = registry.clone(); + match case { + 0 => changed.nodes[0].confirmed = false, + 1 => changed.nodes[0].cancelled = true, + 2 => changed.nodes[0].discovery_id = uuid::Uuid::new_v4().to_string(), + 3 => changed.nodes[0].node_id = 3, + _ => changed.cluster_id = uuid::Uuid::new_v4().to_string(), + } + publish(&fixture, &changed).await; + let mut sampled = baseline.clone(); + crowdb_web::sample_node_services_for_tests(&fixture.state, &mut sampled).await; + assert!(sampled + .protocols + .iter() + .all(|value| value.state == DependencyState::Unknown)); + assert_eq!( + calls.load(Ordering::Relaxed), + 4, + "unconfirmed/foreign identity cannot authorize probes" + ); + } + task.abort(); +} diff --git a/app/crowdb-web/tests/node_observation_cache_test.rs b/app/crowdb-web/tests/node_observation_cache_test.rs new file mode 100644 index 000000000..a1a63d8b9 --- /dev/null +++ b/app/crowdb-web/tests/node_observation_cache_test.rs @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/node_health.rs"] +mod fixture; +use fixture::TestNodeHealth; +use std::{sync::atomic::Ordering, time::Duration}; + +#[tokio::test] +async fn node_status_queries_reuse_backend_evidence_and_do_not_refresh_its_age() { + let mut fixture = TestNodeHealth::new(); + assert!(fixture.read(502).await["error"] + .as_str() + .unwrap() + .contains("pending")); + assert_eq!(fixture.calls.load(Ordering::Relaxed), 0); + fixture.start(); + fixture.after(0).await; + assert!(crowdb_web::start_node_observer(&fixture.state).is_err()); + let initial = fixture.read(200).await; + fixture.observer.take(); + tokio::task::yield_now().await; + let calls = fixture.calls.load(Ordering::Relaxed); + for value in futures::future::join_all((0..32).map(|_| fixture.read(200))).await { + assert_eq!(value, initial); + } + assert_eq!(fixture.calls.load(Ordering::Relaxed), calls); + tokio::time::sleep(Duration::from_millis(3050)).await; + assert!(fixture.read(502).await["error"] + .as_str() + .unwrap() + .contains("expired")); + assert_eq!( + crowdb_web::node_observation_time_for_tests(&fixture.state), + initial["observed_at_unix_ms"].as_u64() + ); + assert_eq!(fixture.calls.load(Ordering::Relaxed), calls); +} + +#[tokio::test] +async fn backend_observation_failure_and_recovery_replace_cache_without_queries() { + let mut fixture = TestNodeHealth::new(); + fixture.fail.store(true, Ordering::Relaxed); + fixture.start(); + fixture.after(0).await; + assert!(fixture.read(502).await["error"] + .as_str() + .unwrap() + .contains("backend dependency unavailable")); + let calls = fixture.calls.load(Ordering::Relaxed); + fixture.fail.store(false, Ordering::Relaxed); + fixture.after(calls).await; + assert!(crowdb_web::node_observation_time_for_tests(&fixture.state).is_some()); + assert!(fixture.read(200).await["observed_at_unix_ms"].as_u64().is_some()); +} diff --git a/app/crowdb-web/tests/node_preflight_test.rs b/app/crowdb-web/tests/node_preflight_test.rs index 12c5c6f30..db4b955e6 100644 --- a/app/crowdb-web/tests/node_preflight_test.rs +++ b/app/crowdb-web/tests/node_preflight_test.rs @@ -28,6 +28,7 @@ async fn a_preflight_review_is_not_admission_and_discard_is_operation_fenced_and ssh_port: 2222, ssh_user: "crowdb".into(), confirmed: false, + host_key: None, cancelled: false, }; let path = root.join(format!("node-preflight-{discovery}.json")); diff --git a/app/crowdb-web/tests/protocol_observation_test.rs b/app/crowdb-web/tests/protocol_observation_test.rs new file mode 100644 index 000000000..819721e27 --- /dev/null +++ b/app/crowdb-web/tests/protocol_observation_test.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/protocol_observation.rs"] +mod protocol_observation; + +use crowdb_web::protocol_probe_for_tests; +use protocol_observation::TestListener; +use std::time::{Duration, Instant}; + +#[tokio::test] +async fn protocols_require_their_own_successful_readiness_body() { + for (protocol, body, token) in [ + ( + "s3", + "", + None, + ), + ( + "iceberg", + r#"{"defaults":{},"overrides":{}}"#, + Some("fixture-token"), + ), + ( + "dataset", + r#"{"id":"existing","name":"crowdb_readiness_probe"}"#, + Some("fixture-token"), + ), + ] { + let listener = TestListener::serve(200, body.into(), Duration::ZERO, token.is_some()).await; + assert_eq!( + protocol_probe_for_tests(protocol, &listener.url, token).await, + Some(true) + ); + } + let listener = TestListener::serve(200, r#"{"ready":true}"#.into(), Duration::ZERO, true).await; + assert_eq!( + protocol_probe_for_tests("iceberg", &listener.url, Some("fixture-token")).await, + None + ); +} + +#[tokio::test] +async fn authenticated_dataset_absence_proves_a_read_without_creating_data() { + let listener = TestListener::serve(404, r#"{"code":"not_found"}"#.into(), Duration::ZERO, true).await; + assert_eq!( + protocol_probe_for_tests("dataset", &listener.url, Some("fixture-token")).await, + Some(true) + ); + let unauthorized = TestListener::serve(401, r#"{"code":"not_found"}"#.into(), Duration::ZERO, true).await; + assert_eq!( + protocol_probe_for_tests("dataset", &unauthorized.url, Some("fixture-token")).await, + Some(false) + ); +} + +#[tokio::test] +async fn malformed_oversized_and_redirected_replies_do_not_prove_readiness() { + for (status, body, expected) in [ + (200, "not-json".into(), None), + (200, " ".repeat(65537), None), + (302, r#"{"ready":true}"#.into(), Some(false)), + (503, r#"{"ready":true}"#.into(), Some(false)), + ] { + let listener = TestListener::serve(status, body, Duration::ZERO, false).await; + assert_eq!( + protocol_probe_for_tests("s3", &listener.url, None).await, + expected + ); + } +} + +#[tokio::test] +async fn a_stalled_listener_has_a_bounded_observation() { + let listener = TestListener::serve(200, r#"{"ready":true}"#.into(), Duration::from_secs(30), false).await; + let started = Instant::now(); + assert_eq!(protocol_probe_for_tests("s3", &listener.url, None).await, None); + assert!(started.elapsed() < Duration::from_secs(3)); +} diff --git a/app/crowdb-web/tests/s3_observation_test.rs b/app/crowdb-web/tests/s3_observation_test.rs new file mode 100644 index 000000000..b0eae31a2 --- /dev/null +++ b/app/crowdb-web/tests/s3_observation_test.rs @@ -0,0 +1,64 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{body::Body, http::Request, response::Response, routing::get, Router}; +use crowdb_access_s3::auth::{RawAuthRequest, RequestAuthenticator, SigV4Verifier}; +use crowdb_web::{s3_protocol_probe_for_tests, AppState}; + +struct TestCredentials; +impl crowdb_access_s3::auth::CredentialProvider for TestCredentials { + fn lookup(&self, access_key: &str) -> Option { + (access_key == "AKIDEXAMPLE").then(|| crowdb_access_s3::auth::Credential { + secret_key: b"server-secret".to_vec(), + session_token: None, + enabled: true, + }) + } +} + +#[tokio::test] +async fn local_s3_probe_signs_list_buckets_and_reports_exact_failures() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("s3-observation"); + let credentials = crowdb_monitor::ServerCredentials::load_or_create(root.path()).unwrap(); + credentials + .persist_client(&crowdb_monitor::ClientCredentials { + s3_endpoint: "http://public.example.test:19091".into(), + iceberg_endpoint: "http://public.example.test:19092".into(), + dataset_endpoint: None, + region: "us-east-1".into(), + access_key_id: "AKIDEXAMPLE".into(), + secret_access_key: "server-secret".into(), + }) + .unwrap(); + let state = AppState::with_runtime_root( + crowdb_console_shared::ConsoleConfig::default(), + root.path().to_owned(), + ); + for (status, body, code) in [ + (200, "", None), + (200, "existing", None), + (200, r#"{"ready":true}"#, Some("protocol-response-invalid")), + (200, "", Some("protocol-response-invalid")), + (403, "AccessDenied", Some("protocol-http-status")), + ] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let app = Router::new().route("/", get(move |request: Request| async move { + assert_eq!(request.method(), "GET"); + assert_eq!(request.uri().path(), "/"); + assert!(request.uri().query().is_none()); + SigV4Verifier::new(TestCredentials, "us-east-1".into(), 60) + .authenticate(RawAuthRequest::from_parts(request.method(), request.uri(), request.headers())) + .await.unwrap(); + Response::builder().status(status).body(Body::from(body)).unwrap() + })); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let result = s3_protocol_probe_for_tests(&state, &url).await; + assert_eq!(result.as_ref().err().map(|(code, _)| *code), code, "{result:?}"); + let detail = match result { Ok(detail) | Err((_, detail)) => detail }; + assert!(detail.contains(&format!("HTTP {status}"))); + assert!(!detail.contains("server-secret")); + assert!(!detail.contains("AKIDEXAMPLE")); + task.abort(); + } +} diff --git a/app/crowdb-web/tests/service_dependency_failure_test.rs b/app/crowdb-web/tests/service_dependency_failure_test.rs new file mode 100644 index 000000000..0e2496a1a --- /dev/null +++ b/app/crowdb-web/tests/service_dependency_failure_test.rs @@ -0,0 +1,15 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::http::StatusCode; +use crowdb_console_shared::error::Error; + +#[test] +fn dependency_wait_is_a_conflict_while_execution_failures_remain_errors() { + let (status, body) = + crowdb_web::error::map_config_err(Error::DependenciesNotReady("DiskDB is loading".into())); + assert_eq!(status, StatusCode::CONFLICT); + assert_eq!(body.0.error, "Waiting: DiskDB is loading"); + let (status, _) = crowdb_web::error::map_config_err(Error::Config("operation was superseded".into())); + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); +} diff --git a/app/crowdb-web/tests/service_observation_test.rs b/app/crowdb-web/tests/service_observation_test.rs new file mode 100644 index 000000000..2c1097c11 --- /dev/null +++ b/app/crowdb-web/tests/service_observation_test.rs @@ -0,0 +1,114 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{routing::get, Json, Router}; +use crowdb_console_shared::{config::NodeEntry, ConsoleConfig}; +use crowdb_protocol::mgmt::node::{NodeServiceAction, NodeServiceIntent}; +use crowdb_web::{service_detail_for_tests, AppState}; + +#[tokio::test] +async fn service_evidence_identifies_the_current_instance_and_owned_range() { + let config = ConsoleConfig { + nodes: vec![NodeEntry { + id: 1, + rack_id: 1, + host: "127.0.0.1".into(), + ssh_port: 22, + ssh_user: "crowdb".into(), + ssh_credential_ref: None, + ssh_key: None, + ssh_password: None, + }], + ..Default::default() + }; + let state = AppState::with_runtime_root(config, std::path::PathBuf::from("/unused")); + for instance in [7, 8] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = Router::new().route("/health", get(move || async move { Json(serde_json::json!({ + "instance_id": instance, + "detail": "Instance Serving; catalog generation 9; 1 owned tree range, 0 ready", + "ranges": [{"partition_id":{"high":1,"low":2},"owner_epoch":4,"ready":false,"detail":"Owned range writer is Prepared; waiting for recovery and activation"}] + })) })); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let intent = NodeServiceIntent { + cluster_id: uuid::Uuid::new_v4().to_string(), + node_id: 1, + service_id: "chunk-kv-1".into(), + kind: "chunk-kv".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + action: NodeServiceAction::Start, + configuration: format!("instance_id = 7\nhttp_listen_addr = \"{address}\"\n"), + environment: std::collections::BTreeMap::new(), + }; + let result = service_detail_for_tests(&state, &intent).await; + if instance == 7 { + let detail = result.unwrap(); + assert!(detail.contains("catalog generation 9")); + assert!(detail.contains("Prepared")); + assert!(detail.contains("owner_epoch")); + } else { + assert!(result + .unwrap_err() + .contains("differs from current committed intent")); + } + task.abort(); + } +} + +#[tokio::test] +async fn access_evidence_preserves_failed_dependency_details_from_http_503() { + for metadata in ["unavailable", "unknown"] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = Router::new().route( + "/_crowdb/health/ready", + get(move || async move { + ( + axum::http::StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ + "ready": false, "listener": "ready", "metadata": metadata, + "chunks": "ready", "native_pool": "ready", "cleanup": "ready", + "authentication": "ready", "cleanup_backlog": 0, "cleanup_backlog_limit": 10, + "details": {"metadata": "metadata read exceeded its 750 ms deadline"}, + "observation": {"state": if metadata == "unknown" { "expired" } else { "current" }} + })), + ) + }), + ); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let mut config = ConsoleConfig::default(); + config.nodes.push(NodeEntry { + id: 1, + rack_id: 1, + host: "127.0.0.1".into(), + ssh_port: 22, + ssh_user: "crowdb".into(), + ssh_credential_ref: None, + ssh_key: None, + ssh_password: None, + }); + let state = AppState::with_runtime_root(config, std::path::PathBuf::from("/unused")); + let intent = NodeServiceIntent { + cluster_id: uuid::Uuid::new_v4().to_string(), + node_id: 1, + service_id: "access-1".into(), + kind: "access".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + action: NodeServiceAction::Start, + configuration: format!("[health]\nlisten = \"{address}\"\n"), + environment: std::collections::BTreeMap::new(), + }; + let detail = service_detail_for_tests(&state, &intent).await.unwrap(); + assert!(detail.contains("HTTP 503")); + assert!(detail.contains(&format!("metadata={metadata}"))); + assert!(detail.contains(if metadata == "unknown" { + "expired" + } else { + "current" + })); + assert!(detail.contains("chunks=ready")); + assert!(detail.contains("750 ms deadline")); + task.abort(); + } +} diff --git a/app/crowdb-web/tests/storage_readiness_test.rs b/app/crowdb-web/tests/storage_readiness_test.rs index 3e061d16d..5832be703 100644 --- a/app/crowdb-web/tests/storage_readiness_test.rs +++ b/app/crowdb-web/tests/storage_readiness_test.rs @@ -2,6 +2,8 @@ // Licensed under the Apache License, Version 2.0. #[path = "common/memory_kv.rs"] mod memory_kv; +#[path = "common/storage_readiness.rs"] +mod storage_readiness; use axum::{body::Body, http::Request}; use crowdb_console_shared::{ @@ -44,6 +46,7 @@ async fn chunk_storage_requires_live_complete_ownership_even_when_diskio_pid_is_ .await .unwrap(); } + storage_readiness::publish_disk(&hardware).await; let registry = ServiceRegistryClient::from_shared(kv.clone()); let mut config = ConsoleConfig::default(); let endpoint = "127.0.0.1:13010"; @@ -73,6 +76,34 @@ async fn chunk_storage_requires_live_complete_ownership_even_when_diskio_pid_is_ .heartbeat_diskio(1, endpoint, &[11, 12], &[]) .await .unwrap(); + assert_eq!( + readiness(&app).await["ready"], + false, + "DiskIO ownership cannot substitute for DiskDB's loaded media" + ); + registry + .register_diskdb(1, "127.0.0.1:7102", &[11, 12], &[]) + .await + .unwrap(); + assert_eq!( + readiness(&app).await["ready"], + false, + "DiskDB ownership cannot substitute for its loaded media" + ); + registry + .heartbeat_diskdb( + 1, + "127.0.0.1:7102", + &[11, 12], + &[crowdb_protocol::common::DiskGroupUsageSummary { + disk_group_id: 11, + usage_known: true, + disk_count: 1, + ..Default::default() + }], + ) + .await + .unwrap(); assert_eq!(readiness(&app).await["ready"], true); registry.unregister("diskio", 1).await.unwrap(); assert_eq!(readiness(&app).await["ready"], false); diff --git a/app/crowdb-web/ui/src/App.tsx b/app/crowdb-web/ui/src/App.tsx index bc8dd9cd4..140127f87 100644 --- a/app/crowdb-web/ui/src/App.tsx +++ b/app/crowdb-web/ui/src/App.tsx @@ -532,6 +532,7 @@ function AppContent({ apiPrefix = '/api', readonly = false, modules, onEvent, ma addNodeRequest={addNodeRequest} onCandidatesChanged={handleRefresh} nodeManagement={nodeManagement} + nodeAuthority={nodeAuthority} allServers={allServers} racks={racks} servers={servers} diff --git a/app/crowdb-web/ui/src/api.ts b/app/crowdb-web/ui/src/api.ts index c6891ea68..ca463ff53 100644 --- a/app/crowdb-web/ui/src/api.ts +++ b/app/crowdb-web/ui/src/api.ts @@ -1196,6 +1196,7 @@ export async function removeDisk(nodeId: number, dgId: number, diskId: string, o /** `GET /api/servers` — list all deployed server entries. */ export interface ServerSummary { + start_pending?: boolean; id?: string; node_id?: number; mgmt_url?: string; diff --git a/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.test.tsx b/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.test.tsx index c3be51441..341c079a2 100644 --- a/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.test.tsx +++ b/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.test.tsx @@ -14,7 +14,7 @@ it('keeps inherited and current journal identities and recovery dependencies sep state: 'Serving', transition_id: 'transition-1', artifact: { tree_id: '9007199254740995', stream_name: stream, tail_overlay: { source_partition_id: { ...stream, low: '1' }, source_epoch: '9007199254740991', - source_stream_name: { ...stream, low: '3' }, source_stream_manifest_generation: '15', + source_stream_name: { ...stream, low: '3' }, source_stream_generation: '15', replay_offset: '4096', cutover_offset: '8192', cutover_seq: '50', base_root_manifest_generation: '7', base_tree_manifest: '6', base_applied_seq: '40', target_stream_start_seq: '51', }, @@ -22,7 +22,7 @@ it('keeps inherited and current journal identities and recovery dependencies sep }; const props = { partition, generation: '9007199254740997', currentGeneration: '9007199254740997', active: false, catalogPage: 0, catalogOffset: 0, onBack: vi.fn(), onChunk: vi.fn(), propertyHost: null, - query: { tab: 'Journal', stream: { offset: 0 }, extent: null }, onQuery: vi.fn() }; + query: { tab: 'Journal', stream: { offset: 0 }, chunkRecord: null }, onQuery: vi.fn() }; const { rerender } = render(); const inherited = screen.getByRole('heading', { name: 'Inherited parent stream' }).parentElement!; expect(inherited).toHaveTextContent('ffffffffffffffff0000000000000003'); diff --git a/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.tsx b/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.tsx index 1ac3801c7..bb10df3f7 100644 --- a/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.tsx +++ b/app/crowdb-web/ui/src/chunk-kv/PartitionDetail.tsx @@ -20,7 +20,7 @@ export function PartitionDetail({ partition: p, generation, currentGeneration, a const tab = query.tab; const setTab = (tab: string) => onQuery({ ...query, tab }); const overlay = p.artifact.tail_overlay; - const runtime = useRuntimeObservation({ active, partition: p, generation, catalogPage, catalogOffset, cursor: query.stream, onCursor: stream => onQuery({ ...query, stream, extent: null }) }); + const runtime = useRuntimeObservation({ active, partition: p, generation, catalogPage, catalogOffset, cursor: query.stream, onCursor: stream => onQuery({ ...query, stream, chunkRecord: null }) }); return
{propertyHost && createPortal(

Split properties

@@ -38,10 +38,10 @@ export function PartitionDetail({ partition: p, generation, currentGeneration, a {tab === 'Tree' && <> onQuery({ ...query, page })} propertyHost={propertyHost} />} {tab === 'Journal' && <> - {overlay &&

Inherited parent stream

} + {overlay &&

Inherited parent stream

}

Partition journal stream

Streams have independent byte offsets. Durable and applied sequence numbers refer to this partition's journal.

- onQuery({ ...query, extent })} key={`${runtime.value?.journal?.generation}/${runtime.value?.journal?.offset}`} value={runtime.value?.journal} disabled={!active || runtime.busy || !!runtime.error} propertyHost={propertyHost} onPage={runtime.pageStream} onChunk={onChunk} /> + onQuery({ ...query, chunkRecord })} key={`${runtime.value?.journal?.generation}/${runtime.value?.journal?.offset}`} value={runtime.value?.journal} disabled={!active || runtime.busy || !!runtime.error} propertyHost={propertyHost} onChunk={onChunk} /> } {tab === 'Dependencies' && (overlay ? <>

Parent recovery dependency is retained in this catalog generation. Serving does not imply that materialization has completed.

:

No parent-tail overlay is recorded in this catalog entry. Runtime readiness and other retention pins require separate observations.

)} diff --git a/app/crowdb-web/ui/src/chunk-kv/StorageObservation.test.tsx b/app/crowdb-web/ui/src/chunk-kv/StorageObservation.test.tsx index 1950f6db3..eefb4b762 100644 --- a/app/crowdb-web/ui/src/chunk-kv/StorageObservation.test.tsx +++ b/app/crowdb-web/ui/src/chunk-kv/StorageObservation.test.tsx @@ -7,31 +7,28 @@ import { identity } from './catalog'; import type { JournalObservation } from './useRuntimeObservation'; afterEach(cleanup); -it('replaces extent windows and preserves exact identities and selected byte boundaries', () => { +it('preserves exact chunk identities, logical boundaries and mapping sizes', () => { const chunk = identity({ high: '18446744073709551615', low: '2' }); expect(chunk).toBe('ffffffffffffffff0000000000000002'); - const window = (offset: number, count: number): JournalObservation => ({ + const value: JournalObservation = { generation: '9007199254740999', writer_epoch: '9007199254740993', metadata_group_id: '7', - trim_offset: '4096', sealed_tail: '18446744073709551615', closed: false, - active: { chunk_id: chunk, physical_start: '64', logical_start: '8192', acknowledged_cursor: '128', capacity: '4096' }, - offset, next_offset: offset ? null : 100, - extent_pages: Array.from({ length: count }, (_, index) => ({ page_index: String(offset + index), first_logical: String((offset + index) * 64), end_logical: String((offset + index + 1) * 64) })), - }); - const onPage = vi.fn(); const onChunk = vi.fn(); const onSelected = vi.fn(); + trim_offset: '4096', published_tail: '18446744073709551615', closed: false, offset: 0, next_offset: null, + chunk_record: { chunk_id: chunk, state: 'Active', logical_start: '8192', logical_end: '18446744073709551615', mapping_count: '4096', mapping_bytes: '247' }, + }; + const onChunk = vi.fn(); const onSelected = vi.fn(); const propertyHost = document.createElement('div'); document.body.append(propertyHost); try { - const props = { value: window(0, 100), disabled: false, onPage, onChunk, onSelected, propertyHost, selected: null as string | null }; + const props = { value, disabled: false, onChunk, onSelected, propertyHost, selected: null as string | null }; const { rerender } = render(); - expect(screen.getByLabelText('Extent page map').querySelectorAll('button')).toHaveLength(100); expect(screen.getByLabelText('Journal storage')).toHaveTextContent('18446744073709551615'); - fireEvent.click(screen.getByRole('button', { name: 'Inspect active Chunk' })); expect(onChunk).toHaveBeenCalledWith(chunk); - fireEvent.click(screen.getByRole('button', { name: 'Next extent pages' })); expect(onPage).toHaveBeenCalledWith(100); - rerender(); - expect(screen.getByLabelText('Extent page map').querySelectorAll('button')).toHaveLength(5); - expect(screen.getByLabelText('Selected extent page')).toHaveTextContent('6464'); - expect(screen.getByRole('button', { name: 'Next extent pages' })).toBeDisabled(); - expect(screen.queryByRole('button', { name: 'Extent page 0 [0, 64)' })).toBeNull(); - rerender(); - expect(screen.queryByLabelText('Selected extent page')).toBeNull(); + fireEvent.click(screen.getByRole('button', { name: 'Inspect Chunk' })); expect(onChunk).toHaveBeenCalledWith(chunk); + fireEvent.click(screen.getByRole('button', { name: 'Select stream chunk record' })); expect(onSelected).toHaveBeenCalledWith(chunk); + rerender(); + expect(screen.getByLabelText('Selected stream chunk record')).toHaveTextContent('8192'); + expect(screen.getByLabelText('Selected stream chunk record')).toHaveTextContent('247'); + rerender(); + expect(screen.queryByLabelText('Selected stream chunk record')).toBeNull(); + rerender(); + expect(screen.getByRole('button', { name: 'Inspect Chunk' })).toBeDisabled(); } finally { propertyHost.remove(); } }); diff --git a/app/crowdb-web/ui/src/chunk-kv/StorageObservation.tsx b/app/crowdb-web/ui/src/chunk-kv/StorageObservation.tsx index 1f41fc38e..11e8dd11a 100644 --- a/app/crowdb-web/ui/src/chunk-kv/StorageObservation.tsx +++ b/app/crowdb-web/ui/src/chunk-kv/StorageObservation.tsx @@ -23,25 +23,20 @@ export function TreeStorage({ value }: { value?: TreeObservation }) {
; } -export function JournalStorage({ value, disabled, onPage, onChunk, propertyHost, selected, onSelected }: { selected: string | null; onSelected: (index: string) => void; propertyHost: HTMLDivElement | null; onChunk: (id: string) => void; value?: JournalObservation | null; disabled: boolean; onPage: (offset: number) => void }) { +export function JournalStorage({ value, disabled, onChunk, propertyHost, selected, onSelected }: { selected: string | null; onSelected: (id: string) => void; propertyHost: HTMLDivElement | null; onChunk: (id: string) => void; value?: JournalObservation | null; disabled: boolean }) { if (!value) return

Journal storage observation unavailable.

; - const extent = value.extent_pages.find(page => page.page_index === selected); + const record = value.chunk_record; + const fields = { 'Chunk ID': record.chunk_id, 'Chunk state': record.state, 'Logical start (bytes)': record.logical_start, + 'Logical end (exclusive bytes)': record.logical_end, 'Append mappings': record.mapping_count, 'Compressed mapping bytes': record.mapping_bytes }; return
- -

Active chunk

- {value.active ? :

No active chunk in this manifest.

} - {value.active && } + +

Stream chunk record

+

Published tail record. Logical bounds describe its checkpoint; current appends may extend beyond it.

+
+ +
-

Published extent index

-

{value.extent_pages.length} loaded page fences · offset {value.offset}. Ordered blocks show logical byte ranges; widths do not represent sizes. Extent records and journal payloads are not read.

-
{value.extent_pages.map(page => )}
- {!value.extent_pages.length &&

No published extent pages in this window.

} - {extent && propertyHost && createPortal(, propertyHost)} -
-
+ {selected === record.chunk_id && propertyHost && createPortal(, propertyHost)}
; } diff --git a/app/crowdb-web/ui/src/chunk-kv/catalog.ts b/app/crowdb-web/ui/src/chunk-kv/catalog.ts index 34d2eb5f9..4483272f4 100644 --- a/app/crowdb-web/ui/src/chunk-kv/catalog.ts +++ b/app/crowdb-web/ui/src/chunk-kv/catalog.ts @@ -6,7 +6,7 @@ export interface Overlay { source_partition_id: Identity; source_epoch: string; source_stream_name: Identity; - source_stream_manifest_generation: string; + source_stream_generation: string; replay_offset: string; cutover_offset: string; base_root_manifest_generation: string; diff --git a/app/crowdb-web/ui/src/chunk-kv/query.ts b/app/crowdb-web/ui/src/chunk-kv/query.ts index 8d408768c..e525f2ac8 100644 --- a/app/crowdb-web/ui/src/chunk-kv/query.ts +++ b/app/crowdb-web/ui/src/chunk-kv/query.ts @@ -4,9 +4,9 @@ import type { PageCursor } from './PageExplorer'; export interface SplitQuery { tab: string; stream: { generation?: string; offset: number }; - extent: string | null; + chunkRecord: string | null; page?: PageCursor; } -export const initialSplitQuery = (): SplitQuery => ({ tab: 'Overview', stream: { offset: 0 }, extent: null }); +export const initialSplitQuery = (): SplitQuery => ({ tab: 'Overview', stream: { offset: 0 }, chunkRecord: null }); export interface GraphQuery { serverPage: number; offsets: Record; collapsed: string[] } export const initialGraphQuery = (): GraphQuery => ({ serverPage: 0, offsets: {}, collapsed: [] }); diff --git a/app/crowdb-web/ui/src/chunk-kv/useRuntimeObservation.test.tsx b/app/crowdb-web/ui/src/chunk-kv/useRuntimeObservation.test.tsx index 6650be447..179a98d29 100644 --- a/app/crowdb-web/ui/src/chunk-kv/useRuntimeObservation.test.tsx +++ b/app/crowdb-web/ui/src/chunk-kv/useRuntimeObservation.test.tsx @@ -24,24 +24,25 @@ it('rejects a late old-owner observation and exposes stale runtime cursors', asy expect(result.current.value).toBeNull(); }); -it('rejects inconsistent stream generations and oversized windows before publishing runtime data', async () => { - const journal = (generation: string, count: number) => ({ generation, offset: 100, extent_pages: Array.from({ length: count }, (_, index) => ({ page_index: String(index) })) }); +it('rejects inconsistent generations and malformed chunk records before publishing runtime data', async () => { + const record = { chunk_id: 'ffffffffffffffff0000000000000002', state: 'Active', logical_start: '0', logical_end: '9007199254740999', mapping_count: '8', mapping_bytes: '10' }; + const journal = (generation: string) => ({ generation, offset: 0, chunk_record: record }); const fetcher = vi.fn() - .mockResolvedValueOnce(new Response(JSON.stringify({ journal: journal('18', 5) }))) - .mockResolvedValueOnce(new Response(JSON.stringify({ journal: journal('17', 101) }))) - .mockResolvedValueOnce(new Response(JSON.stringify({ journal: { ...journal('17', 1), offset: 0 } }))); + .mockResolvedValueOnce(new Response(JSON.stringify({ journal: journal('18') }))) + .mockResolvedValueOnce(new Response(JSON.stringify({ journal: { ...journal('17'), chunk_record: { ...record, logical_start: '9999999999999999999' } } }))) + .mockResolvedValueOnce(new Response(JSON.stringify({ journal: journal('17') }))); vi.stubGlobal('fetch', fetcher); const props = { active: true, partition: { id: 'partition', epoch: '9007199254740993' } as Partition, generation: '9007199254740997', catalogPage: 0, catalogOffset: 0, - cursor: { generation: '17', offset: 100 }, onCursor: vi.fn() }; + cursor: { generation: '17', offset: 0 }, onCursor: vi.fn() }; const { result, rerender } = renderHook(useRuntimeObservation, { initialProps: props }); - await waitFor(() => expect(result.current.error).toContain('Stream manifest changed')); + await waitFor(() => expect(result.current.error).toContain('Stream chunk record changed')); expect(result.current.value).toBeNull(); rerender({ ...props, cursor: { generation: '17', offset: 0 } }); - await waitFor(() => expect(result.current.error).toContain('Extent index exceeds 100')); + await waitFor(() => expect(result.current.error).toContain('Invalid stream chunk record')); expect(result.current.value).toBeNull(); act(() => result.current.refresh()); expect(props.onCursor).toHaveBeenCalledWith({ offset: 0 }); - await waitFor(() => expect(result.current.value?.journal?.extent_pages).toHaveLength(1)); + await waitFor(() => expect(result.current.value?.journal?.chunk_record.chunk_id).toBe(record.chunk_id)); expect(new URL(fetcher.mock.calls[0][0], 'http://localhost').searchParams.get('epoch')).toBe('9007199254740993'); }); diff --git a/app/crowdb-web/ui/src/chunk-kv/useRuntimeObservation.ts b/app/crowdb-web/ui/src/chunk-kv/useRuntimeObservation.ts index 876cd9790..b66294940 100644 --- a/app/crowdb-web/ui/src/chunk-kv/useRuntimeObservation.ts +++ b/app/crowdb-web/ui/src/chunk-kv/useRuntimeObservation.ts @@ -8,9 +8,8 @@ import type { Partition } from './catalog'; export interface JournalObservation { generation: string; writer_epoch: string; metadata_group_id: string; - trim_offset: string; sealed_tail: string; closed: boolean; - active: { chunk_id: string; physical_start: string; logical_start: string; acknowledged_cursor: string; capacity: string } | null; - extent_pages: { page_index: string; first_logical: string; end_logical: string }[]; + trim_offset: string; published_tail: string; closed: boolean; + chunk_record: { chunk_id: string; state: string; logical_start: string; logical_end: string; mapping_count: string; mapping_bytes: string }; offset: number; next_offset: number | null; } export interface TreeObservation { @@ -42,8 +41,8 @@ export function useRuntimeObservation({ active, partition, generation, catalogPa fetch(`${getApiBase()}/chunk-kv/runtime?${query}`, { signal: abort.signal, headers: token ? { Authorization: `Bearer ${token}` } : {} }) .then(response => readJson(response)) .then(value => { - if ((value.journal?.extent_pages.length ?? 0) > 100) throw new Error('Extent index exceeds 100 entries'); - if (cursor.generation && (value.journal?.generation !== cursor.generation || value.journal.offset !== cursor.offset)) throw new Error('Stream manifest changed; refresh runtime'); + if (value.journal && !validChunkRecord(value.journal.chunk_record)) throw new Error('Invalid stream chunk record'); + if (cursor.generation && (value.journal?.generation !== cursor.generation || value.journal.offset !== cursor.offset)) throw new Error('Stream chunk record changed; refresh runtime'); if (!abort.signal.aborted) setValue(value); }) .catch(error => { if (!abort.signal.aborted) setError(String(error)); }) @@ -52,6 +51,12 @@ export function useRuntimeObservation({ active, partition, generation, catalogPa }, [active, partition.id, partition.epoch, generation, catalogPage, catalogOffset, cursor, revision]); return { value, error, busy, refresh: () => { onCursor({ offset: 0 }); setRevision(value => value + 1); }, - pageStream: (offset: number) => { if (value?.journal) onCursor({ generation: value.journal.generation, offset }); }, }; } + +function validChunkRecord(record: JournalObservation['chunk_record']): boolean { + if (!record || !/^[0-9a-f]{32}$/.test(record.chunk_id) || !['Active', 'Sealed'].includes(record.state)) return false; + const integers = [record.logical_start, record.logical_end, record.mapping_count, record.mapping_bytes]; + return integers.every(value => typeof value === 'string' && /^\d{1,20}$/.test(value)) + && BigInt(record.logical_start) <= BigInt(record.logical_end); +} diff --git a/app/crowdb-web/ui/src/data/useClusterTree.test.ts b/app/crowdb-web/ui/src/data/useClusterTree.test.ts new file mode 100644 index 000000000..3e782091a --- /dev/null +++ b/app/crowdb-web/ui/src/data/useClusterTree.test.ts @@ -0,0 +1,46 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import { act, cleanup, renderHook } from '@testing-library/react'; +import { afterEach, expect, it, vi } from 'vitest'; +import { listNodes, listRacks } from '../api'; +import type { Rack } from '../types'; +import { useClusterTree } from './useClusterTree'; + +vi.mock('../api', () => ({ listRacks: vi.fn(), listNodes: vi.fn(), + listNodeDiskGroups: vi.fn(), listDisksInGroup: vi.fn() })); +afterEach(() => { cleanup(); vi.useRealTimers(); vi.resetAllMocks(); }); + +it('lets a slow post-mutation refresh finish without polling cancelling its response', async () => { + vi.useFakeTimers(); + vi.mocked(listRacks).mockResolvedValue([]); + vi.mocked(listNodes).mockResolvedValue([]); + const { result } = renderHook(() => useClusterTree({ pollIntervalActive: 1000 })); + await act(async () => {}); + let finish!: (racks: Rack[]) => void; + vi.mocked(listRacks).mockReturnValueOnce(new Promise(resolve => { finish = resolve; })); + let refreshed!: Promise; + act(() => { refreshed = result.current.refresh(); }); + const signal = vi.mocked(listRacks).mock.calls[1][1]!.signal!; + await act(async () => { await vi.advanceTimersByTimeAsync(2100); }); + expect(signal.aborted).toBe(false); + expect(listRacks).toHaveBeenCalledTimes(2); + const racks: Rack[] = [{ id: 61, name: 'Rack Sixty-One', nodes: [] }]; + await act(async () => { finish(racks); await refreshed; }); + expect(result.current.racks).toEqual(racks); + expect(result.current.loading).toBe(false); +}); + +it('starts a fresh read after a mutation and fences an older observation', async () => { + vi.useFakeTimers(); + let finish!: (racks: Rack[]) => void; + vi.mocked(listRacks).mockReturnValueOnce(new Promise(resolve => { finish = resolve; })) + .mockResolvedValue([{ id: 61, name: 'new', nodes: [] }]); + vi.mocked(listNodes).mockResolvedValue([]); + const { result } = renderHook(() => useClusterTree()); + const previous = vi.mocked(listRacks).mock.calls[0][1]!.signal!; + await act(async () => { await result.current.refresh(); }); + expect(previous.aborted).toBe(true); + await act(async () => { finish([]); }); + expect(result.current.racks).toEqual([{ id: 61, name: 'new', nodes: [] }]); +}); diff --git a/app/crowdb-web/ui/src/data/useClusterTree.ts b/app/crowdb-web/ui/src/data/useClusterTree.ts index 39194f74c..2aa2e7c10 100644 --- a/app/crowdb-web/ui/src/data/useClusterTree.ts +++ b/app/crowdb-web/ui/src/data/useClusterTree.ts @@ -61,8 +61,10 @@ export function useClusterTree({ const hasLoadedRef = useRef(false); const requestRef = useRef(null); - const fetchData = useCallback(async () => { + const fetchData = useCallback(async (fresh = false) => { if (!enabled || document.visibilityState === 'hidden') return; + // Polling must not starve a slow read or a read started after a mutation. + if (requestRef.current && !requestRef.current.signal.aborted && !fresh) return; requestRef.current?.abort(); const controller = new AbortController(); requestRef.current = controller; @@ -110,6 +112,7 @@ export function useClusterTree({ setError(err instanceof Error ? err : new Error('Unknown error fetching cluster tree')); } finally { if (!controller.signal.aborted) { hasLoadedRef.current = true; setLoading(false); } + if (requestRef.current === controller) requestRef.current = null; } }, [enabled, recursive, managed]); @@ -156,7 +159,7 @@ export function useClusterTree({ [nodes], ); - const refresh = useCallback(async () => { await Promise.all([fetchData(), disks.refresh()]); }, [fetchData, disks.refresh]); + const refresh = useCallback(async () => { await Promise.all([fetchData(true), disks.refresh()]); }, [fetchData, disks.refresh]); return { services, diff --git a/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx b/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx index 6ea3282f3..6cedf86f8 100644 --- a/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx +++ b/app/crowdb-web/ui/src/services/DeployServiceDialog.tsx @@ -31,7 +31,7 @@ export function DeployServiceDialog({ nodeId, kind, servers, stores, diskGroups, const [bootstrap, setBootstrap] = useState(!!firstGroup); const [group, setGroup] = useState(String(firstGroup?.group_id ?? '')); const [dynamicOwnership, setDynamicOwnership] = useState(false); - const [testMode, setTestMode] = useState(false); + const [localDeployment, setLocalDeployment] = useState(false); const defaults = useDeploymentDefaults(true); useEffect(() => { const value = defaults.values?.[kind]; @@ -57,7 +57,8 @@ export function DeployServiceDialog({ nodeId, kind, servers, stores, diskGroups, try { await serviceRequest(`/nodes/${nodeId}/services/deploy`, 'POST', { ...(kind === 'chunkdb' ? { dynamic_ownership: dynamicOwnership } : {}), - kind, instance_id: instance, test_single_node: testMode, + kind, instance_id: instance, + ...(localDeployment ? { local_deployment: true } : {}), ...(kind !== 'diskio' ? { http_port: Number(httpPort) } : {}), ...(kind === 'access-server' ? { s3_port: Number(s3Port), dataset_port: Number(datasetPort), health_port: Number(healthPort) } : { rpc_port: Number(rpcPort) }), ...(kind === 'diskio' ? { disk_group_id: Number(diskGroup) } : {}), @@ -90,8 +91,8 @@ export function DeployServiceDialog({ nodeId, kind, servers, stores, diskGroups, {kind === 'chunkdb' && } - - {kind === 'diskio' &&

Production requires disks with device paths. Test deployments may use in-memory disks.

} + + {kind === 'diskio' &&

Production requires disks with device paths. Local deployments may use in-memory disks.

} {defaults.error &&

{defaults.error}

} {!defaults.values && !defaults.error &&

Finding available IDs and ports…

} {error &&

{error}

} diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx index 1a214d917..25564a019 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.test.tsx @@ -2,11 +2,11 @@ // Licensed under the Apache License, Version 2.0. import { act, renderHook } from '@testing-library/react'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { deployServer, deployDiskdb, listServers } from '../api'; +import { deployServer, deployDiskdb, listServers, restartServer, restartDiskdb } from '../api'; import { serviceRequest } from './client'; import { newPlan, serviceOrder, useNodeServicePlans } from './useNodeServicePlans'; import type { EnrichedStoreView } from '../types'; -vi.mock('../api', () => ({ listServers: vi.fn(), deployServer: vi.fn(), deployDiskdb: vi.fn() })); +vi.mock('../api', () => ({ listServers: vi.fn(), deployServer: vi.fn(), deployDiskdb: vi.fn(), restartServer: vi.fn(), restartDiskdb: vi.fn() })); vi.mock('./client', () => ({ serviceNames: {}, serviceRequest: vi.fn() })); const stores = [{ store_id: '0', groups: [{ group_id: '0' }, { group_id: '1' }] }] as unknown as EnrichedStoreView[]; beforeEach(() => { @@ -19,6 +19,41 @@ beforeEach(() => { }); afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); describe('node service plans', () => { + it('observes an accepted start waiting for dependencies without declaring it stopped or relaunching it', async () => { + let started = false; + vi.mocked(listServers).mockImplementation(async () => [{ + id: 'chunkdb-1', node_id: 1, service_type: 'chunkdb', + pid: started ? 42 : undefined, start_pending: !started, + }] as Awaited>); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await result.current.start(1, ['chunkdb']); }); + expect(result.current.plans[1].chunkdb.state).toBe('waiting'); + started = true; + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + expect(result.current.plans[1].chunkdb.state).toBe('deployed'); + expect(vi.mocked(serviceRequest).mock.calls.some(([path, method]) => path.endsWith('/services/deploy') && method === 'POST')).toBe(false); + expect(restartServer).not.toHaveBeenCalled(); + }); + + it('restarts a registered failed deployment only after explicit Retry', async () => { + const steps = newPlan(); + for (const kind of serviceOrder) steps[kind] = { state: 'disabled' }; + steps.diskdb = { state: 'failed', detail: 'Listener bind failed' }; + vi.mocked(listServers).mockResolvedValue([{ id: 'diskdb-1', node_id: 1, service_type: 'diskdb' }] as Awaited>); + const request = vi.mocked(serviceRequest).getMockImplementation()!; + vi.mocked(serviceRequest).mockImplementation(async (path, method, body) => path === '/service-plans' + ? { 1: { revision: 4, steps, overrides: { diskdb: { rpc_port: 7100 } } } } : request(path, method, body)); + const { result } = renderHook(() => useNodeServicePlans(stores, {}, async () => {}, true)); + await act(async () => { await vi.advanceTimersByTimeAsync(4000); }); + expect(restartDiskdb).not.toHaveBeenCalled(); + await act(async () => { await result.current.start(1); }); + expect(restartDiskdb).toHaveBeenCalledTimes(1); + expect(restartDiskdb).toHaveBeenCalledWith(1); + expect(result.current.plans[1].diskdb.state).toBe('deployed'); + expect(deployDiskdb).not.toHaveBeenCalled(); + expect(restartServer).not.toHaveBeenCalled(); + }); + it('keeps Access queued while DiskIO ownership is unavailable and resumes without Retry', async () => { const first = newPlan(); for (const kind of serviceOrder) first[kind] = { state: 'deployed' }; @@ -276,7 +311,10 @@ describe('node service plans', () => { expect(result.current.plans[1]['access-server'].state).toBe('waiting'); expect(result.current.plans[1].diskio.state).toBe('deployed'); expect(result.current.plans[1]['chunk-kv'].state).toBe('waiting'); - expect(serviceRequest).toHaveBeenCalledWith('/nodes/1/services/deploy', 'POST', expect.objectContaining({ kind: 'chunkdb', test_single_node: false })); + expect(serviceRequest).toHaveBeenCalledWith('/nodes/1/services/deploy', 'POST', expect.objectContaining({ kind: 'chunkdb' })); + const deployment = vi.mocked(serviceRequest).mock.calls.find(([path, method]) => path === '/nodes/1/services/deploy' && method === 'POST'); + expect(deployment?.[2]).not.toHaveProperty('test_single_node'); + expect(deployment?.[2]).not.toHaveProperty('local_deployment'); vi.mocked(listServers).mockResolvedValue([{ node_id: 2, service_type: 'chunk-kv', pid: 100 }] as Awaited>); await act(async () => { await vi.advanceTimersByTimeAsync(10000); }); expect(result.current.plans[1]['access-server'].state).toBe('deployed'); diff --git a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts index d9a0f37be..3fd226e8f 100644 --- a/app/crowdb-web/ui/src/services/useNodeServicePlans.ts +++ b/app/crowdb-web/ui/src/services/useNodeServicePlans.ts @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. import { useCallback, useEffect, useRef, useState } from 'react'; -import { deployServer, deployDiskdb, listServers } from '../api'; +import { deployServer, deployDiskdb, listServers, restartServer, restartDiskdb } from '../api'; import type { EnrichedStoreView } from '../types'; import type { NodeDiskGroups } from '../data/useClusterTree'; import { serviceRequest } from './client'; @@ -10,6 +10,7 @@ import type { DeploymentDefaults } from './useDeploymentDefaults'; export const serviceOrder = ['access-server', 'chunk-kv', 'chunkdb', 'diskdb', 'diskio', 'paxos-kv'] as const; export type ServiceKind = typeof serviceOrder[number]; const deploymentOrder: ServiceKind[] = ['paxos-kv', 'diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server']; +const startingService = 'Waiting: target monitor is starting the registered service'; export const serviceLabels = { 'access-server': 'crowdb-access-server', 'chunk-kv': 'crowdb-chunk-kv', @@ -43,6 +44,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< const running = useRef>(Promise.resolve()); const registrations = useRef>(Promise.resolve()); const overrides = useRef>({}); + const retryRegistered = useRef(new Set()); const update = useCallback((id: number, plan: NodeServicePlan) => { plansRef.current = { ...plansRef.current, [id]: plan }; setPlans(plansRef.current); @@ -98,6 +100,9 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< const enabled = new Set(selected ?? serviceOrder.filter(kind => previous[kind].state !== 'disabled')); intended = Object.fromEntries(serviceOrder.map(kind => [kind, !enabled.has(kind) ? { state: 'disabled' } : ['failed', 'warning', 'disabled'].includes(previous[kind].state) ? { state: 'pending' } : previous[kind]])) as NodeServicePlan; await persist(id, intended); + for (const kind of enabled) { + if (['failed', 'warning'].includes(previous[kind].state)) retryRegistered.current.add(`${id}/${kind}`); + } } catch (error) { failProgress(id, String(error), intended); throw error; } finally { busy.current = false; wake.current(); } }); @@ -110,6 +115,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< stopped.current = true; await registrations.current; await running.current; + retryRegistered.current.clear(); plansRef.current = {}; setPlans({}); }, []); @@ -148,10 +154,30 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< if (plan[kind].state === 'failed' || plan[kind].state === 'warning') continue; const registered = existing.find(server => server.node_id === id && server.service_type === kind); if (registered) { - const step: ServiceStep = registered.pid ? { state: 'deployed' } : { state: 'failed', detail: 'Registered service is stopped; restart it from the Node menu before resuming.' }; + const retryKey = `${id}/${kind}`; + if (!registered.pid && retryRegistered.current.has(retryKey)) { + if (deploymentStarted) continue; + retryRegistered.current.delete(retryKey); + await write(kind, { state: 'deploying' }); + try { + if (kind === 'paxos-kv') await restartServer(id); + else if (kind === 'diskdb') await restartDiskdb(id); + else if (registered.id) await serviceRequest(`/services/${encodeURIComponent(registered.id)}/restart`, 'POST'); + else throw new Error('Registered service has no lifecycle identity'); + await write(kind, { state: 'deployed' }); + wakePending.current = true; + } catch (error) { await write(kind, { state: 'failed', detail: String(error) }); } + deploymentStarted = true; + continue; + } + retryRegistered.current.delete(retryKey); + const step: ServiceStep = registered.pid ? { state: 'deployed' } : registered.start_pending + ? { state: 'waiting', detail: startingService } + : { state: 'failed', detail: 'Registered service is stopped; restart it from the Node menu before resuming.' }; if (plan[kind].state !== step.state) await write(kind, step); continue; } + retryRegistered.current.delete(`${id}/${kind}`); if (plan[kind].state === 'deployed') { await write(kind, { state: 'failed', detail: 'Previously deployed service is missing; Retry after checking its lifecycle.' }); continue; } const { stores: currentStores, groups: currentGroups } = input.current; const metadata = currentStores.flatMap(store => store.groups.filter(group => String(group.group_id) !== '0').map(group => ({ store: store.store_id, group: group.group_id })))[0]; @@ -188,7 +214,7 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< if (kind === 'paxos-kv') await deployServer(id, { rest_port: value.http_port!, rpc_port: value.rpc_port! }); else if (kind === 'diskdb') await deployDiskdb(id, { rpc_port: value.rpc_port!, ...(value.keepalive_interval_secs === undefined ? {} : { keepalive_interval_secs: value.keepalive_interval_secs }) }); else await serviceRequest(`/nodes/${id}/services/deploy`, 'POST', { - kind, ...value, test_single_node: false, + kind, ...value, ...(kind === 'diskio' ? { disk_group_id: diskGroup?.id ?? 0 } : {}), ...(kind === 'chunk-kv' ? { metadata_store_id: Number(metadata!.store), bootstrap_group_id: Number(metadata!.group) } : {}), }); @@ -225,8 +251,9 @@ export function useNodeServicePlans(stores: EnrichedStoreView[], groups: Record< const hasQueuedWork = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].state === 'waiting')); const waitingForAuthority = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail === 'Waiting: initialize Group 0 in Paxos KV')); const waitingForRestart = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail?.startsWith('Waiting: restart registered DiskIO services'))); + const waitingForStart = Object.values(plansRef.current).some(plan => serviceOrder.some(kind => plan[kind].detail === startingService)); const hasDisks = Object.values(input.current.groups).some(node => Object.values(node.disksByDg).some(disks => disks.length > 0)); - const delay = wakePending.current ? 0 : hasQueuedWork && !hasDisks && !waitingForAuthority && !waitingForRestart ? 10000 : 2000; + const delay = wakePending.current ? 0 : hasQueuedWork && !hasDisks && !waitingForAuthority && !waitingForRestart && !waitingForStart ? 10000 : 2000; wakePending.current = false; timer = setTimeout(() => { running.current = tick(); }, delay); } diff --git a/app/crowdb-web/ui/src/shell/AdmitCandidateDialog.test.tsx b/app/crowdb-web/ui/src/shell/AdmitCandidateDialog.test.tsx index 27126a4c4..4beaccc1e 100644 --- a/app/crowdb-web/ui/src/shell/AdmitCandidateDialog.test.tsx +++ b/app/crowdb-web/ui/src/shell/AdmitCandidateDialog.test.tsx @@ -121,3 +121,23 @@ it('resumes previously accepted choices without replacing them with new defaults await waitFor(() => expect(admitCandidate).toHaveBeenCalledOnce()); expect(vi.mocked(admitCandidate).mock.calls[0][0].setup).toEqual(choices); }); + +it('reviews and submits the target startup choices without substituting browser defaults', async () => { + const choices = defaultNodeSetup(); + choices.storage = { mode: 'automatic', capacity_bytes: 8 * 1024 ** 3, reserve_bytes: 1024 ** 3, sparse: false }; + choices.services.forEach(service => { service.enabled = service.kind === 'diskdb'; }); + choices.services.find(service => service.kind === 'diskdb')!.listeners.rpc_port = 29920; + vi.mocked(prepareCandidate).mockResolvedValue({ operation_id: 'policy-admission', inventory: empty, policy_choices: choices }); + render(); + fireEvent.click(screen.getByRole('button', { name: 'Verify and review' })); + await screen.findByText(/Access verified/); + expect(screen.getByLabelText('Managed file capacity (GiB)')).toHaveValue('8'); + expect(screen.getByLabelText('Managed file capacity (GiB)')).toBeDisabled(); + expect(screen.getByLabelText('crowdb-disk-db RPC port')).toHaveValue('29920'); + expect(screen.getByLabelText('crowdb-disk-db RPC port')).toBeDisabled(); + expect(screen.getByRole('checkbox', { name: 'crowdb-disk-db' })).toBeChecked(); + expect(screen.getByRole('checkbox', { name: 'crowdb-disk-io' })).not.toBeChecked(); + fireEvent.click(screen.getByRole('button', { name: 'Join and start' })); + await waitFor(() => expect(admitCandidate).toHaveBeenCalledOnce()); + expect(vi.mocked(admitCandidate).mock.calls[0][0].setup).toEqual(choices); +}); diff --git a/app/crowdb-web/ui/src/shell/AdmitCandidateDialog.tsx b/app/crowdb-web/ui/src/shell/AdmitCandidateDialog.tsx index 41bf25cc0..d8cdd7229 100644 --- a/app/crowdb-web/ui/src/shell/AdmitCandidateDialog.tsx +++ b/app/crowdb-web/ui/src/shell/AdmitCandidateDialog.tsx @@ -20,6 +20,7 @@ export function AdmitCandidateDialog({ discoveryId, racks, initialRackId, curren const [choices, setChoices] = useState(defaultNodeSetup); const [submitted, setSubmitted] = useState(); const credential = useRef(null); + const fixedSetup = !!prepared?.accepted_choices || !!prepared?.policy_choices; const close = async () => { if (busy) return; const operation = prepared?.operation_id ?? preparedNode?.operation_id; @@ -40,7 +41,7 @@ export function AdmitCandidateDialog({ discoveryId, racks, initialRackId, curren if (!current && !prepared) { const result = await prepareCandidate({ ...access, preflight_operation_id: preparedNode?.operation_id }); credential.current = initialPassword || null; - setPrepared(result); setChoices(previous => result.accepted_choices ?? { ...previous, storage: discoveredStorage(result.inventory) }); + setPrepared(result); setChoices(previous => result.accepted_choices ?? result.policy_choices ?? { ...previous, storage: discoveredStorage(result.inventory) }); return; } if (current) await updateCandidate(access); @@ -66,7 +67,8 @@ export function AdmitCandidateDialog({ discoveryId, racks, initialRackId, curren {!current && !prepared &&

Verify access first to review actual disks. This step does not admit the node or initialize storage.

} {prepared && <>

Access verified · {prepared.accepted_choices ? 'resuming the previously accepted setup.' : 'node remains a candidate until you submit.'} Accepted setup continues after this dialog or browser closes.

{prepared.accepted_choices &&

Previously accepted choices are fixed. This submission resumes the same setup.

} - + {!prepared.accepted_choices && prepared.policy_choices &&

This node's startup settings fix these choices.

} + {!validNodeSetup(choices) &&

Resolve listener conflicts, storage capacity or selected-device agreement before joining.

} } {error &&

{error}

} diff --git a/app/crowdb-web/ui/src/shell/CandidateNodes.test.tsx b/app/crowdb-web/ui/src/shell/CandidateNodes.test.tsx new file mode 100644 index 000000000..9f40925cf --- /dev/null +++ b/app/crowdb-web/ui/src/shell/CandidateNodes.test.tsx @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +import { act, render, screen, waitFor } from '@testing-library/react'; +import { afterEach, expect, it, vi } from 'vitest'; +import { listNodeCandidates, nodeAdmissions, nodeDeploymentStatus } from '../api'; +import { CandidateNodes } from './CandidateNodes'; + +vi.mock('../api', () => ({ listNodeCandidates: vi.fn(), nodeAdmissions: vi.fn(), + nodeDeploymentStatus: vi.fn(), cancelNodeAdmission: vi.fn(), discardCandidateReview: vi.fn() })); + +afterEach(() => { vi.useRealTimers(); vi.clearAllMocks(); }); + +it('updates authority status without waiting for a hung admissions read', async () => { + const advertisement = { discovery_id: 'owned', protocol_version: 1, + monitor_endpoints: ['http://192.0.2.1:9095'], cluster_id: 'cluster' }; + vi.mocked(listNodeCandidates).mockResolvedValue({ local: { advertisement, + physical_host_id: 'host', rack_hint: null }, + candidates: [{ advertisement, state: 'same_cluster' }], diagnostics: [] }); + vi.mocked(nodeAdmissions).mockResolvedValue([]); + vi.mocked(nodeDeploymentStatus).mockResolvedValue({ phase: 'active', available: true }); + let rerender!: ReturnType['rerender']; + vi.useFakeTimers(); + await act(async () => { ({ rerender } = render()); }); + expect(screen.getByTestId('deployment-phase')).toHaveTextContent('Active cluster'); + let rejectAdmissions!: (error: Error) => void; + vi.mocked(nodeAdmissions).mockReturnValue(new Promise((_, reject) => { rejectAdmissions = reject; })); + await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + await act(async () => { rerender(); }); + vi.useRealTimers(); + await waitFor(() => expect(screen.getByTestId('deployment-phase')) + .toHaveTextContent('Group 0 unavailable · confirmed view is stale')); + expect(screen.getByTestId('candidate-owned')).toHaveTextContent('Cluster member'); + await act(async () => { rejectAdmissions(new Error('Group 0 unavailable')); }); + expect(screen.getByRole('status')).toHaveTextContent('last observations shown'); + expect(nodeDeploymentStatus).not.toHaveBeenCalled(); +}); diff --git a/app/crowdb-web/ui/src/shell/CandidateNodes.tsx b/app/crowdb-web/ui/src/shell/CandidateNodes.tsx index 511a2a39f..bb21f746e 100644 --- a/app/crowdb-web/ui/src/shell/CandidateNodes.tsx +++ b/app/crowdb-web/ui/src/shell/CandidateNodes.tsx @@ -3,7 +3,7 @@ import { useEffect, useRef, useState } from 'react'; import { createPortal } from 'react-dom'; -import { listNodeCandidates, nodeDeploymentStatus, nodeAdmissions, cancelNodeAdmission, discardCandidateReview, type NodeAdmission, type NodeCandidates, type NodeDeploymentStatus } from '../api'; +import { listNodeCandidates, nodeAdmissions, cancelNodeAdmission, discardCandidateReview, type NodeAdmission, type NodeCandidates, type NodeDeploymentStatus } from '../api'; import { AdmitCandidateDialog } from './AdmitCandidateDialog'; import { Button } from '../components/ui/Button'; import { NodeRecoveryDialog } from './NodeRecoveryDialog'; @@ -15,13 +15,12 @@ const labels = { }; /** Discovery observations remain separate from the confirmed hardware tree. */ -export function CandidateNodes({ racks, readonly, onChange, addNodeRequest }: { addNodeRequest?: { rackId: number; nonce: number }; racks: Rack[]; readonly?: boolean; onChange?: () => void }) { +export function CandidateNodes({ racks, readonly, onChange, addNodeRequest, deployment }: { deployment?: NodeDeploymentStatus; addNodeRequest?: { rackId: number; nonce: number }; racks: Rack[]; readonly?: boolean; onChange?: () => void }) { const section = useRef(null); const dialogRoot = section.current?.closest('.crowdb-console'); const [snapshot, setSnapshot] = useState(); const [actionError, setActionError] = useState(''); const [unavailable, setUnavailable] = useState(false); - const [deployment, setDeployment] = useState(); const [admissions, setAdmissions] = useState([]); const [recovery, setRecovery] = useState<'resume' | 'cleanup'>(); const [selected, setSelected] = useState(); @@ -35,19 +34,30 @@ export function CandidateNodes({ racks, readonly, onChange, addNodeRequest }: { }, [addNodeRequest]); useEffect(() => { const controller = new AbortController(); - let timer: ReturnType; - const refresh = async () => { - try { - const [result, deployment, admissions] = await Promise.all([listNodeCandidates({ signal: controller.signal }), nodeDeploymentStatus({ signal: controller.signal }), nodeAdmissions({ signal: controller.signal })]); - if (!controller.signal.aborted) { setSnapshot(result); setDeployment(deployment); setAdmissions(admissions); setUnavailable(false); } - } catch { - if (!controller.signal.aborted) setUnavailable(true); - } finally { - if (!controller.signal.aborted) timer = setTimeout(refresh, 2000); - } + const failed = [false, false]; + const poll = (index: number, request: () => Promise, update: (value: T) => void) => { + let timer: ReturnType; + const refresh = async () => { + try { + const value = await request(); + if (!controller.signal.aborted) { update(value); failed[index] = false; } + } catch { + if (!controller.signal.aborted) failed[index] = true; + } finally { + if (!controller.signal.aborted) { + setUnavailable(failed.some(Boolean)); + timer = setTimeout(refresh, 2000); + } + } + }; + void refresh(); + return () => clearTimeout(timer); }; - void refresh(); - return () => { controller.abort(); clearTimeout(timer); }; + const stop = [ + poll(0, () => listNodeCandidates({ signal: controller.signal }), setSnapshot), + poll(1, () => nodeAdmissions({ signal: controller.signal }), setAdmissions), + ]; + return () => { controller.abort(); stop.forEach(cancel => cancel()); }; }, []); return
diff --git a/app/crowdb-web/ui/src/shell/InitializationProgress.test.tsx b/app/crowdb-web/ui/src/shell/InitializationProgress.test.tsx index 57b2f7980..0b931f6d5 100644 --- a/app/crowdb-web/ui/src/shell/InitializationProgress.test.tsx +++ b/app/crowdb-web/ui/src/shell/InitializationProgress.test.tsx @@ -10,7 +10,7 @@ vi.mock('../api', () => ({ clusterInitialization: vi.fn(), initCluster: vi.fn() const observe = vi.mocked(clusterInitialization); describe('InitializationProgress', () => { - beforeEach(() => { vi.useFakeTimers(); vi.clearAllMocks(); }); + beforeEach(() => { vi.useFakeTimers(); vi.resetAllMocks(); }); afterEach(() => { vi.useRealTimers(); }); it('shows accepted work as running and retains the last observation on failure', async () => { @@ -21,7 +21,7 @@ describe('InitializationProgress', () => { expect(screen.getByRole('status')).toHaveTextContent('Preparing selected nodes'); expect(screen.getByText('Selected voters: 3, 1, 2')).toBeVisible(); expect(screen.queryByText('Initialization complete')).toBeNull(); - await act(async () => { await vi.advanceTimersByTimeAsync(2000); }); + await act(async () => { await vi.advanceTimersByTimeAsync(1000); }); expect(screen.getByText('Status unavailable · last observation shown')).toBeVisible(); expect(screen.getByText('Operation fixed')).toBeVisible(); view.unmount(); @@ -30,6 +30,17 @@ describe('InitializationProgress', () => { expect(initCluster).not.toHaveBeenCalled(); }); + it('shows completed initialization within one second while accepted work is running', async () => { + observe.mockResolvedValueOnce({ phase: 'data-group', members: [1, 2, 3], operation_id: 'fixed' }) + .mockResolvedValue({ phase: 'active', members: [1, 2, 3], operation_id: 'fixed' }); + render(); + await act(async () => {}); + expect(screen.getByRole('status')).toHaveTextContent('Creating initial data group'); + await act(async () => { await vi.advanceTimersByTimeAsync(1000); }); + expect(screen.getByRole('status')).toHaveTextContent('Initialization complete'); + expect(initCluster).not.toHaveBeenCalled(); + }); + it('retries only the accepted members without changing the data-group choice', async () => { observe.mockResolvedValueOnce({ phase: 'failed', members: [2, 3, 1], operation_id: 'fixed', error: 'peer unavailable' }) .mockResolvedValueOnce({ phase: 'preparing', members: [2, 3, 1], operation_id: 'fixed' }); diff --git a/app/crowdb-web/ui/src/shell/InitializationProgress.tsx b/app/crowdb-web/ui/src/shell/InitializationProgress.tsx index 90efa9c40..b896c2da1 100644 --- a/app/crowdb-web/ui/src/shell/InitializationProgress.tsx +++ b/app/crowdb-web/ui/src/shell/InitializationProgress.tsx @@ -22,13 +22,15 @@ export function InitializationProgress({ readonly = false }: { readonly?: boolea const controller = new AbortController(); let timer: ReturnType; const refresh = async () => { + let interval = 2000; try { const result = await clusterInitialization({ signal: controller.signal }); + if (!['idle', 'active', 'failed'].includes(result.phase)) interval = 500; if (!controller.signal.aborted) { setStatus(result); setUnavailable(false); } } catch { if (!controller.signal.aborted) setUnavailable(true); } finally { - if (!controller.signal.aborted) timer = setTimeout(refresh, 2000); + if (!controller.signal.aborted) timer = setTimeout(refresh, interval); } }; void refresh(); diff --git a/app/crowdb-web/ui/src/shell/Sidebar.tsx b/app/crowdb-web/ui/src/shell/Sidebar.tsx index bb5be621e..b257b0f39 100644 --- a/app/crowdb-web/ui/src/shell/Sidebar.tsx +++ b/app/crowdb-web/ui/src/shell/Sidebar.tsx @@ -11,7 +11,7 @@ import { crowdbKvServerByNodeId } from '../data/crowdbKvServers'; import { DEFAULT_DC_ID, DEFAULT_DC_NAME } from '../data/defaultDatacenter'; import { groupLabel, localReplicaLabel, nodeLabel, rackLabel, storeLabel, toUiHealth, toUiReplicaRole, toUiRole } from '../utils/entityDisplay'; import type { NodeDiskGroups } from '../data/useCapacityTree'; -import type { ServerSummary } from '../api'; +import type { NodeDeploymentStatus, ServerSummary } from '../api'; import { isAuxiliaryKind, serviceInstanceLabel } from '../services/client'; import { serviceOrder, type NodeServicePlan, type ServiceKind } from '../services/useNodeServicePlans'; import { domainTabs } from './domainTabs'; @@ -36,6 +36,7 @@ interface SidebarProps { candidateSummaries?: Record; onCandidatesChanged?: () => void; nodeManagement?: boolean; + nodeAuthority?: NodeDeploymentStatus; allServers?: ServerSummary[]; racks?: Rack[]; servers?: CrowdbKVServerView[]; @@ -69,6 +70,7 @@ export function Sidebar({ candidateSummaries = {}, onCandidatesChanged, nodeManagement = false, + nodeAuthority, allServers = [], racks = [], servers = [], @@ -310,7 +312,7 @@ export function Sidebar({ )} )} - {nodeManagement && domain === Domain.Cluster && } + {nodeManagement && domain === Domain.Cluster && } ); } diff --git a/app/crowdb-web/ui/src/shell/nodeSetup.ts b/app/crowdb-web/ui/src/shell/nodeSetup.ts index d3b179e80..631816fa7 100644 --- a/app/crowdb-web/ui/src/shell/nodeSetup.ts +++ b/app/crowdb-web/ui/src/shell/nodeSetup.ts @@ -13,7 +13,7 @@ export interface SetupMedium { path: string; backend: DiskBackend; backing_ident export type SetupStorage = { mode: 'automatic'; capacity_bytes: number; reserve_bytes: number; sparse: boolean } | { mode: 'diskless' } | { mode: 'supplied'; disks: SetupMedium[] }; export interface NodeSetupChoices { storage: SetupStorage; services: SetupService[] } -export interface PreparedNodeAccess { operation_id: string; inventory: DiskInventory; accepted_choices?: NodeSetupChoices | null } +export interface PreparedNodeAccess { operation_id: string; inventory: DiskInventory; accepted_choices?: NodeSetupChoices | null; policy_choices?: NodeSetupChoices | null } export interface NodeAdmissionRequest { discovery_id: string; rack_id: number; ssh_user: string; ssh_port: number; ssh_password: string | null; preflight_operation_id?: string; setup?: NodeSetupChoices } export type SetupStepPhase = 'pending' | 'waiting' | 'preparing' | 'accepted' | 'ready' | 'failed' | 'warning' | 'disabled'; diff --git a/container/crowdb-monitor/Cargo.toml b/container/crowdb-monitor/Cargo.toml index 2b2026061..d2b6b6e53 100644 --- a/container/crowdb-monitor/Cargo.toml +++ b/container/crowdb-monitor/Cargo.toml @@ -16,7 +16,8 @@ test-util = [] [dependencies] crowdb-console-shared = { path = "../../lib/crowdb-console-shared" } -clap = { version = "4", features = ["derive"] } +clap = { version = "4", features = ["derive", "env"] } +crowdb-chunk-kv-client = { path = "../../lib/crowdb-chunk-kv-client" } crowdb-kv-client = { path = "../../lib/crowdb-kv-client" } crowdb-diskio-client = { path = "../../lib/crowdb-diskio-client" } crowdb-protocol = { path = "../../lib/crowdb-protocol" } @@ -30,6 +31,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" thiserror.workspace = true +tracing.workspace = true tokio = { workspace = true, features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } toml = "0.8" uuid = { version = "1", features = ["v4", "v7", "serde"] } @@ -39,6 +41,7 @@ ssh-key = "0.6" if-addrs = "0.13" [dev-dependencies] +ssh-key = { version = "0.6", features = ["ed25519"] } crowdb-kv = { path = "../../lib/crowdb-kv" } crowdb-monitor = { path = ".", features = ["test-util"] } crowdb-rpc-ffi = { path = "../../lib/crowdb-rpc/ffi", features = ["test-util"] } diff --git a/container/crowdb-monitor/src/bootstrap/kv.rs b/container/crowdb-monitor/src/bootstrap/kv.rs index 273b4ac58..81bfac413 100644 --- a/container/crowdb-monitor/src/bootstrap/kv.rs +++ b/container/crowdb-monitor/src/bootstrap/kv.rs @@ -188,6 +188,7 @@ impl KvBootstrap { replica_id: group.replica_id, initial_role: None, start_election: Some(true), + deployment: None, }) .send() .await diff --git a/container/crowdb-monitor/src/bootstrap/s3.rs b/container/crowdb-monitor/src/bootstrap/s3.rs index 08cb2eea0..f2380d67f 100644 --- a/container/crowdb-monitor/src/bootstrap/s3.rs +++ b/container/crowdb-monitor/src/bootstrap/s3.rs @@ -116,6 +116,7 @@ impl S3Bootstrap { let (access_key_id, secret_access_key) = parse_token(&output.stdout)?; let client = ClientCredentials { s3_endpoint, + dataset_endpoint: service.env.get("CROWDB_DATASET_PUBLIC_URI").cloned(), iceberg_endpoint, region: service .env diff --git a/container/crowdb-monitor/src/credentials.rs b/container/crowdb-monitor/src/credentials.rs index d35cea1f1..1478335be 100644 --- a/container/crowdb-monitor/src/credentials.rs +++ b/container/crowdb-monitor/src/credentials.rs @@ -1,3 +1,6 @@ +mod client; +pub use client::{show_client_credentials, ClientCredentials}; + use std::fs::{self, File, OpenOptions}; use std::io::Write; use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}; @@ -133,12 +136,17 @@ impl ServerCredentials { &self.iceberg_manage_token } + #[must_use] + pub fn iceberg_read_token(&self) -> &str { + &self.iceberg_read_token + } + /// # Errors /// Requires an existing client file to match the authoritative user and endpoints. pub fn verify_client(&self, client: &ClientCredentials) -> Result<(), CredentialError> { client.validate()?; let existing = read_private(&self.directory.join(CLIENT_FILE))?; - if existing != client.env(&self.iceberg_write_token) { + if existing != client.env(&self.iceberg_write_token, &self.dataset_token()) { return Err(CredentialError::Invalid("existing client credentials conflict")); } Ok(()) @@ -151,12 +159,17 @@ impl ServerCredentials { let path = self.directory.join(CLIENT_FILE); if fs::symlink_metadata(&path).is_ok() { let existing = read_private(&path)?; - if existing == client.env(&self.iceberg_write_token) { + if existing == client.env(&self.iceberg_write_token, &self.dataset_token()) { return Ok(()); } return Err(CredentialError::Invalid("existing client credentials conflict")); } - atomic_private_write(&path, client.env(&self.iceberg_write_token).as_bytes()) + atomic_private_write( + &path, + client + .env(&self.iceberg_write_token, &self.dataset_token()) + .as_bytes(), + ) } fn parse(directory: PathBuf, body: &str) -> Result { @@ -195,106 +208,6 @@ impl ServerCredentials { } } -pub struct ClientCredentials { - pub s3_endpoint: String, - pub iceberg_endpoint: String, - pub region: String, - pub access_key_id: String, - pub secret_access_key: String, -} - -impl ClientCredentials { - fn validate(&self) -> Result<(), CredentialError> { - if !has_http_authority(&self.s3_endpoint) - || !has_http_authority(&self.iceberg_endpoint) - || ![ - &self.s3_endpoint, - &self.iceberg_endpoint, - &self.region, - &self.access_key_id, - &self.secret_access_key, - ] - .iter() - .all(|value| { - !value.is_empty() - && value - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || b"-._~:/".contains(&byte)) - }) - { - return Err(CredentialError::Invalid("client credential value is invalid")); - } - Ok(()) - } - - fn env(&self, writer_token: &str) -> String { - format!( - "AWS_ENDPOINT_URL={}\nAWS_DEFAULT_REGION={}\nAWS_ACCESS_KEY_ID={}\nAWS_SECRET_ACCESS_KEY={}\nICEBERG_URI={}\nICEBERG_TOKEN={}\n", - self.s3_endpoint, - self.region, - self.access_key_id, - self.secret_access_key, - self.iceberg_endpoint, - writer_token, - ) - } -} - -fn has_http_authority(value: &str) -> bool { - value - .strip_prefix("http://") - .or_else(|| value.strip_prefix("https://")) - .is_some_and(|authority| !authority.is_empty() && !authority.starts_with('/')) -} - -/// # Errors -/// Rejects missing, symlinked, malformed, or publicly readable client files. -pub fn show_client_credentials(data_root: &Path) -> Result { - let directory = data_root.join("secrets"); - let metadata = fs::symlink_metadata(&directory)?; - if !metadata.file_type().is_dir() || metadata.permissions().mode() & 0o777 != 0o700 { - return Err(CredentialError::Invalid("secrets directory must have mode 0700")); - } - let body = read_private(&directory.join(CLIENT_FILE))?; - validate_client_env(&body)?; - Ok(body) -} - -fn validate_client_env(body: &str) -> Result<(), CredentialError> { - let mut values = body.lines(); - let endpoint = take_value(&mut values, "AWS_ENDPOINT_URL")?; - let region = take_value(&mut values, "AWS_DEFAULT_REGION")?; - let access_key = take_value(&mut values, "AWS_ACCESS_KEY_ID")?; - let secret_key = take_value(&mut values, "AWS_SECRET_ACCESS_KEY")?; - let iceberg_uri = take_value(&mut values, "ICEBERG_URI")?; - let writer_token = take_value(&mut values, "ICEBERG_TOKEN")?; - if values.next().is_some() { - return Err(CredentialError::Invalid( - "client credentials contain extra values", - )); - } - ClientCredentials { - s3_endpoint: endpoint.into(), - iceberg_endpoint: iceberg_uri.into(), - region: region.into(), - access_key_id: access_key.into(), - secret_access_key: secret_key.into(), - } - .validate()?; - if writer_token.len() != 64 || !writer_token.bytes().all(|byte| byte.is_ascii_hexdigit()) { - return Err(CredentialError::Invalid("Iceberg writer token is invalid")); - } - Ok(()) -} - -fn take_value<'a>(lines: &mut impl Iterator, key: &str) -> Result<&'a str, CredentialError> { - lines - .next() - .and_then(|line| line.strip_prefix(key)) - .and_then(|line| line.strip_prefix('=')) - .ok_or(CredentialError::Invalid("credential file is incomplete")) -} - fn take_hex<'a>(lines: &mut impl Iterator, key: &str) -> Result { let value = lines .next() diff --git a/container/crowdb-monitor/src/credentials/client.rs b/container/crowdb-monitor/src/credentials/client.rs new file mode 100644 index 000000000..7272f13f6 --- /dev/null +++ b/container/crowdb-monitor/src/credentials/client.rs @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{read_private, CredentialError, CLIENT_FILE}; +use std::{fs, os::unix::fs::PermissionsExt, path::Path}; + +pub struct ClientCredentials { + pub s3_endpoint: String, + pub iceberg_endpoint: String, + pub dataset_endpoint: Option, + pub region: String, + pub access_key_id: String, + pub secret_access_key: String, +} + +impl ClientCredentials { + pub(super) fn validate(&self) -> Result<(), CredentialError> { + crowdb_protocol::mgmt::node::deployment::PublicAccessUris { + s3: Some(self.s3_endpoint.clone()), + iceberg: Some(self.iceberg_endpoint.clone()), + dataset: self.dataset_endpoint.clone(), + } + .validate() + .map_err(CredentialError::Invalid)?; + if ![&self.region, &self.access_key_id, &self.secret_access_key] + .iter() + .all(|value| { + !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-._~:/".contains(&byte)) + }) + { + return Err(CredentialError::Invalid("client credential value is invalid")); + } + Ok(()) + } + + pub(super) fn env(&self, writer_token: &str, dataset_token: &str) -> String { + let mut body = format!( + "AWS_ENDPOINT_URL={}\nAWS_DEFAULT_REGION={}\nAWS_ACCESS_KEY_ID={}\nAWS_SECRET_ACCESS_KEY={}\nICEBERG_URI={}\nICEBERG_TOKEN={}\n", + self.s3_endpoint, + self.region, + self.access_key_id, + self.secret_access_key, + self.iceberg_endpoint, + writer_token, + ); + if let Some(endpoint) = &self.dataset_endpoint { + use std::fmt::Write as _; + writeln!(body, "DATASET_URI={endpoint}\nDATASET_TOKEN={dataset_token}").expect("String write"); + } + body + } +} + +/// # Errors +/// Rejects missing, symlinked, malformed, or publicly readable client files. +pub fn show_client_credentials(data_root: &Path) -> Result { + let directory = data_root.join("secrets"); + let metadata = fs::symlink_metadata(&directory)?; + if !metadata.file_type().is_dir() || metadata.permissions().mode() & 0o777 != 0o700 { + return Err(CredentialError::Invalid("secrets directory must have mode 0700")); + } + let body = read_private(&directory.join(CLIENT_FILE))?; + validate_client_env(&body)?; + Ok(body) +} + +fn validate_client_env(body: &str) -> Result<(), CredentialError> { + let mut values = body.lines(); + let endpoint = take_value(&mut values, "AWS_ENDPOINT_URL")?; + let region = take_value(&mut values, "AWS_DEFAULT_REGION")?; + let access_key = take_value(&mut values, "AWS_ACCESS_KEY_ID")?; + let secret_key = take_value(&mut values, "AWS_SECRET_ACCESS_KEY")?; + let iceberg_uri = take_value(&mut values, "ICEBERG_URI")?; + let writer_token = take_value(&mut values, "ICEBERG_TOKEN")?; + let dataset_endpoint = if let Some(line) = values.next() { + let endpoint = line + .strip_prefix("DATASET_URI=") + .ok_or(CredentialError::Invalid("Dataset URI is missing"))?; + let token = take_value(&mut values, "DATASET_TOKEN")?; + validate_token(token)?; + Some(endpoint.to_owned()) + } else { + None + }; + if values.next().is_some() { + return Err(CredentialError::Invalid( + "client credentials contain extra values", + )); + } + ClientCredentials { + s3_endpoint: endpoint.into(), + iceberg_endpoint: iceberg_uri.into(), + dataset_endpoint, + region: region.into(), + access_key_id: access_key.into(), + secret_access_key: secret_key.into(), + } + .validate()?; + validate_token(writer_token) +} + +fn validate_token(token: &str) -> Result<(), CredentialError> { + if token.len() != 64 || !token.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(CredentialError::Invalid("client bearer token is invalid")); + } + Ok(()) +} + +fn take_value<'a>(lines: &mut impl Iterator, key: &str) -> Result<&'a str, CredentialError> { + lines + .next() + .and_then(|line| line.strip_prefix(key)) + .and_then(|line| line.strip_prefix('=')) + .ok_or(CredentialError::Invalid("credential file is incomplete")) +} diff --git a/container/crowdb-monitor/src/lib.rs b/container/crowdb-monitor/src/lib.rs index fad501f75..e4f72233e 100644 --- a/container/crowdb-monitor/src/lib.rs +++ b/container/crowdb-monitor/src/lib.rs @@ -9,6 +9,7 @@ mod liveness; mod manifest; mod monitor_log; mod node; +#[cfg(feature = "test-util")] mod preview; mod probe; mod process; @@ -30,18 +31,23 @@ pub use liveness::{probe_liveness, LivenessError, LivenessServer}; pub use manifest::{BootstrapManifest, BootstrapSession, ManifestError, ManifestState}; pub use monitor_log::{MonitorEvent, MonitorEventKind, MonitorLog, MonitorLogError}; #[cfg(feature = "test-util")] +pub use node::unique_member_endpoints_for_tests; +#[cfg(feature = "test-util")] pub use node::{ - accept_file_for_tests, accept_startup_mode_for_tests, check_retained_medium_for_tests, - file_operations_for_tests, prepare_file_for_tests, prepare_supplied_disk_for_tests, - publish_file_for_tests, verify_blank_medium_for_tests, InitializationForTests, NodeServiceForTests, - NodeSetupForTests, SuppliedOperationsForTests, + accept_file_for_tests, accept_member_trust_for_tests, accept_startup_mode_for_tests, + check_member_trust_for_tests, check_retained_medium_for_tests, file_operations_for_tests, + member_trust_ready_for_tests, node_health_for_tests, prepare_file_for_tests, + prepare_supplied_disk_for_tests, publish_file_for_tests, resume_file_operations_for_tests, + runtime_observation_for_tests, verify_blank_medium_for_tests, InitializationForTests, + NodeServiceForTests, NodeSetupForTests, SuppliedOperationsForTests, }; pub use node::{ - control_node, discover_disks, run_node, run_single_node, serve_node_management, CandidateCache, - DiscoveryConfig, DiscoveryError, NodeDiscovery, NodeIdentity, NodeIdentityError, NodeManagementError, - NodeRuntimeConfig, NODE_SERVICE_TYPE, + accept_node_policy, control_node, discover_disks, read_node_policy, run_node, run_node_with_inputs, + serve_node_management, CandidateCache, DiscoveryConfig, DiscoveryError, NodeDiscovery, NodeIdentity, + NodeIdentityError, NodeManagementError, NodeRuntimeConfig, NODE_SERVICE_TYPE, }; -pub use preview::{preview_step_names, run_preview, PreviewError}; +#[cfg(feature = "test-util")] +pub use preview::{preview_step_names_for_tests, run_preview_for_tests, PreviewError}; pub use probe::{ProbeError, ProbeExecutor}; pub use process::{ProcessError, ProcessManager}; pub use profile::{ @@ -51,3 +57,17 @@ pub use profile::{ pub use render::{render_configs, RenderError, RenderedConfig}; pub use status::{MonitorPhase, MonitorStatus, ServiceStatus, StatusError, StatusStore}; pub use supervisor::{Supervisor, SupervisorError}; + +#[cfg(feature = "test-util")] +pub use node::rebind_recovery_services_for_tests; + +pub use node::read_recovery_identity; + +#[cfg(feature = "test-util")] +pub use node::cleanup_node_for_tests; + +#[cfg(feature = "test-util")] +pub use node::capture_recovery_owners_for_tests; + +#[cfg(feature = "test-util")] +pub use node::NodeDispatchForTests; diff --git a/container/crowdb-monitor/src/main.rs b/container/crowdb-monitor/src/main.rs index ca648fe3c..ca27814eb 100644 --- a/container/crowdb-monitor/src/main.rs +++ b/container/crowdb-monitor/src/main.rs @@ -1,6 +1,8 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +mod node_args; + use std::path::PathBuf; use std::time::Duration; @@ -32,18 +34,19 @@ enum Command { physical_host_id: String, }, Run { - #[arg(long, default_value = "/opt/crowdb/etc/profile.toml")] - profile: PathBuf, - #[arg(long, default_value = "single", value_parser = ["single", "manual"])] - mode: String, - #[arg(long, default_value = "eth0")] - interface: Vec, - #[arg(long)] - physical_host_id: Option, + #[command(flatten)] + args: node_args::NodeRunArgs, + }, + /// Seal startup policy before the container creates its SSH identity. + AcceptPolicy { + #[command(flatten)] + args: node_args::NodeRunArgs, }, Control { #[arg(long, default_value = "/opt/crowdb/run/node-control.sock")] socket: PathBuf, + #[arg(long, default_value = "/opt/crowdb/data")] + data_root: PathBuf, #[arg(long)] json: String, }, @@ -128,53 +131,31 @@ async fn run(cli: Cli) -> Result<(), Box> { }) .await?; } - Command::Run { - profile, - mode, - interface, - physical_host_id, + Command::Run { args } => node_args::run(args).await?, + Command::AcceptPolicy { args } => println!("{}", node_args::accept(&args)?), + Command::Control { + socket, + data_root, + json, } => { - let addresses = if_addrs::get_if_addrs()? - .into_iter() - .filter(|address| interface.contains(&address.name)) - .map(|address| address.ip()) - .collect(); - let config = crowdb_monitor::NodeRuntimeConfig { - profile, - bind: "0.0.0.0:9095".parse()?, - discovery: crowdb_monitor::DiscoveryConfig { - interfaces: interface, - addresses, - monitor_port: 9095, - cluster_id: None, - seeds: std::env::var("CROWDB_DISCOVERY_SEEDS") - .unwrap_or_default() - .split(',') - .filter(|seed| !seed.is_empty()) - .map(str::to_owned) - .collect(), - }, - physical_host_id: match physical_host_id { - Some(id) => id, - None if mode == "single" => "unspecified-single-node-host".into(), - None => return Err("manual mode requires physical-host-id".into()), - }, - }; - let result = if mode == "single" { - crowdb_monitor::run_single_node(config).await - } else { - crowdb_monitor::run_node(config).await - }; - result.map_err(|error| error.to_string())?; - } - Command::Control { socket, json } => { if json.len() > 65536 { return Err("control request exceeds limit".into()); } let request = serde_json::from_str(&json)?; - let reply = crowdb_monitor::control_node(&socket, &request) - .await - .map_err(|error| error.to_string())?; + let reply = if matches!( + request, + crowdb_protocol::mgmt::node::NodeControl::RecoveryIdentity + ) { + serde_json::to_value( + crowdb_monitor::read_recovery_identity(&data_root) + .await + .map_err(|error| error.to_string())?, + )? + } else { + crowdb_monitor::control_node(&socket, &request) + .await + .map_err(|error| error.to_string())? + }; println!("{}", serde_json::to_string(&reply)?); } Command::Validate { profile } => { @@ -193,7 +174,12 @@ async fn run(cli: Cli) -> Result<(), Box> { data_root, }, } => { - print!("{}", show_client_credentials(&data_root)?); + let credential_root = if crowdb_monitor::read_node_policy(&data_root)?.is_some() { + data_root.join("console") + } else { + data_root + }; + print!("{}", show_client_credentials(&credential_root)?); } } Ok(()) @@ -201,9 +187,7 @@ async fn run(cli: Cli) -> Result<(), Box> { fn validate_profile(path: PathBuf) -> Result<(), Box> { let profile = DeploymentProfile::load(path)?; - if profile.name == "single-node-container" { - crowdb_monitor::preview_step_names(&profile)?; - } + profile.validate_node_catalog()?; println!("{}", profile.name); Ok(()) } diff --git a/container/crowdb-monitor/src/node.rs b/container/crowdb-monitor/src/node.rs index 8f09303e7..44bdd6239 100644 --- a/container/crowdb-monitor/src/node.rs +++ b/container/crowdb-monitor/src/node.rs @@ -9,7 +9,6 @@ mod durable; mod identity; mod management; mod runtime; -mod single; pub(crate) mod startup; mod storage; #[cfg(feature = "test-util")] @@ -24,19 +23,29 @@ pub use storage::retained::check_retained_medium_for_tests; #[cfg(feature = "test-util")] pub use storage::supplied::SuppliedOperationsForTests; mod trust; -pub use single::run_single_node; +#[cfg(feature = "test-util")] +pub use runtime::node_health_for_tests; #[cfg(feature = "test-util")] pub use runtime::prepare_supplied_disk_for_tests; #[cfg(feature = "test-util")] -pub use runtime::publish_file_for_tests; +pub use runtime::runtime_observation_for_tests; +#[cfg(feature = "test-util")] +pub use runtime::unique_member_endpoints_for_tests; #[cfg(feature = "test-util")] pub use runtime::InitializationForTests; #[cfg(feature = "test-util")] pub use runtime::NodeSetupForTests; -pub use runtime::{control_node, run_node, NodeRuntimeConfig}; +#[cfg(feature = "test-util")] +pub use runtime::{ + accept_member_trust_for_tests, check_member_trust_for_tests, member_trust_ready_for_tests, +}; +pub use runtime::{control_node, run_node, run_node_with_inputs, NodeRuntimeConfig}; +#[cfg(feature = "test-util")] +pub use runtime::{publish_file_for_tests, resume_file_operations_for_tests}; #[cfg(feature = "test-util")] pub use startup::accept_startup_mode_for_tests; +pub use startup::{accept_node_policy, read_node_policy}; pub use management::{serve_node_management, NodeManagementError}; @@ -45,6 +54,7 @@ pub use discovery::{DiscoveryConfig, DiscoveryError, NodeDiscovery, NODE_SERVICE pub use identity::{NodeIdentity, NodeIdentityError}; +#[cfg(feature = "test-util")] pub(crate) fn write_single_binding( root: &std::path::Path, binding: &crowdb_protocol::mgmt::node::NodeBinding, @@ -54,3 +64,17 @@ pub(crate) fn write_single_binding( #[cfg(feature = "test-util")] pub use runtime::NodeServiceForTests; + +#[cfg(feature = "test-util")] +pub use runtime::rebind_recovery_services_for_tests; + +pub use runtime::read_recovery_identity; + +#[cfg(feature = "test-util")] +pub use runtime::cleanup_node_for_tests; + +#[cfg(feature = "test-util")] +pub use runtime::capture_recovery_owners_for_tests; + +#[cfg(feature = "test-util")] +pub use runtime::NodeDispatchForTests; diff --git a/container/crowdb-monitor/src/node/candidates.rs b/container/crowdb-monitor/src/node/candidates.rs index 987cc0e57..300fdf764 100644 --- a/container/crowdb-monitor/src/node/candidates.rs +++ b/container/crowdb-monitor/src/node/candidates.rs @@ -25,6 +25,10 @@ impl CandidateCache { .cluster_id .as_ref() .is_some_and(|id| Uuid::parse_str(id).is_err()) + || advertisement + .formation + .as_ref() + .is_some_and(|policy| policy.validate().is_err()) || advertisement.monitor_endpoints.is_empty() || advertisement.monitor_endpoints.len() > MAX_ENDPOINTS || advertisement @@ -61,7 +65,9 @@ impl CandidateCache { let conflict = local_identity || observations.iter().any(|observation| *observation != first); let state = if conflict { CandidateState::IdentityConflict - } else if first.protocol_version != NODE_PROTOCOL_VERSION { + } else if first.protocol_version != NODE_PROTOCOL_VERSION + || (local.formation.is_some() && first.formation != local.formation) + { CandidateState::Incompatible } else { match (&first.cluster_id, &local.cluster_id) { diff --git a/container/crowdb-monitor/src/node/discovery.rs b/container/crowdb-monitor/src/node/discovery.rs index 4cbe40129..a13e8a281 100644 --- a/container/crowdb-monitor/src/node/discovery.rs +++ b/container/crowdb-monitor/src/node/discovery.rs @@ -3,6 +3,7 @@ use std::net::{IpAddr, SocketAddr}; +use crowdb_protocol::mgmt::node::deployment::{DeploymentMode, FormationPolicy}; use crowdb_protocol::mgmt::node::{CandidateNode, NodeAdvertisement, NODE_PROTOCOL_VERSION}; use mdns_sd::{DaemonEvent, IfKind, Receiver, ServiceDaemon, ServiceEvent, ServiceInfo}; use thiserror::Error; @@ -58,6 +59,18 @@ impl NodeDiscovery { if cluster == self.local.cluster_id { return Ok(()); } + self.advertise(cluster.as_deref())?; + self.local.cluster_id = cluster; + Ok(()) + } + + pub(super) fn set_formation(&mut self, policy: FormationPolicy) -> Result<(), DiscoveryError> { + policy.validate().map_err(DiscoveryError::Invalid)?; + self.local.formation = Some(policy); + self.advertise(self.local.cluster_id.as_deref()) + } + + fn advertise(&self, cluster: Option<&str>) -> Result<(), DiscoveryError> { let instance = self .fullname .strip_suffix(NODE_SERVICE_TYPE) @@ -67,8 +80,23 @@ impl NodeDiscovery { ("id".to_owned(), self.local.discovery_id.clone()), ("version".to_owned(), self.local.protocol_version.to_string()), ]; - if let Some(cluster) = &cluster { - properties.push(("cluster".to_owned(), cluster.clone())); + if let Some(cluster) = cluster { + properties.push(("cluster".to_owned(), cluster.to_owned())); + } + if let Some(policy) = &self.local.formation { + properties.extend([ + ("scope".into(), policy.cluster.clone()), + ("initial-voters".into(), policy.initial_voters.to_string()), + ("policy-version".into(), policy.version.to_string()), + ( + "deployment-mode".into(), + match policy.mode { + DeploymentMode::Local => "local", + DeploymentMode::Production => "production", + } + .into(), + ), + ]); } let mut addresses = Vec::new(); let mut port = 0; @@ -93,7 +121,6 @@ impl NodeDiscovery { port, properties.as_slice(), )?)?; - self.local.cluster_id = cluster; Ok(()) } /// # Errors @@ -174,6 +201,7 @@ impl NodeDiscovery { protocol_version: NODE_PROTOCOL_VERSION, monitor_endpoints: endpoints, cluster_id: config.cluster_id.map(|id| id.to_string()), + formation: None, }, cache: CandidateCache::default(), }) @@ -218,6 +246,21 @@ impl NodeDiscovery { }) .collect(), cluster_id: service.get_property_val_str("cluster").map(str::to_owned), + formation: (|| { + Some(FormationPolicy { + version: service.get_property_val_str("policy-version")?.parse().ok()?, + mode: match service.get_property_val_str("deployment-mode")? { + "local" => DeploymentMode::Local, + "production" => DeploymentMode::Production, + _ => return None, + }, + cluster: service.get_property_val_str("scope")?.into(), + initial_voters: service + .get_property_val_str("initial-voters")? + .parse() + .ok()?, + }) + })(), }; self.cache .observe(service.get_fullname().to_owned(), advertisement); diff --git a/container/crowdb-monitor/src/node/identity.rs b/container/crowdb-monitor/src/node/identity.rs index 2c293603e..ebd2b50f0 100644 --- a/container/crowdb-monitor/src/node/identity.rs +++ b/container/crowdb-monitor/src/node/identity.rs @@ -30,6 +30,14 @@ impl NodeIdentity { self.0 } + /// Read the existing identity without creating state or accepting a new node. + /// + /// # Errors + /// Rejects absent, corrupt, exposed or symlinked identity records. + pub fn load_existing(root: &Path) -> Result { + Self::load(&root.join(IDENTITY_FILE)) + } + /// # Errors /// Rejects non-directory roots and corrupt or symlinked identity files. /// Concurrent initialization publishes one complete, durable identity. diff --git a/container/crowdb-monitor/src/node/management.rs b/container/crowdb-monitor/src/node/management.rs index 76d029c1c..dc936db7e 100644 --- a/container/crowdb-monitor/src/node/management.rs +++ b/container/crowdb-monitor/src/node/management.rs @@ -23,6 +23,10 @@ pub enum NodeManagementError { Io(#[from] std::io::Error), #[error("physical host identity must be explicitly configured")] PhysicalHost, + #[error( + "discovery seeds require at most 64 HTTP origins without credentials, paths, queries or fragments" + )] + DiscoverySeeds, } #[derive(Clone)] @@ -45,6 +49,8 @@ pub async fn serve_node_management( if physical_host_id.is_empty() || physical_host_id.len() > 256 { return Err(NodeManagementError::PhysicalHost); } + DiscoveryConfig::validate_seed_origins(&config.seeds)?; + let formation = super::read_node_policy(root)?.map(|policy| policy.formation()); let identity = NodeIdentity::load_or_create(root)?; let listener = tokio::net::TcpListener::bind(bind).await?; if listener.local_addr()?.port() != config.monitor_port { @@ -54,8 +60,10 @@ pub async fn serve_node_management( ) .into()); } - validate_seeds(&config.seeds)?; let mut discovery = NodeDiscovery::start(identity, config)?; + if let Some(formation) = formation { + discovery.set_formation(formation)?; + } let seed_client = reqwest::Client::builder() .no_proxy() .redirect(reqwest::redirect::Policy::none()) @@ -136,24 +144,30 @@ pub async fn serve_node_management( Ok(()) } -fn validate_seeds(seeds: &[String]) -> Result<(), NodeManagementError> { - if seeds.len() > 64 { - return Err(NodeManagementError::PhysicalHost); - } - for seed in seeds { - let url = reqwest::Url::parse(seed).map_err(|_| NodeManagementError::PhysicalHost)?; - if url.scheme() != "http" - || url.host_str().is_none() - || !url.username().is_empty() - || url.password().is_some() - || url.path() != "/" - || url.query().is_some() - || url.fragment().is_some() - { - return Err(NodeManagementError::PhysicalHost); +impl DiscoveryConfig { + /// Validate startup origins before accepting policy or creating node resources. + /// + /// # Errors + /// Rejects invalid/credential-bearing origins without exposing their values. + pub fn validate_seed_origins(seeds: &[String]) -> Result<(), NodeManagementError> { + if seeds.len() > 64 { + return Err(NodeManagementError::DiscoverySeeds); } + for seed in seeds { + let url = reqwest::Url::parse(seed).map_err(|_| NodeManagementError::DiscoverySeeds)?; + if url.scheme() != "http" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.path() != "/" + || url.query().is_some() + || url.fragment().is_some() + { + return Err(NodeManagementError::DiscoverySeeds); + } + } + Ok(()) } - Ok(()) } fn local_handshake(root: &Path, discovery: &NodeDiscovery, physical_host_id: String) -> NodeHandshake { diff --git a/container/crowdb-monitor/src/node/runtime.rs b/container/crowdb-monitor/src/node/runtime.rs index 2331ffaf0..11148c244 100644 --- a/container/crowdb-monitor/src/node/runtime.rs +++ b/container/crowdb-monitor/src/node/runtime.rs @@ -7,6 +7,7 @@ use std::collections::BTreeMap; use std::path::PathBuf; use std::time::Duration; +use crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides; use crowdb_protocol::mgmt::node::{NodeBinding, NodeControl}; use crowdb_protocol::mgmt::SystemBootstrapIdentity; use serde::{Deserialize, Serialize}; @@ -18,10 +19,17 @@ use crate::{DeploymentProfile, ProcessManager, ServiceProfile}; mod cleanup; mod control; +mod dispatch; +#[cfg(feature = "test-util")] +pub use dispatch::NodeDispatch as NodeDispatchForTests; mod health; +#[cfg(feature = "test-util")] +pub use health::node_health_for_tests; mod ipc; mod kv; mod observation; +#[cfg(feature = "test-util")] +pub use observation::runtime_observation_for_tests; mod probes; mod services; use kv::start_kv; @@ -32,6 +40,8 @@ mod file_admission; #[cfg(feature = "test-util")] pub use file_admission::publish_file_for_tests; mod file_preparation; +#[cfg(feature = "test-util")] +pub use file_preparation::resume_file_operations_for_tests; mod initialization; mod retained_storage; mod setup; @@ -44,8 +54,15 @@ pub use setup::NodeSetupForTests; #[cfg(feature = "test-util")] pub use supplied_storage::prepare_supplied_disk_for_tests; mod preparation; +mod recovery; pub use ipc::control_node; use ipc::read_request; +#[cfg(feature = "test-util")] +pub use recovery::unique_member_endpoints_for_tests; +#[cfg(feature = "test-util")] +pub use recovery::{ + accept_member_trust_for_tests, check_member_trust_for_tests, member_trust_ready_for_tests, +}; type Result = std::result::Result>; @@ -72,9 +89,34 @@ pub(super) struct AcceptedNode { /// # Errors /// Rejects conflicting persistent state and failed monitor/service startup. pub async fn run_node(config: NodeRuntimeConfig) -> Result<()> { + run_node_with_inputs( + config, + &NodeDeploymentOverrides::default(), + &NodeDeploymentOverrides::default(), + ) + .await +} + +/// Start a node with explicit/file inputs sealed before node effects. +/// +/// # Errors +/// Rejects incompatible policy, concurrent root use and runtime startup failures. +pub async fn run_node_with_inputs( + config: NodeRuntimeConfig, + file: &NodeDeploymentOverrides, + explicit: &NodeDeploymentOverrides, +) -> Result<()> { + if config.physical_host_id.is_empty() || config.physical_host_id.len() > 256 { + return Err("node requires a bounded physical host identity".into()); + } + DiscoveryConfig::validate_seed_origins(&config.discovery.seeds)?; let profile = kv::node_profile(&config)?; let root = profile.paths.data_root.clone(); - super::startup::accept(&root, super::startup::Mode::Standard)?; + super::startup::accept_node_policy(&root, file, explicit)?; + validate_retained_startup(&profile)?; + // Reuse the existing socket ownership guard on the shared durable root, + // independently of each container's private runtime directory. + let _root_liveness = crate::LivenessServer::start(&root)?; NodeIdentity::load_or_create(&root)?; std::fs::create_dir_all(&profile.paths.run_root)?; std::fs::create_dir_all(&profile.paths.log_root)?; @@ -85,20 +127,9 @@ pub async fn run_node(config: NodeRuntimeConfig) -> Result<()> { let mut processes = ProcessManager::new(profile.paths.log_root.clone(), profile.logs.clone()).await?; let web = web_service(&profile, config.bind.port())?; processes.start(&web, &BTreeMap::new()).await?; - if let Some(accepted) = durable::read::(&root.join("accepted-node.json"))? { - if accepted.prepared - && !admission::is_cancelled(&root, &accepted) - && !services::kv_paused(&profile)? - && !root - .join(format!( - "retired-bootstrap-{}.json", - accepted.bootstrap.operation_id - )) - .exists() - { - start_kv(&profile, &mut processes, &accepted).await?; - } - } + recovery::publish_local(&root, &config.discovery.addresses)?; + let endpoint_recovery = recovery::start(root.clone(), config.bind.port()); + kv::resume(&profile, &mut processes).await?; let (stop, stopped) = tokio::sync::oneshot::channel(); let discovery_root = root.clone(); let mut discovery = tokio::spawn(async move { @@ -123,18 +154,15 @@ pub async fn run_node(config: NodeRuntimeConfig) -> Result<()> { let mut storage_health = storage_health::StorageHealth::default(); let mut setup = setup::NodeSetup::default(); let mut heartbeat = tokio::time::interval(Duration::from_secs(1)); + let mut dispatch = dispatch::NodeDispatch::default(); let result = async { loop { tokio::select! { - _ = heartbeat.tick() => { - file_allocator.tick(&root)?; - supplied_storage.tick(&root)?; - storage_health.tick(&root).await; - setup.tick(&profile).await?; - health.tick(&profile, &web, &mut processes).await?; - } - connection = listener.accept() => { - let (mut stream, _) = connection?; + biased; + _ = terminate.recv() => break, + _ = tokio::signal::ctrl_c() => break, + outcome = &mut discovery => { outcome??; return Err("discovery unexpectedly stopped".into()); }, + work = dispatch.next(&listener, &mut heartbeat) => if let Some(mut stream) = work? { let reply = match read_request(&mut stream).await { Ok(request) => match control::execute(&profile, &mut processes, &mut initialization, &mut file_allocator, &mut supplied_storage, &mut setup, request).await { Ok(value) => json!({"ok": true, "value": value}), @@ -147,13 +175,18 @@ pub async fn run_node(config: NodeRuntimeConfig) -> Result<()> { } let _ = stream.shutdown().await; } - _ = terminate.recv() => break, - _ = tokio::signal::ctrl_c() => break, - outcome = &mut discovery => { outcome??; return Err("discovery unexpectedly stopped".into()); }, + else { + file_allocator.tick(&root)?; + supplied_storage.tick(&root)?; + storage_health.tick(&root).await; + setup.tick(&profile).await?; + health.tick(&profile, &web, &mut processes).await?; + } } } Ok::<(), Box>(()) }.await; + drop(endpoint_recovery); initialization.stop().await; setup.stop().await; storage_health.stop().await; @@ -168,6 +201,24 @@ pub async fn run_node(config: NodeRuntimeConfig) -> Result<()> { result } +fn validate_retained_startup(profile: &DeploymentProfile) -> Result<()> { + let root = &profile.paths.data_root; + // KV-only admissions can precede credential provisioning. Once secrets or + // an Access intent are retained, validate them before launching children. + if root.join("secrets").try_exists()? + || services::retained(profile)? + .iter() + .any(|intent| intent.kind == "access") + { + crate::ServerCredentials::load_existing(root)?; + } + if let Some(root) = std::env::var_os("CROWDB_CORE_DIR") { + let crashes = crate::CrashRetention::open(root.into())?; + std::env::set_current_dir(crashes.root())?; + } + Ok(()) +} + async fn stop_services(profile: &DeploymentProfile, processes: &mut ProcessManager) -> Result<()> { let mut order = vec!["web".into(), "kv".into()]; order.extend( @@ -283,3 +334,14 @@ fn disk_inventory(root: &std::path::Path) -> Result { #[cfg(feature = "test-util")] pub use services::NodeServiceForTests; + +#[cfg(feature = "test-util")] +pub use recovery::rebind_recovery_services_for_tests; + +pub use recovery::read_recovery_identity; + +#[cfg(feature = "test-util")] +pub use cleanup::cleanup_node_for_tests; + +#[cfg(feature = "test-util")] +pub use recovery::capture_recovery_owners_for_tests; diff --git a/container/crowdb-monitor/src/node/runtime/cleanup.rs b/container/crowdb-monitor/src/node/runtime/cleanup.rs index b46a2a3aa..4af24c9da 100644 --- a/container/crowdb-monitor/src/node/runtime/cleanup.rs +++ b/container/crowdb-monitor/src/node/runtime/cleanup.rs @@ -2,7 +2,7 @@ // Licensed under the Apache License, Version 2.0. use super::super::durable; -use super::{start_kv, AcceptedNode, Result}; +use super::{AcceptedNode, Result}; use crate::{DeploymentProfile, ProcessManager}; use crowdb_protocol::mgmt::SystemBootstrapIdentity; use serde_json::{json, Value}; @@ -42,7 +42,7 @@ pub(super) async fn execute( .await?; } } - start_kv(profile, processes, &accepted).await?; + super::kv::start_for_cleanup(profile, processes, &accepted).await?; let client = reqwest::Client::builder() .no_proxy() .timeout(Duration::from_secs(30)) @@ -58,6 +58,11 @@ pub(super) async fn execute( "node-binding.json", "accepted-node.json", "prepared-bootstrap.json", + "member-trust.json", + "confirmed-member-trust.json", + "member-endpoints.json", + "recovery-groups.json", + "service-address.json", "initialization.json", "initialization-status.json", ] { @@ -78,3 +83,17 @@ pub(super) async fn execute( durable::write(&root.join("node-cleaned.json"), &bootstrap)?; Ok(json!({})) } + +#[cfg(feature = "test-util")] +/// Execute the authenticated local cleanup after its worker quiescence checks. +/// +/// # Errors +/// Rejects unconfirmed or mismatched deletion and incomplete process/storage cleanup. +pub async fn cleanup_node_for_tests( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + bootstrap: SystemBootstrapIdentity, + confirmed: bool, +) -> Result { + execute(profile, processes, bootstrap, confirmed).await +} diff --git a/container/crowdb-monitor/src/node/runtime/control.rs b/container/crowdb-monitor/src/node/runtime/control.rs index 6615ab1da..baa6b7a79 100644 --- a/container/crowdb-monitor/src/node/runtime/control.rs +++ b/container/crowdb-monitor/src/node/runtime/control.rs @@ -21,9 +21,14 @@ pub(super) async fn execute( ) -> Result { let root = &profile.paths.data_root; match request { + NodeControl::DeploymentPolicy => Ok(serde_json::to_value(crate::node::read_node_policy(root)?)?), NodeControl::AcceptNodeSetup { request } => Ok(serde_json::to_value(setup.accept(root, request)?)?), NodeControl::NodeSetup => Ok(serde_json::to_value(setup::NodeSetup::status(root)?)?), NodeControl::NodeSetupIntent => Ok(serde_json::to_value(setup::NodeSetup::intent(root)?)?), + NodeControl::RuntimeObservation => { + Ok(serde_json::to_value(observation::runtime(profile, processes)?)?) + } + NodeControl::RecoveryIdentity => Ok(serde_json::to_value(super::recovery::identity(root)?)?), NodeControl::CheckRetainedDisk { request } => Ok(serde_json::to_value( retained_storage::check(root, request).await?, )?), @@ -35,14 +40,11 @@ pub(super) async fn execute( | NodeControl::DiskFilePreparations { .. }) => { storage(root, file_allocator, supplied_storage, request).await } - NodeControl::Initialize { - manifest, - credentials, - create_data_group, - } => { - authorize_bound(root).await?; - initialization.accept(profile, manifest, credentials, create_data_group) - } + request @ (NodeControl::Initialize { .. } + | NodeControl::PrepareBootstrap { .. } + | NodeControl::AcceptMemberTrust { .. } + | NodeControl::AcceptConfirmedMemberTrust { .. } + | NodeControl::StartKv { .. }) => bootstrap(profile, processes, initialization, request).await, NodeControl::Initialization => initialization::Initialization::status(root), NodeControl::DiskInventory => disk_inventory(root), NodeControl::Processes { cluster_id, node_id } => { @@ -59,26 +61,65 @@ pub(super) async fn execute( node_id, admission, } => admission::cancel(profile, processes, bootstrap, node_id, admission).await, - NodeControl::Identity => identity(root), - NodeControl::InstallKey { - operation_id, - public_key, + request @ (NodeControl::Identity + | NodeControl::InstallKey { .. } + | NodeControl::RemoveKey { .. } + | NodeControl::TrustHost { .. }) => keys(root, request).await, + NodeControl::Bind { binding } => kv::bind(profile, processes, binding).await, + NodeControl::Cleanup { + bootstrap, + confirm_delete_system_store, + } => { + supplied_storage::SuppliedStorage::retire(root, &bootstrap, confirm_delete_system_store)?; + if initialization.running() { + return Err("initialization is still running".into()); + } + if file_allocator.running() { + return Err("disk file preparation is still running".into()); + } + if supplied_storage.running() { + return Err("supplied disk provisioning is still running".into()); + } + setup.prepare_cleanup(root, &bootstrap, confirm_delete_system_store)?; + cleanup::execute(profile, processes, bootstrap, confirm_delete_system_store).await + } + } +} + +async fn bootstrap( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + initialization: &mut initialization::Initialization, + request: NodeControl, +) -> Result { + let root = &profile.paths.data_root; + match request { + NodeControl::Initialize { + manifest, + credentials, + create_data_group, } => { authorize_bound(root).await?; - trust::install(root, &operation_id, &public_key)?; + initialization.accept(profile, manifest, credentials, create_data_group) + } + NodeControl::PrepareBootstrap { + node_id, + bootstrap, + manifest, + credentials, + } => { + authorize_bound(root).await?; + preparation::accept(root, node_id, bootstrap, Some(manifest), Some(credentials), None)?; Ok(json!({})) } - NodeControl::RemoveKey { operation_id } => { - trust::remove(root, &operation_id)?; + NodeControl::AcceptMemberTrust { trust } => { + authorize_bound(root).await?; + super::recovery::accept(root, &serde_json::from_value(trust)?)?; Ok(json!({})) } - NodeControl::TrustHost { - host, - port, - public_key, - } => { + NodeControl::AcceptConfirmedMemberTrust { trust } => { authorize_bound(root).await?; - trust::host(root, &host, port, &public_key)?; + super::recovery::accept_confirmed(root, &serde_json::from_value(trust)?).await?; Ok(json!({})) } NodeControl::StartKv { @@ -94,27 +135,11 @@ pub(super) async fn execute( authorize_bound(root).await?; kv::ensure_start_allowed(profile).await?; let accepted = preparation::accept(root, node_id, bootstrap, manifest, credentials, admission)?; + super::recovery::prepare_admitted(root, &accepted).await?; start_kv(profile, processes, &accepted).await?; Ok(json!({"node_id": node_id, "management_port": 10000, "rpc_port": 10100})) } - NodeControl::Bind { binding } => kv::bind(profile, processes, binding).await, - NodeControl::Cleanup { - bootstrap, - confirm_delete_system_store, - } => { - supplied_storage::SuppliedStorage::retire(root, &bootstrap, confirm_delete_system_store)?; - if initialization.running() { - return Err("initialization is still running".into()); - } - if file_allocator.running() { - return Err("disk file preparation is still running".into()); - } - if supplied_storage.running() { - return Err("supplied disk provisioning is still running".into()); - } - setup.prepare_cleanup(root, &bootstrap, confirm_delete_system_store)?; - cleanup::execute(profile, processes, bootstrap, confirm_delete_system_store).await - } + _ => Err("unsupported bootstrap control request".into()), } } @@ -152,3 +177,31 @@ fn identity(root: &std::path::Path) -> Result { "public_key": std::fs::read_to_string(root.join("ssh/id_ed25519.pub"))?, "host_key": std::fs::read_to_string(root.join("ssh/ssh_host_ed25519_key.pub"))?})) } + +async fn keys(root: &std::path::Path, request: NodeControl) -> Result { + match request { + NodeControl::Identity => identity(root), + NodeControl::InstallKey { + operation_id, + public_key, + } => { + authorize_bound(root).await?; + trust::install(root, &operation_id, &public_key)?; + Ok(json!({})) + } + NodeControl::RemoveKey { operation_id } => { + trust::remove(root, &operation_id)?; + Ok(json!({})) + } + NodeControl::TrustHost { + host, + port, + public_key, + } => { + authorize_bound(root).await?; + trust::host(root, &host, port, &public_key)?; + Ok(json!({})) + } + _ => Err("unsupported key control request".into()), + } +} diff --git a/container/crowdb-monitor/src/node/runtime/dispatch.rs b/container/crowdb-monitor/src/node/runtime/dispatch.rs new file mode 100644 index 000000000..0ea0ef6e9 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/dispatch.rs @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Fair progress between private control and background node work. + +use tokio::{ + net::{UnixListener, UnixStream}, + time::Interval, +}; + +#[derive(Debug, Default)] +pub struct NodeDispatch { + health_first: bool, +} + +impl NodeDispatch { + /// Select the next accepted control stream or a background health pass. + /// # Errors + /// Propagates Unix listener failures. + pub async fn next( + &mut self, + listener: &UnixListener, + heartbeat: &mut Interval, + ) -> std::io::Result> { + let stream = if self.health_first { + tokio::select! { + biased; + _ = heartbeat.tick() => None, + connection = listener.accept() => Some(connection?.0), + } + } else { + tokio::select! { + biased; + connection = listener.accept() => Some(connection?.0), + _ = heartbeat.tick() => None, + } + }; + // When both are ready, alternate priority so neither accepted + // commands nor service reconciliation can starve the other. + self.health_first = stream.is_some(); + Ok(stream) + } +} diff --git a/container/crowdb-monitor/src/node/runtime/file_preparation.rs b/container/crowdb-monitor/src/node/runtime/file_preparation.rs index 0b47d2ffe..0f62eef96 100644 --- a/container/crowdb-monitor/src/node/runtime/file_preparation.rs +++ b/container/crowdb-monitor/src/node/runtime/file_preparation.rs @@ -48,7 +48,7 @@ impl FileAllocator { loop { let page = crate::node::storage::file_operations::list(root, after.as_deref(), 32)?; for state in page.items { - if resumable(state.phase) || state.phase == FilePreparationPhase::Unknown { + if resumable(&state) || state.phase == FilePreparationPhase::Unknown { self.enqueue(state.request.operation_id); } } @@ -76,7 +76,7 @@ impl FileAllocator { .collect(); for id in finished { self.tasks.remove(&id); - if resumable(files::status(root, &id)?.phase) { + if resumable(&files::status(root, &id)?) { self.queued.push_back(id); } else { self.known.remove(&id); @@ -144,14 +144,15 @@ impl FileAllocator { } } -fn resumable(phase: FilePreparationPhase) -> bool { - matches!( - phase, - FilePreparationPhase::Accepted - | FilePreparationPhase::Creating - | FilePreparationPhase::Publishing - | FilePreparationPhase::WaitingForDiskIO - ) +fn resumable(state: &FilePreparation) -> bool { + (state.phase == FilePreparationPhase::Prepared && state.request.admission.is_some()) + || matches!( + state.phase, + FilePreparationPhase::Accepted + | FilePreparationPhase::Creating + | FilePreparationPhase::Publishing + | FilePreparationPhase::WaitingForDiskIO + ) } async fn authorize(root: &Path, request: &PrepareDiskFile) -> Result<()> { @@ -168,3 +169,15 @@ async fn authorize(root: &Path, request: &PrepareDiskFile) -> Result<()> { .await??; Ok(()) } + +#[cfg(feature = "test-util")] +/// Resume durable file work through the production queue and wait for one pass. +/// +/// # Errors +/// Propagates durable queue discovery failures. +pub async fn resume_file_operations_for_tests(root: &Path) -> Result<()> { + let mut allocator = FileAllocator::default(); + allocator.resume(root)?; + allocator.stop().await; + Ok(()) +} diff --git a/container/crowdb-monitor/src/node/runtime/health.rs b/container/crowdb-monitor/src/node/runtime/health.rs index 6f4415194..f5377daf9 100644 --- a/container/crowdb-monitor/src/node/runtime/health.rs +++ b/container/crowdb-monitor/src/node/runtime/health.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; use std::time::Duration; -use super::{durable, start_kv, AcceptedNode, Result}; +use super::{durable, AcceptedNode, Result}; use crate::{ DeploymentProfile, MonitorPhase, MonitorStatus, ProcessManager, ServiceProfile, ServiceStatus, StatusStore, @@ -42,18 +42,15 @@ impl NodeHealth { if let Some(accepted) = durable::read::(&profile.paths.data_root.join("accepted-node.json"))? { - let retired = profile.paths.data_root.join(format!( - "retired-bootstrap-{}.json", - accepted.bootstrap.operation_id - )); - if accepted.prepared - && !super::admission::is_cancelled(&profile.paths.data_root, &accepted) - && !retired.exists() - && !super::services::kv_paused(profile)? + if super::admission::is_cancelled(&profile.paths.data_root, &accepted) { + self.retire_services(profile, processes).await?; + return Ok(()); + } + if super::kv::resume_allowed(profile, &accepted)? && (!processes.owns("kv") || !processes.alive("kv")?) { self.restart_allowed("kv")?; - start_kv(profile, processes, &accepted).await?; + super::kv::ensure_process(profile, processes, &accepted).await?; } } self.reconcile_services(profile, processes).await?; @@ -65,7 +62,18 @@ impl NodeHealth { if processes.owns(id) { let mut healthy = processes.alive(id)?; if healthy { - if let Some(intent) = retained + if id == "web" || id == "kv" { + let service = if id == "web" { + web.clone() + } else { + super::service(profile, "kv")? + }; + healthy = self + .probes + .probe_service(&service, &BTreeMap::new()) + .await + .is_ok(); + } else if let Some(intent) = retained .iter() .find(|intent| intent.service_id == *id && intent.kind != "kv") { @@ -82,6 +90,10 @@ impl NodeHealth { self.status.services.insert( id.clone(), ServiceStatus { + operation_id: retained + .iter() + .find(|intent| intent.service_id == *id || (id == "kv" && intent.kind == "kv")) + .map(|intent| intent.operation_id.clone()), pid: processes.pid(id), generation: 1 + u64::from(*self.attempts.get(id.as_str()).unwrap_or(&0)), healthy, @@ -99,7 +111,7 @@ impl NodeHealth { Ok(()) } - async fn reconcile_services( + pub(in crate::node::runtime) async fn reconcile_services( &mut self, profile: &DeploymentProfile, processes: &mut ProcessManager, @@ -107,16 +119,24 @@ impl NodeHealth { if !profile.paths.data_root.join("node-binding.json").exists() { return Ok(()); } - let Ok(intents) = super::services::desired(profile).await else { - return Ok(()); + let intents = match super::services::desired(profile).await { + Ok(intents) => intents, + Err(error) => { + tracing::debug!(%error, "service recovery awaits committed intents"); + return Ok(()); + } }; - for intent in intents { - if !super::services::pending(profile, &intent, processes)? { + for intent in &intents { + if !super::services::pending(profile, intent, processes)? { + continue; + } + if let Err(error) = super::services::dependencies_ready(profile, intent, &intents).await { + tracing::debug!(service = %intent.service_id, %error, "service recovery waits for dependencies"); continue; } match super::services::execute(profile, processes, intent.clone()).await { Ok(_) => { - *self.attempts.entry(intent.service_id).or_default() += 1; + *self.attempts.entry(intent.service_id.clone()).or_default() += 1; } Err(error) => eprintln!("service recovery deferred for {}: {error}", intent.service_id), } @@ -134,4 +154,41 @@ impl NodeHealth { *attempts += 1; Ok(()) } + + async fn retire_services( + &mut self, + profile: &DeploymentProfile, + processes: &mut ProcessManager, + ) -> Result<()> { + let mut ids: Vec<_> = super::services::retained(profile)? + .into_iter() + .map(|intent| intent.service_id) + .collect(); + ids.reverse(); + ids.push("kv".into()); + for id in ids { + if processes.owns(&id) && processes.alive(&id)? { + processes.stop(&id, Duration::from_secs(5)).await?; + } + } + self.status.services.clear(); + self.status.phase = MonitorPhase::Failed; + self.store.publish(&mut self.status)?; + Ok(()) + } +} + +#[cfg(feature = "test-util")] +/// Collect one production health pass for owned processes. +/// +/// # Errors +/// Propagates process, durable state and status publication failures. +pub async fn node_health_for_tests( + profile: &DeploymentProfile, + web: &ServiceProfile, + processes: &mut ProcessManager, +) -> Result { + let mut health = NodeHealth::new(profile)?; + health.tick(profile, web, processes).await?; + Ok(health.status) } diff --git a/container/crowdb-monitor/src/node/runtime/initialization.rs b/container/crowdb-monitor/src/node/runtime/initialization.rs index 82421e32a..d11c217e1 100644 --- a/container/crowdb-monitor/src/node/runtime/initialization.rs +++ b/container/crowdb-monitor/src/node/runtime/initialization.rs @@ -157,6 +157,7 @@ async fn execute(profile: &DeploymentProfile, request: &Request, local: u64) -> let root = &profile.paths.data_root; let operation = &request.operation; let credentials = ServerCredentials::load_existing(root)?.server_env(); + prepare_members(profile, operation, local, &credentials).await?; let results = futures::future::join_all(operation.intent.members().iter().map(async |member| { control( profile, @@ -212,29 +213,63 @@ async fn execute(profile: &DeploymentProfile, request: &Request, local: u64) -> } if request.create_data_group { progress(root, request, "data-group", None)?; - ensure_data_group(&ctx, operation.intent.members()).await?; + ensure_data_group(&ctx, operation).await?; } progress(root, request, "active", None) } -async fn ensure_data_group(ctx: &OpContext, nodes: &[u64]) -> Result<()> { - if ctx.sysmd().get_group(0, 1).await?.is_some() { - let mut actual: Vec<_> = ctx - .sysmd() - .list_replicas_in_group(0, 1) - .await? - .iter() - .map(|replica| replica.node_id) - .collect(); - let mut expected = nodes.to_vec(); - actual.sort_unstable(); - expected.sort_unstable(); - if actual != expected { - return Err("existing Group 1 membership differs from initialization nodes".into()); - } - return Ok(()); +async fn prepare_members( + profile: &DeploymentProfile, + operation: &PreparedBootstrap, + local: u64, + credentials: &str, +) -> Result<()> { + let prepares = futures::future::join_all(operation.intent.members().iter().map(async |member| { + control( + profile, + operation, + local, + *member, + NodeControl::PrepareBootstrap { + node_id: *member, + bootstrap: operation.identity.clone(), + manifest: serde_json::to_value(operation)?, + credentials: NodeServiceCredentials { + environment: credentials.to_owned(), + }, + }, + ) + .await + })) + .await; + for result in prepares { + result?; } - crowdb_console_shared::ops::kv_logical::add_group(ctx, 0, 1, 1, nodes).await?; + let trust = super::recovery::capture(&profile.paths.data_root, operation)?; + let pins = futures::future::join_all(operation.intent.members().iter().map(async |member| { + control( + profile, + operation, + local, + *member, + NodeControl::AcceptMemberTrust { + trust: serde_json::to_value(&trust)?, + }, + ) + .await + })) + .await; + for result in pins { + result?; + } + Ok(()) +} + +async fn ensure_data_group(ctx: &OpContext, operation: &PreparedBootstrap) -> Result<()> { + // Manual initialization and accepted node setup share the same durable + // ownership before either runner creates the ordinary metadata group. + let accepted = crowdb_console_shared::deployment::preparation::reserve(ctx, &operation.identity).await?; + crowdb_console_shared::ops::kv_logical::ensure_default_group(ctx, &accepted).await?; Ok(()) } diff --git a/container/crowdb-monitor/src/node/runtime/kv.rs b/container/crowdb-monitor/src/node/runtime/kv.rs index ab5cb7566..aa92efc66 100644 --- a/container/crowdb-monitor/src/node/runtime/kv.rs +++ b/container/crowdb-monitor/src/node/runtime/kv.rs @@ -7,8 +7,37 @@ use crowdb_protocol::mgmt::node::NodeBinding; use serde_json::{json, Value}; use std::{collections::BTreeMap, time::Duration}; +pub(super) fn resume_allowed(profile: &DeploymentProfile, accepted: &AcceptedNode) -> Result { + let root = &profile.paths.data_root; + Ok(accepted.prepared + && !super::admission::is_cancelled(root, accepted) + && !root + .join(format!( + "retired-bootstrap-{}.json", + accepted.bootstrap.operation_id + )) + .exists() + && !super::services::kv_paused(profile)? + && super::recovery::ready(root, accepted)?) +} + +pub(super) async fn resume(profile: &DeploymentProfile, processes: &mut ProcessManager) -> Result<()> { + if let Some(accepted) = durable::read(&profile.paths.data_root.join("accepted-node.json"))? { + if resume_allowed(profile, &accepted)? { + ensure_process(profile, processes, &accepted).await?; + } + } + Ok(()) +} + pub(super) fn node_profile(config: &super::NodeRuntimeConfig) -> Result { let mut profile = DeploymentProfile::load(&config.profile)?; + profile.validate_node_catalog()?; + for service in &mut profile.services { + service + .env + .insert("CROWDB_PHYSICAL_HOST_ID".into(), config.physical_host_id.clone()); + } let address = config .discovery .addresses @@ -68,6 +97,59 @@ pub(super) async fn start_kv( profile: &DeploymentProfile, processes: &mut ProcessManager, accepted: &AcceptedNode, +) -> Result<()> { + ensure_process(profile, processes, accepted).await?; + wait_ready(processes).await +} + +async fn wait_ready(processes: &mut ProcessManager) -> Result<()> { + let client = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(1)) + .build()?; + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + if client + .get("http://127.0.0.1:10000/health") + .send() + .await + .is_ok_and(|reply| reply.status().is_success()) + { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline || !processes.alive("kv")? { + return Err("KV server did not become ready".into()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + +// Retained replicas need authenticated discovery before they can form a quorum. +// Starting their process must not make discovery wait for that quorum. +pub(super) async fn ensure_process( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + accepted: &AcceptedNode, +) -> Result<()> { + super::recovery::check(&profile.paths.data_root, accepted)?; + spawn(profile, processes, accepted).await +} + +// Explicit cleanup already validates the accepted bootstrap and seals retirement. +// Its local deletion endpoint must remain available after recovery is fenced. +pub(super) async fn start_for_cleanup( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + accepted: &AcceptedNode, +) -> Result<()> { + spawn(profile, processes, accepted).await?; + wait_ready(processes).await +} + +async fn spawn( + profile: &DeploymentProfile, + processes: &mut ProcessManager, + accepted: &AcceptedNode, ) -> Result<()> { if processes.owns("kv") && processes.alive("kv")? { return Ok(()); @@ -76,6 +158,10 @@ pub(super) async fn start_kv( processes.stop("kv", Duration::from_secs(5)).await?; } let mut kv = service(profile, "kv")?; + kv.env.insert( + "CROWDB_KV_CLUSTER_ID".into(), + accepted.bootstrap.cluster_id.clone(), + ); let root = profile .paths .data_root @@ -83,6 +169,8 @@ pub(super) async fn start_kv( .join(format!("node-{}", accepted.node_id)); std::fs::create_dir_all(&root)?; kv.args = vec![ + "--node-recovery-root".into(), + profile.paths.data_root.to_string_lossy().into_owned(), "--root".into(), root.to_string_lossy().into_owned(), "--management-port".into(), @@ -105,25 +193,7 @@ pub(super) async fn start_kv( } } processes.start(&kv, &BTreeMap::new()).await?; - let client = reqwest::Client::builder() - .no_proxy() - .timeout(Duration::from_secs(1)) - .build()?; - let deadline = tokio::time::Instant::now() + Duration::from_secs(10); - loop { - if client - .get("http://127.0.0.1:10000/health") - .send() - .await - .is_ok_and(|reply| reply.status().is_success()) - { - return Ok(()); - } - if tokio::time::Instant::now() >= deadline || !processes.alive("kv")? { - return Err("KV server did not become ready".into()); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } + Ok(()) } pub(super) async fn ensure_start_allowed(profile: &crate::DeploymentProfile) -> super::Result<()> { use crowdb_protocol::mgmt::node::NodeServiceAction; diff --git a/container/crowdb-monitor/src/node/runtime/observation.rs b/container/crowdb-monitor/src/node/runtime/observation.rs index 3ea4d307f..04ad92d0b 100644 --- a/container/crowdb-monitor/src/node/runtime/observation.rs +++ b/container/crowdb-monitor/src/node/runtime/observation.rs @@ -9,6 +9,19 @@ use serde_json::{json, Value}; use super::{durable, Result}; use crate::ProcessManager; +mod readiness; +pub(super) use readiness::observe as runtime; + +/// # Errors +/// Uses the production read-only projection without starting monitor workers. +#[cfg(feature = "test-util")] +pub fn runtime_observation_for_tests( + profile: &crate::DeploymentProfile, + manager: &mut ProcessManager, +) -> Result { + runtime(profile, manager) +} + pub(super) fn processes( root: &Path, manager: &mut ProcessManager, @@ -21,7 +34,7 @@ pub(super) fn processes( } // This authenticated read observes only local children. It needs no live // Group-0 grant and remains available while that authority is unavailable. - Ok( - json!({"cluster_id":cluster,"node_id":node,"processes":manager.live_pids()?,"service_failures":super::services::failures(root)?}), - ) + let live = manager.live_pids()?; + let failures = super::services::failures(root, &live)?; + Ok(json!({"cluster_id":cluster,"node_id":node,"processes":live,"service_failures":failures})) } diff --git a/container/crowdb-monitor/src/node/runtime/observation/readiness.rs b/container/crowdb-monitor/src/node/runtime/observation/readiness.rs new file mode 100644 index 000000000..c0b165b95 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/observation/readiness.rs @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Read local durable and runtime facts without accepting or repairing work. + +use super::super::{durable, services, setup, Result}; +use crate::{DeploymentProfile, NodeIdentity, ProcessManager, StatusStore}; +use crowdb_protocol::mgmt::node::observation::NodeRuntimeObservation; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +pub(in crate::node::runtime) fn observe( + profile: &DeploymentProfile, + manager: &mut ProcessManager, +) -> Result { + let root = &profile.paths.data_root; + let identity = NodeIdentity::load_existing(root)?; + let policy = crate::read_node_policy(root)?.ok_or("accepted node policy is absent")?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH)? + .as_millis() + .try_into()?; + let health = StatusStore::open(&profile.paths.run_root) + .and_then(|store| store.read(Duration::from_secs(3))) + .ok() + .filter(|health| health.deployment_id == identity.uuid() && health.monitor_pid == std::process::id()); + let live = manager.live_pids()?; + let services = services::observe(profile, &live, health.as_ref(), now)?; + Ok(NodeRuntimeObservation { + version: 1, + discovery_id: identity.uuid().to_string(), + policy, + binding: durable::read(&root.join("node-binding.json"))?, + setup: setup::NodeSetup::status(root)?, + observed_at_unix_ms: now, + services, + protocols: Vec::new(), + }) +} diff --git a/container/crowdb-monitor/src/node/runtime/recovery.rs b/container/crowdb-monitor/src/node/runtime/recovery.rs new file mode 100644 index 000000000..22ac60f34 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/recovery.rs @@ -0,0 +1,283 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Seal authenticated member pins before starting bootstrap replicas. + +use std::path::Path; + +use crowdb_console_shared::deployment::{ + recovery::{MemberTrust, TrustedMember}, + PreparedBootstrap, +}; +use ssh_key::PublicKey; + +use super::{durable, AcceptedNode, Result}; +use crate::NodeIdentity; +mod confirmed; +mod discovery; +mod owners; +mod publication; +pub(super) use confirmed::{accept as accept_confirmed, prepare_admitted}; +pub(super) use discovery::start; +#[cfg(feature = "test-util")] +pub use discovery::unique_member_endpoints_for_tests; + +pub(super) fn publish_local(root: &Path, addresses: &[std::net::IpAddr]) -> Result<()> { + durable::write( + &root.join("node-endpoint.json"), + &crowdb_protocol::mgmt::node::recovery::MemberEndpoint { + discovery_id: NodeIdentity::load_existing(root)?.uuid().to_string(), + node_id: 0, + boot_id: uuid::Uuid::new_v4().to_string(), + addresses: addresses.to_vec(), + }, + )?; + Ok(()) +} + +pub(super) fn identity(root: &Path) -> Result { + use crowdb_protocol::mgmt::node::{ + recovery::{MemberEndpoint, MemberRecoveryIdentity}, + NodeBinding, + }; + let accepted: AcceptedNode = + durable::read(&root.join("accepted-node.json"))?.ok_or("node is not prepared")?; + check(root, &accepted)?; + if accepted.admission.is_some() && !ready(root, &accepted)? { + return Err("member admission is not confirmed for address recovery".into()); + } + if super::admission::is_cancelled(root, &accepted) + || root + .join(format!( + "retired-bootstrap-{}.json", + accepted.bootstrap.operation_id + )) + .exists() + { + return Err("member identity is retired".into()); + } + let mut endpoint: MemberEndpoint = + durable::read(&root.join("node-endpoint.json"))?.ok_or("node endpoint is absent")?; + endpoint.node_id = accepted.node_id; + if endpoint.discovery_id != NodeIdentity::load_existing(root)?.uuid().to_string() { + return Err("local endpoint identity differs".into()); + } + let binding = durable::read(&root.join("node-binding.json"))?.unwrap_or(NodeBinding { + node_id: accepted.node_id, + bootstrap: accepted.bootstrap.clone(), + management_seeds: Vec::new(), + }); + Ok(MemberRecoveryIdentity { + version: 1, + binding, + endpoint, + }) +} + +pub(super) fn capture(root: &Path, operation: &PreparedBootstrap) -> Result { + operation.validate()?; + let local = NodeIdentity::load_existing(root)?.uuid().to_string(); + let members = operation + .nodes + .iter() + .map(|node| { + let key = if node.discovery_id == local { + std::fs::read_to_string(root.join("ssh/ssh_host_ed25519_key.pub"))? + } else { + super::trust::pinned_host(root, &node.host, node.ssh_port)? + }; + Ok(TrustedMember { + discovery_id: node.discovery_id.clone(), + node_id: node.node_id, + admission_operation_id: node.operation_id.clone(), + host_key: PublicKey::new(PublicKey::from_openssh(&key)?.key_data().clone(), "") + .to_openssh()?, + }) + }) + .collect::>>()?; + let trust = MemberTrust { + version: 1, + bootstrap: operation.identity.clone(), + members, + }; + trust.validate_bootstrap(operation)?; + Ok(trust) +} + +pub(super) fn accept(root: &Path, trust: &MemberTrust) -> Result<()> { + let accepted: AcceptedNode = + durable::read(&root.join("accepted-node.json"))?.ok_or("member trust requires accepted bootstrap")?; + if !accepted.prepared + || accepted.admission.is_some() + || super::admission::is_cancelled(root, &accepted) + || root + .join(format!( + "retired-bootstrap-{}.json", + accepted.bootstrap.operation_id + )) + .exists() + { + return Err("member trust requires active accepted bootstrap preparation".into()); + } + let operation = PreparedBootstrap::load(&root.join("prepared-bootstrap.json"))?; + trust.validate_bootstrap(&operation)?; + let local = NodeIdentity::load_existing(root)?.uuid().to_string(); + if trust.bootstrap != accepted.bootstrap + || !trust + .members + .iter() + .any(|member| member.node_id == accepted.node_id && member.discovery_id == local) + { + return Err("member trust differs from local acceptance".into()); + } + // Every pin must agree with the keys established by mutual SSH admission, + // including this node's own host key. The coordinator cannot replace them. + let expected = capture(root, &operation)?; + if *trust != expected { + return Err("member trust differs from authenticated admission keys".into()); + } + trust.seal(&root.join("member-trust.json"))?; + let group = crowdb_protocol::mgmt::node::recovery::RecoveryGroup { + store_id: 0, + group_id: 0, + deployment: None, + members: operation + .intent + .members() + .iter() + .enumerate() + .map( + |(index, node)| crowdb_protocol::mgmt::node::recovery::RecoveryReplica { + replica_id: index as u64 + 1, + node_id: *node, + }, + ) + .collect(), + }; + if !root.join("recovery-groups.json").exists() { + durable::write( + &root.join("recovery-groups.json"), + &crowdb_protocol::mgmt::node::recovery::RecoveryGroups { + bootstrap: operation.identity, + groups: vec![group], + }, + )?; + } + Ok(()) +} + +pub(super) fn check(root: &Path, accepted: &AcceptedNode) -> Result<()> { + if super::admission::is_cancelled(root, accepted) + || root + .join(format!( + "retired-bootstrap-{}.json", + accepted.bootstrap.operation_id + )) + .exists() + { + return Err("member recovery is retired".into()); + } + let trust = if accepted.admission.is_some() { + confirmed::admitted(root, accepted)? + } else { + MemberTrust::load(&root.join("member-trust.json"))? + }; + let operation = PreparedBootstrap::load(&root.join("prepared-bootstrap.json"))?; + if accepted.admission.is_none() { + trust.validate_bootstrap(&operation)?; + } + let local = NodeIdentity::load_existing(root)?.uuid().to_string(); + let member = trust + .members + .iter() + .find(|member| member.node_id == accepted.node_id && member.discovery_id == local) + .ok_or("retained member trust differs from local identity")?; + let current = PublicKey::from_openssh(&std::fs::read_to_string( + root.join("ssh/ssh_host_ed25519_key.pub"), + )?)?; + if trust.bootstrap != accepted.bootstrap + || accepted + .admission + .as_ref() + .is_some_and(|grant| member.admission_operation_id != grant.operation_id) + || current.key_data() != PublicKey::from_openssh(&member.host_key)?.key_data() + { + return Err("retained member trust differs from local identity".into()); + } + Ok(()) +} + +pub(super) fn ready(root: &Path, accepted: &AcceptedNode) -> Result { + if accepted.admission.is_some() { + for path in [ + root.join("prepared-bootstrap.json"), + root.join("member-trust.json"), + confirmed::admitted_path(root, accepted)?, + root.join("confirmed-member-trust.json"), + ] { + match std::fs::symlink_metadata(path) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) => return Err(error.into()), + } + } + check(root, accepted)?; + let operation = PreparedBootstrap::load(&root.join("prepared-bootstrap.json"))?; + let admitted = confirmed::admitted(root, accepted)?; + let own = admitted + .members + .iter() + .find(|member| member.node_id == accepted.node_id) + .ok_or("local admitted pin is absent")?; + return Ok(confirmed::snapshot(root, &operation)? + .is_some_and(|snapshot| snapshot.current.members.contains(own))); + } + match std::fs::symlink_metadata(root.join("member-trust.json")) { + Ok(_) => { + check(root, accepted)?; + Ok(true) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error.into()), + } +} + +#[cfg(feature = "test-util")] +/// # Errors +/// Propagates the real acceptance, key validation and persistence failures. +pub fn accept_member_trust_for_tests(root: &Path, trust: &MemberTrust) -> Result<()> { + accept(root, trust) +} + +#[cfg(feature = "test-util")] +/// # Errors +/// Propagates the real retained identity validation failures. +pub fn check_member_trust_for_tests(root: &Path) -> Result<()> { + let accepted = durable::read(&root.join("accepted-node.json"))?.ok_or("missing accepted node")?; + check(root, &accepted) +} + +#[cfg(feature = "test-util")] +/// # Errors +/// Uses the production retained-trust admission fence without starting services. +pub fn member_trust_ready_for_tests(root: &Path) -> Result { + let accepted = durable::read(&root.join("accepted-node.json"))?.ok_or("missing accepted node")?; + ready(root, &accepted) +} + +#[cfg(feature = "test-util")] +pub use publication::rebind_recovery_services_for_tests; + +/// Read a live monitor's retained identity without waiting for lifecycle control. +/// +/// # Errors +/// Rejects an inactive monitor, changed keys, missing trust and retired identity. +pub async fn read_recovery_identity( + root: &Path, +) -> Result { + crate::probe_liveness(root).await?; + identity(root) +} + +#[cfg(feature = "test-util")] +pub use owners::capture_recovery_owners_for_tests; diff --git a/container/crowdb-monitor/src/node/runtime/recovery/confirmed.rs b/container/crowdb-monitor/src/node/runtime/recovery/confirmed.rs new file mode 100644 index 000000000..19c060f23 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/recovery/confirmed.rs @@ -0,0 +1,241 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Current authority publishes non-voter pins; retained pins recover only existing routes. + +use super::super::{durable, AcceptedNode, Result}; +use crowdb_console_shared::{ + deployment::{ + recovery::{ConfirmedMemberTrust, MemberTrust, TrustedMember}, + registry, PreparedBootstrap, + }, + ConsoleConfig, +}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_protocol::mgmt::node::NodeBinding; +use ssh_key::PublicKey; +use std::{path::Path, time::Duration}; + +async fn authority( + root: &Path, + accepted: &AcceptedNode, +) -> Result<(PreparedBootstrap, registry::NodeRegistry)> { + let seeds = durable::read::(&root.join("node-binding.json"))? + .filter(|binding| binding.bootstrap == accepted.bootstrap) + .map(|binding| binding.management_seeds) + .or_else(|| { + accepted + .admission + .as_ref() + .map(|grant| grant.management_seeds.clone()) + }) + .ok_or("confirmed recovery requires authority contacts")?; + tokio::time::timeout(Duration::from_secs(3), async { + let client = CrowdbKvClient::new(ClientConfig::new(seeds)); + let GetOutcome::Found { value, .. } = client + .get( + 0, + 0, + crowdb_console_shared::deployment::CLUSTER_KEY, + ReadMode::Linearizable, + None, + ) + .await? + else { + return Err("cluster authority is absent".into()); + }; + let operation: PreparedBootstrap = serde_json::from_slice(&value)?; + operation.validate()?; + let (registry, _) = registry::read(&client) + .await? + .ok_or("confirmed node registry is absent")?; + if operation.identity != accepted.bootstrap || registry.cluster_id != accepted.bootstrap.cluster_id { + return Err("confirmed recovery authority belongs to another bootstrap".into()); + } + Ok((operation, registry)) + }) + .await? +} + +fn own_member( + root: &Path, + accepted: &AcceptedNode, + registry: ®istry::NodeRegistry, +) -> Result { + let operation = &accepted + .admission + .as_ref() + .ok_or("node has no admission grant")? + .operation_id; + let discovery = crate::NodeIdentity::load_existing(root)?.uuid().to_string(); + let node = registry + .nodes + .iter() + .find(|node| { + node.node_id == accepted.node_id + && node.discovery_id == discovery + && node.operation_id == *operation + && !node.cancelled + }) + .ok_or("local admission is not current in confirmed authority")?; + let host_key = node + .host_key + .as_ref() + .ok_or("admission has no authenticated public key")?; + let local = PublicKey::from_openssh(&std::fs::read_to_string( + root.join("ssh/ssh_host_ed25519_key.pub"), + )?)?; + if local.key_data() != PublicKey::from_openssh(host_key)?.key_data() { + return Err("confirmed admission key differs from this node".into()); + } + Ok(TrustedMember { + node_id: node.node_id, + discovery_id: discovery, + admission_operation_id: operation.clone(), + host_key: host_key.clone(), + }) +} + +pub(in crate::node::runtime) async fn prepare_admitted(root: &Path, accepted: &AcceptedNode) -> Result<()> { + if accepted.admission.is_none() { + return Ok(()); + } + let (operation, registry) = authority(root, accepted).await?; + let initial = MemberTrust::from_bootstrap(&operation)?; + let mut admitted = initial.clone(); + admitted.members.push(own_member(root, accepted, ®istry)?); + admitted.validate()?; + let path = root.join("prepared-bootstrap.json"); + if let Some(previous) = durable::read::(&path)? { + if previous != operation { + return Err("retained recovery manifest differs".into()); + } + } else { + durable::write(&path, &operation)?; + } + initial.seal(&root.join("member-trust.json"))?; + admitted.seal(&admitted_path(root, accepted)?)?; + Ok(()) +} + +pub(super) fn admitted_path(root: &Path, accepted: &AcceptedNode) -> Result { + let grant = accepted.admission.as_ref().ok_or("node has no admission grant")?; + let operation = uuid::Uuid::parse_str(&grant.operation_id)?; + Ok(root.join(format!("admitted-member-trust-{operation}.json"))) +} + +pub(super) fn admitted(root: &Path, accepted: &AcceptedNode) -> Result { + let operation = PreparedBootstrap::load(&root.join("prepared-bootstrap.json"))?; + let initial = MemberTrust::load(&root.join("member-trust.json"))?; + initial.validate_bootstrap(&operation)?; + let admitted = MemberTrust::load(&admitted_path(root, accepted)?)?; + if admitted.bootstrap != accepted.bootstrap + || admitted.members.len() != initial.members.len() + 1 + || initial + .members + .iter() + .any(|member| !admitted.members.contains(member)) + { + return Err("admitted recovery pins differ from initial authority".into()); + } + Ok(admitted) +} + +pub(super) fn snapshot(root: &Path, operation: &PreparedBootstrap) -> Result> { + let snapshot = durable::read::(&root.join("confirmed-member-trust.json"))?; + if let Some(snapshot) = &snapshot { + snapshot.validate(operation)?; + if snapshot.initial != MemberTrust::load(&root.join("member-trust.json"))? { + return Err("confirmed recovery snapshot changes initial pins".into()); + } + } + Ok(snapshot) +} + +fn store( + root: &Path, + accepted: &AcceptedNode, + operation: &PreparedBootstrap, + snapshot: &ConfirmedMemberTrust, +) -> Result<()> { + snapshot.validate(operation)?; + if snapshot.initial != MemberTrust::load(&root.join("member-trust.json"))? { + return Err("confirmed recovery snapshot changes initial pins".into()); + } + if accepted.admission.is_some() { + let own = own_member(root, accepted, &snapshot.registry)?; + if !snapshot.current.members.contains(&own) { + return Err("local admission is not confirmed for recovery".into()); + } + let path = root.join("service-address.json"); + if !path.exists() { + let node = snapshot + .registry + .nodes + .iter() + .find(|node| node.node_id == accepted.node_id) + .ok_or("local confirmed address is absent")?; + durable::write(&path, &node.host)?; + } + } + let path = root.join("confirmed-member-trust.json"); + let previous = self::snapshot(root, operation)?; + if previous.as_ref() != Some(snapshot) { + durable::write(&path, snapshot)?; + } + Ok(()) +} + +pub(in crate::node::runtime) async fn accept(root: &Path, snapshot: &ConfirmedMemberTrust) -> Result<()> { + let accepted: AcceptedNode = + durable::read(&root.join("accepted-node.json"))?.ok_or("node not accepted")?; + if super::super::admission::is_cancelled(root, &accepted) + || root + .join(format!( + "retired-bootstrap-{}.json", + accepted.bootstrap.operation_id + )) + .exists() + { + return Err("member recovery is retired".into()); + } + let (operation, current) = authority(root, &accepted).await?; + if current != snapshot.registry { + return Err("confirmed recovery registry was superseded".into()); + } + store(root, &accepted, &operation, snapshot) +} + +pub(super) async fn refresh(root: &Path, accepted: &AcceptedNode) -> Result<()> { + let (operation, registry) = authority(root, accepted).await?; + if let Some(grant) = &accepted.admission { + if snapshot(root, &operation)?.is_some() + && !registry.nodes.iter().any(|node| { + node.node_id == accepted.node_id + && node.operation_id == grant.operation_id + && node.confirmed + && !node.cancelled + }) + { + durable::write( + &root.join(format!("ssh/cancelled-{}.json", grant.operation_id)), + grant, + )?; + return Err("confirmed authority retired this admission".into()); + } + } + let initial = MemberTrust::load(&root.join("member-trust.json"))?; + let snapshot = ConfirmedMemberTrust::new(initial, registry)?; + store(root, accepted, &operation, &snapshot) +} + +pub(super) fn contacts(root: &Path, operation: &PreparedBootstrap) -> Result<(MemberTrust, ConsoleConfig)> { + if let Some(snapshot) = snapshot(root, operation)? { + let config = snapshot.topology(operation)?; + Ok((snapshot.current, config)) + } else { + let initial = MemberTrust::load(&root.join("member-trust.json"))?; + initial.validate_bootstrap(operation)?; + Ok((initial, operation.intent.to_config())) + } +} diff --git a/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs new file mode 100644 index 000000000..4311b1498 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/recovery/discovery.rs @@ -0,0 +1,177 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Discovery supplies hints; only pinned SSH proofs enter the local route view. + +use super::super::{durable, AcceptedNode, Result}; +use crowdb_console_shared::{ + deployment::{recovery::MemberTrust, PreparedBootstrap}, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::node::{ + recovery::{MemberEndpoint, MemberEndpoints}, + CandidateSnapshot, +}; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::Path, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; + +pub(in crate::node::runtime) struct Worker(tokio::task::JoinHandle<()>); +impl Drop for Worker { + fn drop(&mut self) { + self.0.abort(); + } +} + +pub(in crate::node::runtime) fn start(root: std::path::PathBuf, port: u16) -> Worker { + Worker(tokio::spawn(async move { + let mut tick = tokio::time::interval(Duration::from_secs(3)); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tick.tick().await; + if let Err(error) = refresh(&root, port).await { + tracing::debug!(%error, "member endpoint recovery is pending"); + } + } + })) +} + +async fn refresh(root: &Path, port: u16) -> Result<()> { + let Some(accepted) = durable::read::(&root.join("accepted-node.json"))? else { + return Ok(()); + }; + if super::super::admission::is_cancelled(root, &accepted) + || root + .join(format!( + "retired-bootstrap-{}.json", + accepted.bootstrap.operation_id + )) + .exists() + { + return Err("member recovery is retired".into()); + } + if let Err(error) = super::confirmed::refresh(root, &accepted).await { + tracing::debug!(%error, "confirmed recovery contacts await current authority"); + } + super::check(root, &accepted)?; + let operation = PreparedBootstrap::load(&root.join("prepared-bootstrap.json"))?; + let (trust, config) = super::confirmed::contacts(root, &operation)?; + let local = super::identity(root)?.endpoint; + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(1)) + .build()?; + let response = client + .get(format!("http://127.0.0.1:{port}/candidates")) + .send() + .await? + .error_for_status()?; + if response.content_length().is_some_and(|size| size > 1024 * 1024) { + return Err("candidate reply too large".into()); + } + let snapshot: CandidateSnapshot = response.json().await?; + let nodes = destinations(root, &config, &trust, snapshot)?; + let replies = futures::future::join_all(nodes.iter().map(|node| trust.endpoint(node))).await; + let endpoints = std::iter::once(local.clone()).chain( + replies + .into_iter() + .filter_map(std::result::Result::ok) + .map(|identity| identity.endpoint), + ); + let (members, conflicts) = unique(endpoints); + let snapshot = MemberEndpoints { + version: 1, + bootstrap: accepted.bootstrap, + observer_boot_id: local.boot_id, + observed_at_unix_ms: SystemTime::now() + .duration_since(UNIX_EPOCH)? + .as_millis() + .try_into()?, + members, + conflicts, + }; + durable::write(&root.join("member-endpoints.json"), &snapshot)?; + match tokio::time::timeout( + Duration::from_secs(3), + super::publication::reconcile(root, &operation, &snapshot, &config), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::debug!(%error, "recovered endpoint publication awaits authority"), + Err(_) => tracing::debug!("recovered endpoint publication yielded at its deadline"), + } + Ok(()) +} + +fn destinations( + root: &Path, + config: &ConsoleConfig, + trust: &MemberTrust, + snapshot: CandidateSnapshot, +) -> Result> { + let mut probes = Vec::new(); + let mut destinations = BTreeSet::new(); + for candidate in snapshot.nodes.into_iter().take(128) { + let Some(member) = trust + .members + .iter() + .find(|member| member.discovery_id == candidate.advertisement.discovery_id) + else { + continue; + }; + if candidate.advertisement.cluster_id.as_deref() != Some(trust.bootstrap.cluster_id.as_str()) { + continue; + } + for endpoint in candidate.advertisement.monitor_endpoints.iter().take(16) { + let Ok(url) = reqwest::Url::parse(endpoint) else { + continue; + }; + let Some(host) = url + .host_str() + .and_then(|host| host.trim_matches(['[', ']']).parse::().ok()) + else { + continue; + }; + if destinations.len() >= 64 || !destinations.insert((member.node_id, host)) { + continue; + } + let mut node = config + .node(member.node_id) + .cloned() + .ok_or("member topology is absent")?; + node.host = host.to_string(); + node.ssh_password = None; + node.ssh_key = Some(root.join("ssh/id_ed25519").to_string_lossy().into_owned()); + probes.push(node); + } + } + Ok(probes) +} + +fn unique(endpoints: impl Iterator) -> (Vec, Vec) { + let mut members = BTreeMap::::new(); + let mut conflicts = BTreeSet::new(); + for endpoint in endpoints { + if let Some(prior) = members.get(&endpoint.node_id) { + if prior.boot_id != endpoint.boot_id || prior.discovery_id != endpoint.discovery_id { + conflicts.insert(endpoint.discovery_id.clone()); + } + } else { + members.insert(endpoint.node_id, endpoint); + } + } + members.retain(|_, endpoint| !conflicts.contains(&endpoint.discovery_id)); + (members.into_values().collect(), conflicts.into_iter().collect()) +} + +#[cfg(feature = "test-util")] +#[must_use] +pub fn unique_member_endpoints_for_tests( + endpoints: Vec, +) -> (Vec, Vec) { + unique(endpoints.into_iter()) +} diff --git a/container/crowdb-monitor/src/node/runtime/recovery/owners.rs b/container/crowdb-monitor/src/node/runtime/recovery/owners.rs new file mode 100644 index 000000000..b1e9b0143 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/recovery/owners.rs @@ -0,0 +1,142 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Retain authorized replica owners for recovery and installation catch-up. + +use super::super::{durable, Result}; +use crowdb_console_shared::ops::OpContext; +use crowdb_protocol::key::TextKey; +use crowdb_protocol::mgmt::{ + node::recovery::{RecoveryGroup, RecoveryGroups, RecoveryReplica}, + GroupCreationStatus, SystemBootstrapIdentity, +}; +use std::{collections::BTreeMap, path::Path}; + +pub(super) async fn capture( + root: &Path, + ctx: &OpContext, + identity: &SystemBootstrapIdentity, + local: u64, +) -> Result<()> { + let groups = membership_scope(ctx).await?; + let mut captured = Vec::new(); + for ((store_id, group_id), mut members) in groups { + let Some(local_replica) = members.iter().find(|member| member.node_id == local) else { + continue; + }; + let owner = durable::read::(&root.join(format!( + "kv/node-{local}/conf/group-creation-{store_id}-{group_id}.json" + )))?; + if owner.as_ref().is_some_and(|owner| { + owner.retired + || owner.replica_id != local_replica.replica_id + || owner + .deployment + .as_ref() + .is_some_and(|deployment| deployment.cluster_id != identity.cluster_id) + }) { + continue; + } + members.sort_by_key(|member| member.replica_id); + captured.push(RecoveryGroup { + store_id, + group_id, + deployment: owner.and_then(|owner| owner.deployment), + members, + }); + } + let snapshot = RecoveryGroups { + bootstrap: identity.clone(), + groups: captured, + }; + let path = root.join("recovery-groups.json"); + let old: Option = durable::read(&path)?; + if old.as_ref().is_some_and(|old| old.bootstrap != *identity) { + return Err("recovery owner snapshot belongs to another bootstrap".into()); + } + if old.as_ref() != Some(&snapshot) { + durable::write(&path, &snapshot)?; + } + Ok(()) +} + +async fn membership_scope(ctx: &OpContext) -> Result>> { + let replicas = ctx.sysmd().list_all_replicas().await?; + let mut groups = BTreeMap::<(u64, u64), Vec<_>>::new(); + for replica in replicas { + groups + .entry((replica.store_id, replica.group_id)) + .or_default() + .push(RecoveryReplica { + node_id: replica.node_id, + replica_id: replica.replica_id, + }); + } + let memberships = ctx + .kv() + .scan_bounded( + 0, + 0, + crowdb_protocol::key::KvGroupMembersKey::prefix_all().as_bytes(), + b"", + b"", + 4096, + false, + None, + ) + .await?; + if memberships.truncated || memberships.timed_out { + return Err("recovery membership exceeds its complete observation bound".into()); + } + for (key, value) in memberships.items { + let membership: crowdb_protocol::kv_membership::GroupMembership = serde_json::from_slice(&value)?; + membership.validate()?; + let expected_key = crowdb_protocol::key::KvGroupMembersKey { + store_id: membership.store_id, + group_id: membership.group_id, + } + .to_path(); + if expected_key.as_bytes() != key.as_ref() { + return Err("recovery membership identity differs from its key".into()); + } + let mut members = membership.members; + // Joining members need routes before promotion publishes physical + // replica records. Retiring members remain reachable until fencing ends. + if let crowdb_protocol::kv_membership::GroupMembershipState::Installing { previous_members } = + membership.installation + { + for previous in previous_members { + if !members + .iter() + .any(|member| member.replica_id == previous.replica_id) + { + members.push(previous); + } + } + } + groups.insert( + (membership.store_id, membership.group_id), + members + .into_iter() + .map(|member| RecoveryReplica { + node_id: member.node_id, + replica_id: member.replica_id, + }) + .collect(), + ); + } + Ok(groups) +} + +#[cfg(feature = "test-util")] +/// Capture physical recovery membership through the current Group 0 authority. +/// # Errors +/// Propagates authority reads and private snapshot publication failures. +pub async fn capture_recovery_owners_for_tests( + root: &Path, + ctx: &OpContext, + identity: &SystemBootstrapIdentity, + local: u64, +) -> Result<()> { + capture(root, ctx, identity, local).await +} diff --git a/container/crowdb-monitor/src/node/runtime/recovery/publication.rs b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs new file mode 100644 index 000000000..dacd623ea --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/recovery/publication.rs @@ -0,0 +1,217 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Recovered authority commits address changes through the existing operation. + +use super::super::{durable, Result}; +use crowdb_console_shared::{ + deployment::{node_update, recovery::rebind_service, registry, services, PreparedBootstrap}, + ops::OpContext, + ConsoleConfig, +}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; +use crowdb_protocol::mgmt::node::{recovery::MemberEndpoints, NodeServiceAction}; +use std::{path::Path, sync::Arc}; + +pub(super) async fn reconcile( + root: &Path, + operation: &PreparedBootstrap, + endpoints: &MemberEndpoints, + topology: &ConsoleConfig, +) -> Result<()> { + let local = super::identity(root)?.endpoint; + let mut config = topology.clone(); + let mut seeds = Vec::new(); + for node in &mut config.nodes { + let Some(peer) = endpoints.members.iter().find(|peer| peer.node_id == node.id) else { + continue; + }; + let address = peer + .addresses + .iter() + .find(|address| address.is_ipv4()) + .or_else(|| peer.addresses.first()) + .ok_or("member address missing")?; + node.host = address.to_string(); + seeds.push(format!("http://{}", std::net::SocketAddr::new(*address, 10000))); + for server in config + .servers + .iter_mut() + .filter(|server| server.node_id == Some(node.id)) + { + server.url = format!("http://{}", std::net::SocketAddr::new(*address, 10000)); + server.rpc_url = Some(std::net::SocketAddr::new(*address, 10100).to_string()); + } + } + let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); + let GetOutcome::Found { value, .. } = client + .get( + 0, + 0, + crowdb_console_shared::deployment::CLUSTER_KEY, + ReadMode::Linearizable, + None, + ) + .await? + else { + return Err("cluster authority is absent".into()); + }; + let confirmed: PreparedBootstrap = serde_json::from_slice(&value)?; + confirmed.validate()?; + if confirmed.identity != operation.identity { + return Err("recovered authority belongs to another bootstrap".into()); + } + let ctx = OpContext::with_shared_client_preserving_hint(Arc::clone(&client), &seeds, config.clone()); + let (registry, _) = registry::read(&client).await?.ok_or("node registry is absent")?; + if registry.cluster_id != operation.identity.cluster_id { + return Err("recovered registry belongs to another cluster".into()); + } + let mut record = registry + .nodes + .into_iter() + .find(|node| { + node.node_id == local.node_id + && node.discovery_id == local.discovery_id + && node.confirmed + && !node.cancelled + }) + .ok_or("confirmed local identity is absent")?; + let current = config + .node(local.node_id) + .ok_or("local node topology missing")? + .host + .clone(); + if record.host != current { + record.host.clone_from(¤t); + node_update::apply(&ctx, &operation.identity.cluster_id, record).await?; + } + super::owners::capture(root, &ctx, &operation.identity, local.node_id).await?; + let complete = config + .nodes + .iter() + .all(|node| endpoints.members.iter().any(|member| member.node_id == node.id)); + if !rebind( + root, + &client, + operation, + local.node_id, + ¤t, + &seeds, + complete, + ) + .await? + { + return Ok(()); + } + if let Some(mut binding) = + durable::read::(&root.join("node-binding.json"))? + { + if binding.bootstrap != operation.identity || binding.node_id != local.node_id { + return Err("local binding changed during address publication".into()); + } + if binding.management_seeds != seeds { + binding.management_seeds = seeds; + durable::write(&root.join("node-binding.json"), &binding)?; + } + } + Ok(()) +} + +async fn rebind( + root: &Path, + client: &CrowdbKvClient, + operation: &PreparedBootstrap, + node: u64, + current: &str, + seeds: &[String], + complete: bool, +) -> Result { + let path = root.join("service-address.json"); + let old: String = match durable::read(&path)? { + Some(address) => address, + None => operation + .intent + .to_config() + .node(node) + .ok_or("original node missing")? + .host + .clone(), + }; + let binding: Option = + durable::read(&root.join("node-binding.json"))?; + // A partial discovery round is not an address change. In particular, + // never disrupt fresh services merely because one SSH proof is pending. + // Own-address changes can publish a reachable subset; later complete + // evidence refreshes the full contacts without oscillating on lost probes. + if old == current + && (!complete + || binding + .as_ref() + .is_some_and(|binding| binding.management_seeds == seeds)) + { + return Ok(false); + } + // New confirmed contacts leave all existing service seeds usable. Update + // the monitor binding without replacing healthy service launch operations. + if old == current + && binding.as_ref().is_some_and(|binding| { + !binding.management_seeds.is_empty() + && binding.management_seeds.iter().all(|seed| seeds.contains(seed)) + }) + { + durable::write(&path, ¤t)?; + return Ok(true); + } + for intent in services::list(client).await? { + if intent.node_id != node + || intent.cluster_id != operation.identity.cluster_id + || intent.kind == "kv" + || !matches!( + intent.action, + NodeServiceAction::Start | NodeServiceAction::Restart + ) + { + continue; + } + let configuration = rebind_service(&intent.configuration, old.parse()?, current.parse()?, seeds)?; + if configuration == intent.configuration { + continue; + } + let key = format!("/deployment/services/{node}/{}", intent.service_id); + let GetOutcome::Found { value, revision } = client + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await? + else { + continue; + }; + let current_intent: crowdb_protocol::mgmt::node::NodeServiceIntent = serde_json::from_slice(&value)?; + if current_intent != intent { + return Err("service intent changed during address publication".into()); + } + let mut rebound = intent; + rebound.configuration = configuration; + rebound.operation_id = uuid::Uuid::new_v4().to_string(); + client + .put_cas(0, 0, key.as_bytes(), &serde_json::to_vec(&rebound)?, revision) + .await?; + } + durable::write(&path, ¤t)?; + Ok(true) +} + +#[cfg(feature = "test-util")] +/// Exercise contact publication with no reachable authority. +/// +/// # Errors +/// Propagates any attempted authority access or retained-state failure. +pub async fn rebind_recovery_services_for_tests( + root: &Path, + operation: &PreparedBootstrap, + node: u64, + current: &str, + seeds: &[String], + complete: bool, +) -> Result { + let client = CrowdbKvClient::new(ClientConfig::new(vec!["http://127.0.0.1:9".into()])); + rebind(root, &client, operation, node, current, seeds, complete).await +} diff --git a/container/crowdb-monitor/src/node/runtime/services.rs b/container/crowdb-monitor/src/node/runtime/services.rs index 035ba95c2..2664eaae2 100644 --- a/container/crowdb-monitor/src/node/runtime/services.rs +++ b/container/crowdb-monitor/src/node/runtime/services.rs @@ -23,8 +23,13 @@ pub(super) struct Execution { mod test_support; #[cfg(feature = "test-util")] pub use test_support::NodeServiceForTests; +mod dependencies; mod execution; +pub(super) use dependencies::ready as dependencies_ready; +mod observation; +mod policy; pub(super) use execution::execute; +pub(super) use observation::observe; pub(super) fn retained(profile: &DeploymentProfile) -> Result> { let directory = profile.paths.data_root.join("services"); @@ -118,7 +123,14 @@ pub(super) fn pending( if previous.intent != *intent { return Ok(true); } + let id = if intent.kind == "kv" { + "kv" + } else { + &intent.service_id + }; + let alive = processes.owns(id) && processes.alive(id)?; if previous.attempts >= 5 + && !alive && matches!( intent.action, NodeServiceAction::Start | NodeServiceAction::Restart @@ -129,22 +141,20 @@ pub(super) fn pending( if !previous.complete { return Ok(true); } - let id = if intent.kind == "kv" { - "kv" - } else { - &intent.service_id - }; Ok(matches!( intent.action, NodeServiceAction::Start | NodeServiceAction::Restart - ) && (!processes.owns(id) || !processes.alive(id)?)) + ) && !alive) } fn bounded_error(error: &str) -> String { error.chars().take(128).collect() } -pub(super) fn failures(root: &std::path::Path) -> Result { +pub(super) fn failures( + root: &std::path::Path, + live: &std::collections::BTreeMap, +) -> Result { let directory = root.join("services"); let mut failures = serde_json::Map::new(); if !directory.exists() { @@ -157,6 +167,12 @@ pub(super) fn failures(root: &std::path::Path) -> Result { } if let Some(execution) = durable::read::(&entry.path().join("execution.json"))? { if let Some(error) = execution.error { + let process = if execution.intent.kind == "kv" { + "kv" + } else { + &execution.intent.service_id + }; + let alive = execution.pid.is_some() && execution.pid == live.get(process).copied(); if failures.len() >= 64 { return Err("service failure observation exceeds bound".into()); } @@ -164,7 +180,7 @@ pub(super) fn failures(root: &std::path::Path) -> Result { execution.intent.service_id, serde_json::json!({ "operation_id":execution.intent.operation_id, "attempts":execution.attempts, - "error":bounded_error(&error), "exhausted":execution.attempts >= 5, + "error":bounded_error(&error), "exhausted":execution.attempts >= 5 && !alive, }), ); } diff --git a/container/crowdb-monitor/src/node/runtime/services/dependencies.rs b/container/crowdb-monitor/src/node/runtime/services/dependencies.rs new file mode 100644 index 000000000..726a5e1f4 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/services/dependencies.rs @@ -0,0 +1,125 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Recovery waits for live dependencies before consuming another process launch. + +use super::super::{probes, service, Result}; +use crate::{DeploymentProfile, ProbeExecutor}; +use crowdb_protocol::mgmt::node::{NodeBinding, NodeServiceAction, NodeServiceIntent}; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +pub(super) async fn committed_ready( + profile: &DeploymentProfile, + intent: &NodeServiceIntent, + binding: &NodeBinding, +) -> Result<()> { + let kv = crowdb_kv_client::CrowdbKvClient::new(crowdb_kv_client::ClientConfig::new( + binding.management_seeds.clone(), + )); + let intents = tokio::time::timeout( + Duration::from_secs(3), + crowdb_console_shared::deployment::services::list(&kv), + ) + .await??; + ready(profile, intent, &intents).await +} + +pub(in crate::node::runtime) async fn ready( + profile: &DeploymentProfile, + intent: &NodeServiceIntent, + intents: &[NodeServiceIntent], +) -> Result<()> { + if !matches!( + intent.action, + NodeServiceAction::Start | NodeServiceAction::Restart + ) { + return Ok(()); + } + let template = service(profile, &intent.kind)?; + let executor = ProbeExecutor::new(template.dependencies.iter().any(|kind| kind == "diskio"))?; + for dependency in &template.dependencies { + // execute() independently verifies current Group-0 authority before effects. + if dependency == "kv" { + continue; + } + let dependency = intents + .iter() + .find(|candidate| { + candidate.kind == *dependency + && candidate.node_id == intent.node_id + && candidate.cluster_id == intent.cluster_id + }) + .ok_or("service dependency has no committed intent")?; + if !matches!( + dependency.action, + NodeServiceAction::Start | NodeServiceAction::Restart + ) { + return Err("service dependency is explicitly stopped".into()); + } + executor + .probe_service(&probes::for_intent(profile, dependency)?, &BTreeMap::new()) + .await?; + } + if template + .dependencies + .iter() + .any(|kind| kind == "diskdb" || kind == "diskio") + { + diskdb_routes_ready(profile).await?; + } + if template.dependencies.iter().any(|kind| kind == "diskio") { + // These clients require a complete DiskIO topology, including owners on + // other nodes. A local RPC listener alone cannot prove that precondition. + let binding: NodeBinding = super::durable::read(&profile.paths.data_root.join("node-binding.json"))? + .ok_or("node is unbound")?; + tokio::time::timeout( + Duration::from_secs(3), + crowdb_diskio_client::DiskioClient::connect(crowdb_diskio_client::DiskioClientConfig { + management_seeds: binding.management_seeds, + default_timeout: Duration::from_secs(1), + ..Default::default() + }), + ) + .await??; + } + if intent.kind == "access" { + // An idle local ChunkKV listener does not prove that the catalog owner + // has finished WAL replay. Probe the same routed read Access needs. + let binding: NodeBinding = super::durable::read(&profile.paths.data_root.join("node-binding.json"))? + .ok_or("node is unbound")?; + let kv = Arc::new(crowdb_kv_client::CrowdbKvClient::new( + crowdb_kv_client::ClientConfig::new(binding.management_seeds), + )); + let catalog = Arc::new(crowdb_chunk_kv_client::Group0ChunkKvRangeCatalogSource::from_shared(kv)); + let transport = Arc::new(crowdb_chunk_kv_client::ChunkKvRpcTransport::new(1, 1, 1)); + let client = crowdb_chunk_kv_client::ChunkKvClient::new( + crowdb_chunk_kv_client::ClientConfig { + operation_timeout: Duration::from_secs(1), + max_attempts: 1, + max_route_refreshes: 1, + max_owner_connections: 1, + ..Default::default() + }, + catalog, + transport, + )?; + let response = client.get(b"/deployment/access-readiness".to_vec(), None).await?; + if !matches!( + response.result, + Ok(crowdb_protocol::chunk_kv::OperationResult::Value(_)) + ) { + return Err("routed ChunkKV catalog is not ready".into()); + } + } + Ok(()) +} + +async fn diskdb_routes_ready(profile: &DeploymentProfile) -> Result<()> { + let binding: NodeBinding = + super::durable::read(&profile.paths.data_root.join("node-binding.json"))?.ok_or("node is unbound")?; + let kv = Arc::new(crowdb_kv_client::CrowdbKvClient::new( + crowdb_kv_client::ClientConfig::new(binding.management_seeds), + )); + crowdb_console_shared::deployment::services::diskdb_storage_ready(kv).await?; + Ok(()) +} diff --git a/container/crowdb-monitor/src/node/runtime/services/execution.rs b/container/crowdb-monitor/src/node/runtime/services/execution.rs index f580744ac..88f7d8a9c 100644 --- a/container/crowdb-monitor/src/node/runtime/services/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/services/execution.rs @@ -13,6 +13,7 @@ pub(in crate::node::runtime) async fn execute( intent: NodeServiceIntent, ) -> Result { validate(&intent)?; + super::policy::validate(&profile.paths.data_root, &intent)?; let binding: NodeBinding = durable::read(&profile.paths.data_root.join("node-binding.json"))?.ok_or("node is unbound")?; if binding.node_id != intent.node_id || binding.bootstrap.cluster_id != intent.cluster_id { @@ -29,12 +30,18 @@ pub(in crate::node::runtime) async fn execute( std::fs::create_dir_all(&directory)?; let state = directory.join("execution.json"); let mut attempts = 0; + let mut continuing = false; if let Some(previous) = durable::read::(&state)? { if previous.intent.operation_id == intent.operation_id { if previous.intent != intent { return Err("operation identity reused with other inputs".into()); } attempts = previous.attempts; + continuing = matches!( + intent.action, + NodeServiceAction::Start | NodeServiceAction::Restart + ) && processes.owns(process_id) + && processes.alive(process_id)?; if previous.complete && (previous.pid.is_none() || (processes.owns(process_id) && processes.alive(process_id)?)) { @@ -45,7 +52,11 @@ pub(in crate::node::runtime) async fn execute( if matches!( intent.action, NodeServiceAction::Start | NodeServiceAction::Restart - ) { + ) && !continuing + { + super::dependencies::committed_ready(profile, &intent, &binding) + .await + .map_err(|error| crowdb_console_shared::error::Error::DependenciesNotReady(error.to_string()))?; if attempts >= 5 { return Err( "service operation exceeded its durable restart budget; use explicit service controls".into(), @@ -63,10 +74,20 @@ pub(in crate::node::runtime) async fn execute( error: None, }, )?; - let result = apply(profile, processes, &binding, &intent, process_id, &directory).await; + let result = if continuing { + observe_ready(profile, &intent) + .await + .map(|()| processes.pid(process_id)) + } else { + apply(profile, processes, &binding, &intent, process_id, &directory).await + }; let (complete, pid, error) = match &result { Ok(pid) => (true, *pid, None), - Err(error) => (false, None, Some(bounded_error(&error.to_string()))), + Err(error) => ( + false, + processes.pid(process_id), + Some(bounded_error(&error.to_string())), + ), }; durable::write( &state, @@ -109,11 +130,15 @@ async fn apply( processes.pid("kv") } else { let mut service = service(profile, &intent.kind)?; + let physical_host = service.env.get("CROWDB_PHYSICAL_HOST_ID").cloned(); service.id.clone_from(&intent.service_id); let config = directory.join("service.toml"); durable::write_bytes(&config, intent.configuration.as_bytes())?; service.args = vec!["--config".into(), config.to_string_lossy().into_owned()]; service.env = intent.environment.clone(); + if let Some(host) = physical_host { + service.env.insert("CROWDB_PHYSICAL_HOST_ID".into(), host); + } let logs = profile.paths.log_root.join(&intent.service_id); if matches!(intent.kind.as_str(), "diskdb" | "chunkdb" | "chunk-kv") { service @@ -144,6 +169,16 @@ async fn apply( Ok(pid) } +// A live process has already had its initial startup window. Later health +// observations must yield after one probe so control requests can make progress. +async fn observe_ready(profile: &DeploymentProfile, intent: &NodeServiceIntent) -> Result<()> { + let service = super::super::probes::for_intent(profile, intent)?; + crate::ProbeExecutor::new(intent.kind == "diskio")? + .probe_service(&service, &std::collections::BTreeMap::new()) + .await + .map_err(|error| format!("managed service {} readiness failed: {error}", intent.service_id).into()) +} + async fn wait_ready( profile: &DeploymentProfile, processes: &mut ProcessManager, diff --git a/container/crowdb-monitor/src/node/runtime/services/observation.rs b/container/crowdb-monitor/src/node/runtime/services/observation.rs new file mode 100644 index 000000000..1fff3fc7b --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/services/observation.rs @@ -0,0 +1,120 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Pair health with its executed operation before exposing local evidence. + +use super::{durable, Execution, Result}; +use crate::{DeploymentProfile, MonitorStatus}; +use crowdb_protocol::mgmt::{ + cluster::DependencyState, + node::{observation::NodeServiceObservation, NodeServiceAction}, +}; +use std::collections::BTreeMap; + +pub(in crate::node::runtime) fn observe( + profile: &DeploymentProfile, + live: &BTreeMap, + health: Option<&MonitorStatus>, + now: u64, +) -> Result> { + let mut observed = Vec::new(); + let directory = profile.paths.data_root.join("services"); + if directory.try_exists()? { + for entry in std::fs::read_dir(directory)? { + let entry = entry?; + if !entry.file_type()?.is_dir() { + continue; + } + if let Some(execution) = durable::read::(&entry.path().join("execution.json"))? { + observed.push(executed(&execution, live, health, now)); + } + if observed.len() > 64 { + return Err("local service observation exceeds bound".into()); + } + } + } + if !observed.iter().any(|service| service.kind == "kv") { + let status = health.and_then(|health| health.services.get("kv")); + observed.push(NodeServiceObservation { + service_id: "kv".into(), + kind: "kv".into(), + operation_id: None, + state: if status.is_some_and(|status| { + status.healthy && status.pid == live.get("kv").copied() && status.pid.is_some() + }) { + DependencyState::Ready + } else { + DependencyState::Unknown + }, + observed_at_unix_ms: health.map_or(now, |health| health.updated_at_ms), + max_age_ms: 3000, + reason_code: None, + detail: "KV health must match the currently supervised process and observation timestamp".into(), + }); + } + observed.sort_by(|left, right| left.service_id.cmp(&right.service_id)); + Ok(observed) +} + +fn executed( + execution: &Execution, + live: &BTreeMap, + health: Option<&MonitorStatus>, + now: u64, +) -> NodeServiceObservation { + let intent = &execution.intent; + let process = if intent.kind == "kv" { + "kv" + } else { + &intent.service_id + }; + let current = health + .and_then(|health| health.services.get(process)) + .filter(|status| { + status.operation_id.as_ref() == Some(&intent.operation_id) + && status.pid.is_some() + && status.pid == live.get(process).copied() + }); + let state = if execution.complete + && matches!(intent.action, NodeServiceAction::Stop | NodeServiceAction::Delete) + { + DependencyState::Stopped + } else if !execution.complete + && execution.attempts >= 5 + && execution.error.is_some() + && !live.contains_key(process) + { + DependencyState::Failed + } else if execution.complete && current.is_some_and(|status| status.healthy) { + DependencyState::Ready + } else { + DependencyState::Initializing + }; + NodeServiceObservation { + detail: format!( + "Service {}: {:?}; executed operation {}; complete {}; launches {}; live process {}", + intent.service_id, + state, + intent.operation_id, + execution.complete, + execution.attempts, + live.contains_key(process) + ), + service_id: intent.service_id.clone(), + kind: intent.kind.clone(), + operation_id: Some(intent.operation_id.clone()), + state, + observed_at_unix_ms: if state == DependencyState::Ready { + health.map_or(now, |health| health.updated_at_ms) + } else { + now + }, + max_age_ms: 3000, + reason_code: match state { + DependencyState::Failed => Some("service-restart-budget-exhausted".into()), + DependencyState::Stopped => Some("operator-stopped-service".into()), + DependencyState::Ready => None, + _ => Some("service-health-not-current".into()), + }, + } +} diff --git a/container/crowdb-monitor/src/node/runtime/services/policy.rs b/container/crowdb-monitor/src/node/runtime/services/policy.rs new file mode 100644 index 000000000..1a80ee4d2 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/services/policy.rs @@ -0,0 +1,45 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::path::Path; + +use crowdb_protocol::mgmt::node::{deployment::DeploymentMode, NodeServiceAction, NodeServiceIntent}; + +use super::Result; + +pub(super) fn validate(root: &Path, intent: &NodeServiceIntent) -> Result<()> { + if !matches!( + intent.action, + NodeServiceAction::Start | NodeServiceAction::Restart + ) { + return Ok(()); + } + let Some(policy) = crate::node::read_node_policy(root)? else { + return Ok(()); + }; + if policy.mode != DeploymentMode::Production { + return Ok(()); + } + let config: toml::Value = + toml::from_str(&intent.configuration).map_err(|_| "invalid managed service configuration")?; + if matches!(intent.kind.as_str(), "chunkdb" | "access") { + let mode = config + .get("deployment") + .and_then(|deployment| deployment.get("mode")) + .and_then(toml::Value::as_str) + .unwrap_or("production"); + if mode != "production" { + return Err("production node policy forbids local or unsafe service deployment".into()); + } + } + if intent.kind == "chunk-kv" + && config + .get("storage") + .and_then(|storage| storage.get("stream_mirror_copies")) + .and_then(toml::Value::as_integer) + .is_some_and(|copies| copies < 2) + { + return Err("production node policy requires protected Chunk-KV streams".into()); + } + Ok(()) +} diff --git a/container/crowdb-monitor/src/node/runtime/services/test_support.rs b/container/crowdb-monitor/src/node/runtime/services/test_support.rs index 67ede7934..cf0d68968 100644 --- a/container/crowdb-monitor/src/node/runtime/services/test_support.rs +++ b/container/crowdb-monitor/src/node/runtime/services/test_support.rs @@ -9,9 +9,20 @@ use super::{ pub struct NodeServiceForTests { profile: DeploymentProfile, processes: ProcessManager, + health: Option, } impl NodeServiceForTests { + /// # Errors + /// Runs the production dependency probes without launching a process. + pub async fn dependencies_ready_for_tests( + profile: &DeploymentProfile, + intent: &NodeServiceIntent, + intents: &[NodeServiceIntent], + ) -> Result<()> { + super::dependencies_ready(profile, intent, intents).await + } + /// # Errors /// Propagates production namespace validation and bounded read errors. pub fn processes_for_tests( @@ -27,7 +38,47 @@ impl NodeServiceForTests { /// Propagates process log setup errors. pub async fn new_for_tests(profile: DeploymentProfile) -> Result { let processes = ProcessManager::new(profile.paths.log_root.clone(), profile.logs.clone()).await?; - Ok(Self { profile, processes }) + Ok(Self { + profile, + processes, + health: None, + }) + } + + /// Resume retained KV ownership without waiting for a healthy quorum. + /// + /// # Errors + /// Propagates production retained-trust and process startup errors. + pub async fn resume_for_tests(&mut self) -> Result<()> { + super::super::kv::resume(&self.profile, &mut self.processes).await + } + + /// # Errors + /// Propagates production recovery and current health observation errors. + pub async fn tick_for_tests(&mut self) -> Result<()> { + let web = super::super::service(&self.profile, "web")?; + if !self.processes.owns("web") { + self.processes + .start(&web, &std::collections::BTreeMap::new()) + .await?; + } + if self.health.is_none() { + self.health = Some(super::super::health::NodeHealth::new(&self.profile)?); + } + self.health + .as_mut() + .ok_or("missing test health worker")? + .tick(&self.profile, &web, &mut self.processes) + .await + } + + /// # Errors + /// Propagates failure to stop the owned KV child. + pub async fn stop_kv_for_tests(&mut self) -> Result<()> { + self.processes + .stop("kv", std::time::Duration::from_secs(1)) + .await?; + Ok(()) } /// # Errors @@ -36,6 +87,16 @@ impl NodeServiceForTests { execute(&self.profile, &mut self.processes, intent).await } + /// Reconcile committed recovery intents through the production dependency gate. + /// + /// # Errors + /// Propagates authority, journal and process observation errors. + pub async fn reconcile_for_tests(&mut self) -> Result<()> { + super::super::health::NodeHealth::new(&self.profile)? + .reconcile_services(&self.profile, &mut self.processes) + .await + } + /// # Errors /// Propagates production journal and process observation errors. pub fn pending_for_tests(&mut self, intent: &NodeServiceIntent) -> Result { @@ -44,7 +105,7 @@ impl NodeServiceForTests { /// # Errors /// Propagates bounded failure observation errors. - pub fn failures_for_tests(&self) -> Result { - failures(&self.profile.paths.data_root) + pub fn failures_for_tests(&mut self) -> Result { + failures(&self.profile.paths.data_root, &self.processes.live_pids()?) } } diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution.rs b/container/crowdb-monitor/src/node/runtime/setup/execution.rs index b2f7eb63b..720ed0ed4 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution.rs @@ -13,6 +13,7 @@ use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; use crowdb_protocol::mgmt::node::{setup::NodeSetupPhase, NodeBinding}; use std::time::Duration; +mod metadata; mod services; mod storage; mod supplied; @@ -45,10 +46,9 @@ async fn advance(profile: &DeploymentProfile, state: &mut AcceptedSetup) -> Resu binding.bootstrap.operation_id )) .exists() - || state - .binding - .as_ref() - .is_some_and(|accepted| accepted != &binding) + || state.binding.as_ref().is_some_and(|accepted| { + accepted.node_id != binding.node_id || accepted.bootstrap != binding.bootstrap + }) || !crate::node::trust::authorized_operation(root, &state.request.operation_id)? { state.status.phase = NodeSetupPhase::Failed; @@ -78,10 +78,37 @@ async fn advance(profile: &DeploymentProfile, state: &mut AcceptedSetup) -> Resu state.status.cluster_id = Some(binding.bootstrap.cluster_id.clone()); state.status.phase = NodeSetupPhase::Running; journal::save(root, state)?; + let choices = &state.request.choices; + let preparation_needed = !matches!( + choices.storage, + crowdb_protocol::mgmt::node::setup::SetupStorage::Diskless + ) || choices.services.iter().any(|service| service.enabled); + if preparation_needed + && crate::node::read_node_policy(root)?.is_some() + && !metadata::ready(root, &binding, &kv).await? + { + return Ok(()); + } // Storage and service prerequisites progress independently. In particular, // file preparation may wait for DiskIO while DiskIO is still starting. let disk_ready = storage::advance(profile, state, &binding, &kv).await; - let services_ready = services::advance(profile, state, &binding, &kv).await; + if state + .status + .disks + .iter() + .any(|step| step.phase == crowdb_protocol::mgmt::node::setup::SetupStepPhase::Failed) + { + state.status.phase = NodeSetupPhase::Failed; + return disk_ready.map(|_| ()); + } + let services_ready = services::advance( + profile, + state, + &binding, + &kv, + disk_ready.as_ref().is_ok_and(|ready| *ready), + ) + .await; let services_ready = services_ready?; if state .status diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/metadata.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/metadata.rs new file mode 100644 index 000000000..92a24e089 --- /dev/null +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/metadata.rs @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Default metadata precedes both disk admission and planned service startup. + +use std::sync::Arc; + +use crowdb_console_shared::{ + deployment::preparation, + ops::{kv_logical, OpContext}, + ConsoleConfig, +}; +use crowdb_kv_client::CrowdbKvClient; +use crowdb_protocol::mgmt::node::NodeBinding; + +use super::Result; + +pub(super) async fn ready( + root: &std::path::Path, + binding: &NodeBinding, + kv: &Arc, +) -> Result { + let ctx = OpContext::with_shared_client_preserving_hint( + Arc::clone(kv), + &binding.management_seeds, + ConsoleConfig::default(), + ); + let accepted = preparation::reserve(&ctx, &binding.bootstrap).await?; + if let Some(mut groups) = crate::node::durable::read::< + crowdb_protocol::mgmt::node::recovery::RecoveryGroups, + >(&root.join("recovery-groups.json"))? + { + if groups.bootstrap != binding.bootstrap { + return Err("recovery group binding differs".into()); + } + let group = crowdb_protocol::mgmt::node::recovery::RecoveryGroup { + store_id: accepted.metadata.store_id, + group_id: accepted.metadata.group_id, + deployment: Some(crowdb_protocol::mgmt::GroupDeploymentIdentity { + cluster_id: binding.bootstrap.cluster_id.clone(), + operation_id: accepted.operation_id.clone(), + }), + members: accepted + .metadata + .replicas + .iter() + .map(|replica| crowdb_protocol::mgmt::node::recovery::RecoveryReplica { + replica_id: replica.replica_id, + node_id: replica.node_id, + }) + .collect(), + }; + if !groups.groups.contains(&group) { + groups.groups.retain(|existing| { + (existing.store_id, existing.group_id) != (group.store_id, group.group_id) + }); + groups.groups.push(group); + crate::node::durable::write(&root.join("recovery-groups.json"), &groups)?; + } + } + Ok(kv_logical::ensure_default_group(&ctx, &accepted).await?) +} diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs index 3c23b8b19..9304d989c 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/services.rs @@ -19,6 +19,7 @@ pub(super) async fn advance( state: &mut AcceptedSetup, binding: &NodeBinding, kv: &CrowdbKvClient, + storage_ready: bool, ) -> Result { if state.status.services.iter().all(|step| { matches!( @@ -74,6 +75,16 @@ pub(super) async fn advance( } continue; } + if !storage_ready + && matches!( + kind, + SetupServiceKind::Chunkdb | SetupServiceKind::ChunkKv | SetupServiceKind::AccessServer + ) + { + step.phase = SetupStepPhase::Waiting; + step.detail = Some("Waiting for accepted disks to finish registration".into()); + return Ok(false); + } if step.operation_id.is_none() { step.operation_id = Some(uuid::Uuid::new_v4().to_string()); } diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/storage.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/storage.rs index d735d2aaf..0eae8f9e7 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution/storage.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/storage.rs @@ -5,7 +5,7 @@ use super::{journal, AcceptedSetup, Result}; use crate::DeploymentProfile; -use crowdb_kv_client::{CrowdbKvClient, HardwareClient, KVClusterMetaClient}; +use crowdb_kv_client::{CrowdbKvClient, HardwareClient}; use crowdb_protocol::mgmt::node::{ setup::{SetupDiskStatus, SetupStepPhase, SetupStorage}, storage::{DiskInput, FileDiskAdmission, FilePreparationPhase, PrepareDiskFile}, @@ -47,9 +47,10 @@ pub(super) async fn advance( &hardware, kv, rack, - binding.node_id, + binding, None, &std::collections::BTreeSet::new(), + true, ) .await?; state.storage_binding = Some(data_binding); @@ -82,19 +83,33 @@ pub(super) async fn advance( state.storage_binding.clone(), ) .await?; + advance_file(profile, state, request).await +} + +async fn advance_file( + profile: &DeploymentProfile, + state: &mut AcceptedSetup, + request: PrepareDiskFile, +) -> Result { + let root = &profile.paths.data_root; let status = crate::node::storage::file_preparation::status(root, &request.operation_id); + let retry = state.retry_disks.remove(&request.operation_id); let ready = status .as_ref() .is_ok_and(|status| status.phase == FilePreparationPhase::Registered); state.status.disks = vec![disk_status(profile, &request, status.as_ref().ok(), ready)?]; if !ready { - if status - .as_ref() - .is_ok_and(|status| status.phase == FilePreparationPhase::Failed) + if !retry + && status + .as_ref() + .is_ok_and(|status| status.phase == FilePreparationPhase::Failed) { state.status.disks[0].phase = SetupStepPhase::Failed; return Err("default file preparation failed; explicit retry required".into()); } + // Persist consumption before dispatch. Reconcile the same operation on + // a lost reply; a terminal failure needs another explicit retry. + journal::save(root, state)?; super::super::super::control_node( &profile.paths.run_root.join("node-control.sock"), &NodeControl::PrepareDiskFile { request }, @@ -117,6 +132,14 @@ pub(super) async fn ensure_group( ); let groups = crowdb_console_shared::ops::hardware::list_disk_groups_from_group0(&ctx, binding.node_id).await?; + let metadata = + crowdb_console_shared::deployment::preparation::metadata_binding(&ctx, &binding.bootstrap).await?; + if !data_binding + .as_ref() + .is_some_and(|data| data.store_id == metadata.store_id && data.group_id == metadata.group_id) + { + return Err("managed disk metadata differs from accepted default binding".into()); + } if let Some(group) = groups.iter().find(|group| group.id == group_id) { let current = ctx .sysmd() @@ -148,14 +171,24 @@ pub(super) async fn select_group( hardware: &HardwareClient, kv: &std::sync::Arc, rack: u64, - node: u64, + binding: &NodeBinding, media_type: Option, excluded: &std::collections::BTreeSet, + automatic: bool, ) -> Result<(u64, crowdb_protocol::common::BindMapValue)> { + let node = binding.node_id; + let ctx = crowdb_console_shared::ops::OpContext::with_shared_client_preserving_hint( + std::sync::Arc::clone(kv), + &binding.management_seeds, + crowdb_console_shared::ConsoleConfig::default(), + ); + let accepted = + crowdb_console_shared::deployment::preparation::storage_binding(&ctx, &binding.bootstrap, rack, node) + .await?; let mut groups = hardware.list_disk_groups_on_node(rack, node).await?; groups.sort_unstable_by_key(|group| group.dg_id); for group in groups { - if excluded.contains(&group.dg_id) { + if excluded.contains(&group.dg_id) || automatic && group.dg_id != accepted.disk_group_id { continue; } match hardware @@ -166,12 +199,15 @@ pub(super) async fn select_group( Err(crowdb_kv_client::Error::Server(reason)) if reason == "DiskGroup media class or allocation unit is incompatible with this disk" => { - continue + if automatic { + return Err("reserved disk group has incompatible media geometry".into()); + } + continue; } Err(error) => return Err(error.into()), } if let Some(binding) = hardware.get_bind(rack, node, group.dg_id).await? { - if binding.group_id != 0 { + if binding.store_id == accepted.store_id && binding.group_id == accepted.group_id { return Ok(( group.dg_id, crowdb_protocol::common::BindMapValue { @@ -179,22 +215,23 @@ pub(super) async fn select_group( group_id: binding.group_id, }, )); + } else if automatic { + return Err("reserved disk group has a conflicting metadata binding".into()); } + } else if automatic { + return Err("reserved disk group has no accepted metadata binding".into()); } } - let group = KVClusterMetaClient::from_shared(std::sync::Arc::clone(kv)) - .list_all_groups() - .await? - .into_iter() - .filter(|group| group.group_id != 0) - .min_by_key(|group| (group.store_id, group.group_id)) - .ok_or("default disk is waiting for an existing ordinary metadata group")?; - let id = hardware.allocate_disk_group_id(rack, node).await?; + let id = if automatic { + accepted.disk_group_id + } else { + hardware.allocate_disk_group_id(rack, node).await? + }; Ok(( id, crowdb_protocol::common::BindMapValue { - store_id: group.store_id, - group_id: group.group_id, + store_id: accepted.store_id, + group_id: accepted.group_id, }, )) } diff --git a/container/crowdb-monitor/src/node/runtime/setup/execution/supplied.rs b/container/crowdb-monitor/src/node/runtime/setup/execution/supplied.rs index 578328a41..08367ef76 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/execution/supplied.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/execution/supplied.rs @@ -55,7 +55,7 @@ pub(super) async fn advance( { return Err("supplied preparation differs from sealed node setup".into()); } - let retry = state.retry_supplied.remove(&record.operation_id); + let retry = state.retry_disks.remove(&record.operation_id); let mut phase = match current.as_ref().map(|current| current.phase) { Some(SuppliedPreparationPhase::Registered) => SetupStepPhase::Ready, Some(SuppliedPreparationPhase::Failed) if !retry => SetupStepPhase::Failed, @@ -141,9 +141,10 @@ async fn prepare( &hardware, kv, node.rack_id, - binding.node_id, + binding, medium.media_type, &excluded, + false, ) .await?; excluded.insert(group.0); diff --git a/container/crowdb-monitor/src/node/runtime/setup/journal.rs b/container/crowdb-monitor/src/node/runtime/setup/journal.rs index 2348e1377..1d2060d34 100644 --- a/container/crowdb-monitor/src/node/runtime/setup/journal.rs +++ b/container/crowdb-monitor/src/node/runtime/setup/journal.rs @@ -30,7 +30,7 @@ pub(super) struct AcceptedSetup { #[serde(default)] pub(super) supplied: Vec, #[serde(default)] - pub(super) retry_supplied: std::collections::BTreeSet, + pub(super) retry_disks: std::collections::BTreeSet, } #[derive(Clone, Serialize, Deserialize)] @@ -44,6 +44,8 @@ pub(super) fn read(root: &Path) -> Result> { let current: Option = durable::read(&root.join("node-setup.json"))?; if let Some(current) = ¤t { current.request.validate()?; + validate_policy(root, ¤t.request.choices)?; + validate_retries(current)?; supplied::validate(current)?; validate_status(¤t.status)?; for identity in [¤t.request.discovery_id, ¤t.request.operation_id] { @@ -63,12 +65,28 @@ pub(super) fn read(root: &Path) -> Result> { } pub(super) fn save(root: &Path, state: &AcceptedSetup) -> Result<()> { + validate_retries(state)?; supplied::validate(state)?; validate_status(&state.status)?; durable::write(&root.join("node-setup.json"), state)?; Ok(()) } +fn validate_retries(state: &AcceptedSetup) -> Result<()> { + if state.retry_disks.len() > 64 + || state.retry_disks.iter().any(|id| { + !state.file.as_ref().is_some_and(|file| file.operation_id == *id) + && !state + .supplied + .iter() + .any(|disk| disk.request.record.operation_id == *id) + }) + { + return Err("retry references an unselected disk operation".into()); + } + Ok(()) +} + fn validate_status(status: &NodeSetupStatus) -> Result<()> { if serde_json::to_vec(status)?.len() > 60 * 1024 { return Err("node setup progress exceeds its control reply bound".into()); @@ -102,6 +120,7 @@ pub(super) fn accept_with_inputs( inputs: &[DiskInput], ) -> Result { request.validate()?; + validate_policy(root, &request.choices)?; if root .join(format!( "node-setup-retired-{}.json", @@ -166,12 +185,22 @@ pub(super) fn accept_with_inputs( storage_binding: None, supplied_observations, supplied: Vec::new(), - retry_supplied: std::collections::BTreeSet::new(), + retry_disks: std::collections::BTreeSet::new(), }, )?; Ok(status) } +fn validate_policy( + root: &Path, + choices: &crowdb_protocol::mgmt::node::setup::NodeSetupChoices, +) -> Result<()> { + if crate::node::read_node_policy(root)?.is_some_and(|policy| policy.setup != *choices) { + return Err("node setup choices differ from accepted deployment policy".into()); + } + Ok(()) +} + fn replay(root: &Path, current: &mut AcceptedSetup, running: bool) -> Result { if current.status.phase == NodeSetupPhase::Failed && !running { current.status.phase = NodeSetupPhase::Accepted; @@ -184,7 +213,7 @@ fn replay(root: &Path, current: &mut AcceptedSetup, running: bool) -> Result Result<()> { - if state.supplied.len() > 64 || state.supplied_observations.len() > 64 || state.retry_supplied.len() > 64 - { + if state.supplied.len() > 64 || state.supplied_observations.len() > 64 { return Err("sealed supplied plan exceeds bound".into()); } if state.supplied.is_empty() { @@ -67,8 +66,5 @@ pub(super) fn validate(state: &AcceptedSetup) -> Result<()> { return Err("sealed medium identity, placement or authorization changed".into()); } } - if state.retry_supplied.iter().any(|id| !operations.contains(id)) { - return Err("retry references an unselected supplied operation".into()); - } Ok(()) } diff --git a/container/crowdb-monitor/src/node/single.rs b/container/crowdb-monitor/src/node/single.rs deleted file mode 100644 index 100633118..000000000 --- a/container/crowdb-monitor/src/node/single.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{serve_node_management, NodeRuntimeConfig}; - -/// Run the same node discovery beside the automatic disk/topology startup policy. -/// -/// # Errors -/// Propagates bootstrap, discovery and persistent identity failures. -pub async fn run_single_node( - config: NodeRuntimeConfig, -) -> Result<(), Box> { - let profile = crate::DeploymentProfile::load(&config.profile)?; - crate::preview_step_names(&profile)?; - super::startup::accept(&profile.paths.data_root, super::startup::Mode::Single)?; - super::NodeIdentity::load_or_create(&profile.paths.data_root)?; - let (stop, stopped) = tokio::sync::oneshot::channel(); - let root = profile.paths.data_root; - let mut discovery = tokio::spawn(async move { - serve_node_management( - &root, - config.bind, - &config.discovery, - config.physical_host_id, - async { - let _ = stopped.await; - }, - ) - .await - }); - let result = tokio::select! { - result = crate::run_preview(&config.profile) => result.map_err(|error| error.to_string().into()), - result = &mut discovery => { result??; Err("node discovery unexpectedly stopped".into()) }, - }; - let _ = stop.send(()); - if !discovery.is_finished() { - discovery.await??; - } - result -} diff --git a/container/crowdb-monitor/src/node/startup.rs b/container/crowdb-monitor/src/node/startup.rs index ce1bb5c15..8c451a7ae 100644 --- a/container/crowdb-monitor/src/node/startup.rs +++ b/container/crowdb-monitor/src/node/startup.rs @@ -9,6 +9,9 @@ use serde::{Deserialize, Serialize}; use super::durable; +mod policy; +pub use policy::{accept_node_policy, read_node_policy}; + #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub(crate) enum Mode { @@ -27,6 +30,11 @@ pub(crate) fn validate(root: &Path, mode: Mode) -> io::Result<()> { if !crate::layout::is_clean_absolute(root) || !std::fs::symlink_metadata(root)?.file_type().is_dir() { return Err(io::Error::other("startup root must be an absolute directory")); } + if root.join("node-policy.json").try_exists()? { + return Err(io::Error::other( + "accepted node policy cannot use the obsolete startup mode", + )); + } if let Some(current) = durable::read::(&root.join("startup-mode.json"))? { if current.version != 1 || current.mode != mode { return Err(io::Error::other("startup mode differs from durable node policy")); @@ -63,6 +71,7 @@ pub(crate) fn validate(root: &Path, mode: Mode) -> io::Result<()> { Ok(()) } +#[cfg(feature = "test-util")] pub(crate) fn accept(root: &Path, mode: Mode) -> io::Result<()> { validate(root, mode)?; let path = root.join("startup-mode.json"); diff --git a/container/crowdb-monitor/src/node/startup/policy.rs b/container/crowdb-monitor/src/node/startup/policy.rs new file mode 100644 index 000000000..9ef1767b6 --- /dev/null +++ b/container/crowdb-monitor/src/node/startup/policy.rs @@ -0,0 +1,126 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Publish one complete policy before creating identity or provisioning storage. + +use std::{fs, io, path::Path}; + +use crowdb_protocol::mgmt::node::{ + deployment::{NodeDeploymentOverrides, NodeDeploymentPolicy}, + setup::SetupStorage, +}; + +use crate::node::durable; + +const POLICY: &str = "node-policy.json"; + +/// Read accepted inputs without deriving replacement defaults. +/// +/// # Errors +/// Rejects corrupt, unsupported or non-regular policy records. +pub fn read_node_policy(root: &Path) -> io::Result> { + let policy = durable::read::(&root.join(POLICY))?; + if let Some(policy) = &policy { + policy.validate().map_err(io::Error::other)?; + } + Ok(policy) +} + +/// Seal inputs atomically; concurrent publishers cannot overwrite the winner. +/// +/// # Errors +/// Rejects conflicting accepted inputs and unrecognized populated roots. +pub fn accept_node_policy( + root: &Path, + file: &NodeDeploymentOverrides, + explicit: &NodeDeploymentOverrides, +) -> io::Result { + if !crate::layout::is_clean_absolute(root) || !fs::symlink_metadata(root)?.is_dir() { + return Err(io::Error::other("node policy root must be an absolute directory")); + } + let accepted = read_node_policy(root)?; + let selected = explicit + .resolve(file, accepted.as_ref()) + .map_err(io::Error::other)?; + if accepted.is_some() { + validate_retained_identity(root)?; + return Ok(selected); + } + validate_fresh_root(root, &selected)?; + let temporary = root.join(format!(".node-policy-{}.json", uuid::Uuid::new_v4())); + durable::write(&temporary, &selected)?; + let outcome = fs::hard_link(&temporary, root.join(POLICY)); + fs::remove_file(&temporary)?; + match outcome { + Ok(()) => fs::File::open(root)?.sync_all()?, + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error), + } + let winner = read_node_policy(root)?.ok_or_else(|| io::Error::other("accepted node policy missing"))?; + if winner != selected { + return Err(io::Error::other( + "concurrent startup accepted another node policy", + )); + } + Ok(winner) +} + +fn validate_retained_identity(root: &Path) -> io::Result<()> { + if root.join("node-identity").try_exists()? { + return Ok(()); + } + for name in [ + "accepted-node.json", + "node-binding.json", + "initialization.json", + "node-setup.json", + ] { + if root.join(name).try_exists()? { + return Err(io::Error::other( + "retained deployment has no node identity; refusing replacement identity", + )); + } + } + Ok(()) +} + +fn validate_fresh_root(root: &Path, policy: &NodeDeploymentPolicy) -> io::Result<()> { + for entry in fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name(); + let name = name + .to_str() + .ok_or_else(|| io::Error::other("unrecognized node data path"))?; + if name == POLICY { + // A concurrent publisher may already have completed validation. + let current = read_node_policy(root)?.ok_or_else(|| io::Error::other("node policy missing"))?; + if current != *policy { + return Err(io::Error::other( + "concurrent startup accepted another node policy", + )); + } + continue; + } + if temporary_policy(name) && entry.file_type()?.is_file() { + continue; + } + if matches!(name, "ssh" | "disks" | "lost+found") + && entry.file_type()?.is_dir() + && (name != "disks" + || matches!(policy.setup.storage, SetupStorage::Supplied { .. }) + || fs::read_dir(entry.path())?.next().is_none()) + { + continue; + } + return Err(io::Error::other(format!( + "unrecognized populated node root ({name}); use a fresh volume without removing retained data" + ))); + } + Ok(()) +} + +fn temporary_policy(name: &str) -> bool { + name.strip_prefix(".node-policy-") + .and_then(|suffix| suffix.strip_suffix(".json")) + .is_some_and(|id| uuid::Uuid::parse_str(id).is_ok_and(|uuid| uuid.to_string() == id)) +} diff --git a/container/crowdb-monitor/src/node/trust.rs b/container/crowdb-monitor/src/node/trust.rs index 2d0346d25..d97a9a9d5 100644 --- a/container/crowdb-monitor/src/node/trust.rs +++ b/container/crowdb-monitor/src/node/trust.rs @@ -63,6 +63,26 @@ fn read_keys(path: &Path) -> std::io::Result { } } +pub(super) fn pinned_host(root: &Path, host: &str, port: u16) -> std::io::Result { + let token = if port == 22 { + host.to_owned() + } else { + format!("[{host}]:{port}") + }; + let text = read_keys(&root.join("ssh/known_hosts"))?; + let mut pins = text.lines().filter_map(|line| { + let (name, key) = line.split_once(' ')?; + (name == token).then_some(key) + }); + let key = pins + .next() + .ok_or_else(|| std::io::Error::other("missing authenticated peer host key"))?; + if pins.next().is_some() { + return Err(std::io::Error::other("ambiguous authenticated peer host key")); + } + Ok(key.to_owned()) +} + pub(super) fn authorized_operation(root: &Path, operation: &str) -> std::io::Result { let operation = uuid::Uuid::parse_str(operation).map_err(std::io::Error::other)?; if root.join(format!("ssh/cancelled-{operation}.json")).exists() { diff --git a/container/crowdb-monitor/src/node_args.rs b/container/crowdb-monitor/src/node_args.rs new file mode 100644 index 000000000..ef401d7e9 --- /dev/null +++ b/container/crowdb-monitor/src/node_args.rs @@ -0,0 +1,149 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Resolve startup sources once before handing accepted inputs to the node owner. + +use std::{fs, io::Read, path::PathBuf}; + +use clap::{Args, ValueEnum}; +use crowdb_monitor::{DeploymentProfile, DiscoveryConfig, NodeRuntimeConfig}; +use crowdb_protocol::mgmt::node::deployment::{ + DeploymentMode, NodeDeploymentOverrides, NodeDeploymentPolicy, PublicAccessUris, +}; + +#[derive(Clone, Copy, Debug, ValueEnum)] +enum Mode { + Local, + Production, +} + +#[derive(Debug, Args)] +pub(crate) struct NodeRunArgs { + #[arg(long, default_value = "/opt/crowdb/etc/profile.toml")] + profile: PathBuf, + #[arg(long, default_value = "eth0")] + interface: Vec, + #[arg(long, env = "CROWDB_PHYSICAL_HOST_ID")] + physical_host_id: Option, + /// Optional TOML startup inputs; explicit arguments/environment take precedence. + #[arg(long, env = "CROWDB_NODE_POLICY_FILE")] + deployment_config: Option, + #[arg(long, value_enum, env = "CROWDB_DEPLOYMENT_MODE")] + deployment_mode: Option, + #[arg(long, env = "CROWDB_CLUSTER")] + cluster: Option, + /// Initial voting count; accepted membership is never automatically resized. + #[arg(long, env = "CROWDB_INITIAL_VOTERS")] + initial_voters: Option, + #[arg(long, env = "CROWDB_S3_PUBLIC_URI")] + s3_public_uri: Option, + #[arg(long, env = "CROWDB_ICEBERG_PUBLIC_URI")] + iceberg_public_uri: Option, + #[arg(long, env = "CROWDB_DATASET_PUBLIC_URI")] + dataset_public_uri: Option, +} + +pub(crate) async fn run(args: NodeRunArgs) -> Result<(), Box> { + let inputs = resolve(&args)?; + let addresses = if_addrs::get_if_addrs()? + .into_iter() + .filter(|address| args.interface.contains(&address.name)) + .map(|address| address.ip()) + .collect(); + let config = NodeRuntimeConfig { + profile: args.profile, + bind: "0.0.0.0:9095".parse()?, + discovery: DiscoveryConfig { + interfaces: args.interface, + addresses, + monitor_port: 9095, + cluster_id: None, + seeds: discovery_seeds(), + }, + physical_host_id: inputs.physical_host_id, + }; + let result = crowdb_monitor::run_node_with_inputs(config, &inputs.file, &inputs.explicit).await; + result.map_err(|error| error.to_string().into()) +} + +pub(crate) fn accept(args: &NodeRunArgs) -> Result<&'static str, Box> { + let inputs = resolve(args)?; + let policy = crowdb_monitor::accept_node_policy(&inputs.data_root, &inputs.file, &inputs.explicit)?; + Ok(match policy.mode { + DeploymentMode::Local => "local", + DeploymentMode::Production => "production", + }) +} + +struct ResolvedInputs { + data_root: PathBuf, + file: NodeDeploymentOverrides, + explicit: NodeDeploymentOverrides, + physical_host_id: String, +} + +fn resolve(args: &NodeRunArgs) -> Result> { + DiscoveryConfig::validate_seed_origins(&discovery_seeds())?; + let file = read_inputs(args.deployment_config.as_deref())?; + let explicit = NodeDeploymentOverrides { + mode: args.deployment_mode.map(|mode| match mode { + Mode::Local => DeploymentMode::Local, + Mode::Production => DeploymentMode::Production, + }), + cluster: args.cluster.clone(), + initial_voters: args.initial_voters, + public_access: PublicAccessUris { + s3: args.s3_public_uri.clone(), + iceberg: args.iceberg_public_uri.clone(), + dataset: args.dataset_public_uri.clone(), + }, + ..Default::default() + }; + let profile = DeploymentProfile::load(&args.profile)?; + profile.validate_node_catalog()?; + let accepted = crowdb_monitor::read_node_policy(&profile.paths.data_root)?; + let selected = explicit.resolve(&file, accepted.as_ref())?; + let physical_host_id = physical_host(args.physical_host_id.clone(), &selected)?; + Ok(ResolvedInputs { + data_root: profile.paths.data_root, + file, + explicit, + physical_host_id, + }) +} + +fn discovery_seeds() -> Vec { + std::env::var("CROWDB_DISCOVERY_SEEDS") + .unwrap_or_default() + .split(',') + .filter(|seed| !seed.is_empty()) + .map(str::to_owned) + .collect() +} + +fn physical_host(value: Option, policy: &NodeDeploymentPolicy) -> Result { + match value { + Some(value) if !value.is_empty() && value.len() <= 256 => Ok(value), + Some(_) => Err("physical host identity must contain 1 to 256 bytes"), + None if policy.mode == DeploymentMode::Local => Ok("unverified-local-host".into()), + None => Err("production deployment requires physical-host-id"), + } +} + +fn read_inputs( + path: Option<&std::path::Path>, +) -> Result> { + let Some(path) = path else { + return Ok(NodeDeploymentOverrides::default()); + }; + let metadata = fs::symlink_metadata(path)?; + if !metadata.file_type().is_file() || metadata.len() > 32768 { + return Err("node deployment configuration must be a bounded regular TOML file".into()); + } + let mut content = String::new(); + fs::File::open(path)?.take(32769).read_to_string(&mut content)?; + if content.len() > 32768 { + return Err("node deployment configuration exceeds its size bound".into()); + } + Ok(toml::from_str(&content)?) +} diff --git a/container/crowdb-monitor/src/preview.rs b/container/crowdb-monitor/src/preview.rs index e7ec45beb..f3123ec50 100644 --- a/container/crowdb-monitor/src/preview.rs +++ b/container/crowdb-monitor/src/preview.rs @@ -76,7 +76,7 @@ pub enum PreviewError { /// Describe the durable bootstrap sequence for a validated release service set. /// # Errors /// Rejects invalid profiles, unknown services and missing runtime dependencies. -pub fn preview_step_names(profile: &DeploymentProfile) -> Result, PreviewError> { +pub fn preview_step_names_for_tests(profile: &DeploymentProfile) -> Result, PreviewError> { profile.validate()?; if profile.name != PROFILE_NAME { return Err(PreviewError::Invalid("unsupported automatic startup profile")); @@ -88,7 +88,7 @@ pub fn preview_step_names(profile: &DeploymentProfile) -> Result, Pr /// # Errors /// Fails closed on incompatible durable state or any unready child. -pub async fn run_preview(profile_path: &Path) -> Result<(), PreviewError> { +pub async fn run_preview_for_tests(profile_path: &Path) -> Result<(), PreviewError> { let profile_bytes = fs::read(profile_path)?; let mut profile = DeploymentProfile::parse( std::str::from_utf8(&profile_bytes).map_err(|_| PreviewError::Invalid("profile is not UTF-8"))?, @@ -100,7 +100,7 @@ pub async fn run_preview(profile_path: &Path) -> Result<(), PreviewError> { } apply_public_uri_overrides(&mut profile); let config_bytes = config_digest_input(&profile)?; - let steps = preview_step_names(&profile)?; + let steps = preview_step_names_for_tests(&profile)?; let management_seed = management_seed(&profile)?; let step_refs = steps.iter().map(String::as_str).collect::>(); let mut session = BootstrapSession::open( diff --git a/container/crowdb-monitor/src/profile.rs b/container/crowdb-monitor/src/profile.rs index 87ff9b810..9667201a5 100644 --- a/container/crowdb-monitor/src/profile.rs +++ b/container/crowdb-monitor/src/profile.rs @@ -182,6 +182,29 @@ impl DeploymentProfile { validation::validate(self) } + /// Validate the software catalog before node-owned policy and topology effects. + /// + /// # Errors + /// Rejects preset media/membership and missing or extra service programs. + pub fn validate_node_catalog(&self) -> Result<(), ProfileError> { + self.validate()?; + let expected = ["kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web"]; + if !self.nodes.is_empty() + || !self.groups.is_empty() + || !self.disks.is_empty() + || self.services.len() != expected.len() + || self + .services + .iter() + .any(|service| !expected.contains(&service.id.as_str())) + { + return Err(ProfileError::Invalid( + "node profile requires the complete service catalog without preset topology or media".into(), + )); + } + Ok(()) + } + /// # Errors /// Returns an error when the profile or its service dependencies are invalid. pub fn services_in_start_order(&self) -> Result, ProfileError> { diff --git a/container/crowdb-monitor/src/profile/validation.rs b/container/crowdb-monitor/src/profile/validation.rs index 4c5a310bc..720ab04ac 100644 --- a/container/crowdb-monitor/src/profile/validation.rs +++ b/container/crowdb-monitor/src/profile/validation.rs @@ -78,6 +78,9 @@ fn validate_paths(profile: &DeploymentProfile) -> Result<(), ProfileError> { } fn validate_topology(profile: &DeploymentProfile) -> Result<(), ProfileError> { + if profile.nodes.is_empty() && profile.groups.is_empty() && profile.disks.is_empty() { + return Ok(()); + } if profile.nodes.is_empty() || profile.groups.is_empty() || profile.disks.is_empty() { return invalid("nodes, groups, and disks must be non-empty"); } diff --git a/container/crowdb-monitor/src/status.rs b/container/crowdb-monitor/src/status.rs index 240308e2a..d0acf4187 100644 --- a/container/crowdb-monitor/src/status.rs +++ b/container/crowdb-monitor/src/status.rs @@ -35,6 +35,7 @@ pub enum MonitorPhase { #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ServiceStatus { + pub operation_id: Option, pub pid: Option, pub generation: u64, pub healthy: bool, diff --git a/container/crowdb-monitor/src/supervisor.rs b/container/crowdb-monitor/src/supervisor.rs index 9a867d015..19dcb61c5 100644 --- a/container/crowdb-monitor/src/supervisor.rs +++ b/container/crowdb-monitor/src/supervisor.rs @@ -187,6 +187,7 @@ impl Supervisor { self.status.services.insert( id.to_owned(), ServiceStatus { + operation_id: None, pid: Some(pid), generation: 1, healthy: true, diff --git a/container/crowdb-monitor/tests/common/admitted_member.rs b/container/crowdb-monitor/tests/common/admitted_member.rs new file mode 100644 index 000000000..1c6cad06a --- /dev/null +++ b/container/crowdb-monitor/tests/common/admitted_member.rs @@ -0,0 +1,87 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{ + deployment::{ + recovery::{ConfirmedMemberTrust, MemberTrust, TrustedMember}, + registry::{NodeRecord, NodeRegistry}, + PreparedBootstrap, + }, + ConsoleConfig, +}; +use crowdb_monitor::NodeIdentity; +use serde_json::json; +use std::path::Path; + +pub fn fixture(root: &Path) -> ConfirmedMemberTrust { + let identity = NodeIdentity::load_or_create(root).unwrap(); + let mut config = ConsoleConfig::default(); + config + .racks + .push(serde_json::from_value(json!({"id":1,"name":"rack"})).unwrap()); + let nodes: Vec<_> = (1_u64..=3).map(|id| { + config.nodes.push(serde_json::from_value(json!({"id":id,"rack_id":1,"host":format!("192.0.2.{id}"),"ssh_user":"crowdb"})).unwrap()); + config.servers.push(serde_json::from_value(json!({"id":format!("kv-{id}"),"node_id":id,"url":format!("http://192.0.2.{id}:10000"),"rpc_url":format!("192.0.2.{id}:10100")})).unwrap()); + NodeRecord { discovery_id: uuid::Uuid::new_v4().to_string(), node_id:id, physical_host_id:format!("host-{id}"), + rack_id:1, host:format!("192.0.2.{id}"), ssh_port:22, ssh_user:"crowdb".into(), + operation_id:uuid::Uuid::new_v4().to_string(), confirmed:false, cancelled:false, + host_key:Some(super::trust::key(u8::try_from(id).unwrap())) } + }).collect(); + let operation = PreparedBootstrap::open_with_nodes( + &root.join("prepared-bootstrap.json"), + &config, + &[1, 2, 3], + nodes.clone(), + ) + .unwrap(); + let initial = MemberTrust::from_bootstrap(&operation).unwrap(); + initial.seal(&root.join("member-trust.json")).unwrap(); + let own = TrustedMember { + discovery_id: identity.uuid().to_string(), + node_id: 4, + admission_operation_id: uuid::Uuid::new_v4().to_string(), + host_key: super::trust::key(4), + }; + std::fs::create_dir(root.join("ssh")).unwrap(); + std::fs::write(root.join("ssh/ssh_host_ed25519_key.pub"), &own.host_key).unwrap(); + let mut admitted = initial.clone(); + admitted.members.push(own.clone()); + admitted + .seal(&root.join(format!( + "admitted-member-trust-{}.json", + own.admission_operation_id + ))) + .unwrap(); + super::trust::write( + &root.join("accepted-node.json"), + &json!({"node_id":4,"bootstrap":operation.identity,"prepared":true, + "admission":{"operation_id":own.admission_operation_id,"management_seeds":["http://192.0.2.1:10000"]}}), + ); + let mut confirmed = nodes; + for node in &mut confirmed { + node.confirmed = true; + } + confirmed.push(NodeRecord { + discovery_id: own.discovery_id, + node_id: 4, + physical_host_id: "host-4".into(), + rack_id: 1, + host: "192.0.2.4".into(), + ssh_port: 22, + ssh_user: "crowdb".into(), + operation_id: own.admission_operation_id, + host_key: Some(own.host_key), + confirmed: true, + cancelled: false, + }); + let snapshot = ConfirmedMemberTrust::new( + initial, + NodeRegistry { + cluster_id: operation.identity.cluster_id, + nodes: confirmed, + }, + ) + .unwrap(); + super::trust::write(&root.join("confirmed-member-trust.json"), &snapshot); + snapshot +} diff --git a/container/crowdb-monitor/tests/common/member_trust.rs b/container/crowdb-monitor/tests/common/member_trust.rs new file mode 100644 index 000000000..304be3256 --- /dev/null +++ b/container/crowdb-monitor/tests/common/member_trust.rs @@ -0,0 +1,91 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{ + deployment::{ + recovery::{MemberTrust, TrustedMember}, + registry::NodeRecord, + PreparedBootstrap, + }, + ConsoleConfig, +}; +use crowdb_monitor::NodeIdentity; +use serde_json::json; +use std::{os::unix::fs::PermissionsExt, path::Path}; + +pub fn write(path: &Path, value: &impl serde::Serialize) { + std::fs::write(path, serde_json::to_vec(value).unwrap()).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).unwrap(); +} + +pub fn key(byte: u8) -> String { + let private = ssh_key::private::Ed25519PrivateKey::from_bytes(&[byte; 32]); + let public = ssh_key::public::Ed25519PublicKey::from(&private); + ssh_key::PublicKey::new(ssh_key::public::KeyData::Ed25519(public), "") + .to_openssh() + .unwrap() +} + +pub fn fixture(root: &Path) -> MemberTrust { + let identity = NodeIdentity::load_or_create(root).unwrap(); + let ids = [ + identity.uuid().to_string(), + uuid::Uuid::new_v4().to_string(), + uuid::Uuid::new_v4().to_string(), + ]; + let mut config = ConsoleConfig::default(); + let mut nodes = Vec::new(); + for id in 1_u64..=3 { + config.nodes.push( + serde_json::from_value( + json!({"id":id,"rack_id":1,"host":format!("192.0.2.{id}"),"ssh_user":"crowdb"}), + ) + .unwrap(), + ); + config.servers.push(serde_json::from_value(json!({"id":format!("kv-{id}"),"node_id":id,"url":format!("http://192.0.2.{id}:10000"),"rpc_url":format!("192.0.2.{id}:10100")})).unwrap()); + nodes.push(NodeRecord { + discovery_id: ids[usize::try_from(id - 1).unwrap()].clone(), + node_id: id, + physical_host_id: "test-host".into(), + rack_id: 1, + host: format!("192.0.2.{id}"), + ssh_port: 22, + ssh_user: "crowdb".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + confirmed: false, + host_key: None, + cancelled: false, + }); + } + config + .racks + .push(serde_json::from_value(json!({"id":1,"name":"rack"})).unwrap()); + let bootstrap = + PreparedBootstrap::open_with_nodes(&root.join("prepared-bootstrap.json"), &config, &[1, 2, 3], nodes) + .unwrap(); + write( + &root.join("accepted-node.json"), + &json!({"node_id":1,"bootstrap":bootstrap.identity,"prepared":true}), + ); + std::fs::create_dir(root.join("ssh")).unwrap(); + std::fs::write(root.join("ssh/ssh_host_ed25519_key.pub"), key(1)).unwrap(); + std::fs::write( + root.join("ssh/known_hosts"), + format!("192.0.2.2 {}\n192.0.2.3 {}\n", key(2), key(3)), + ) + .unwrap(); + MemberTrust { + version: 1, + bootstrap: bootstrap.identity, + members: bootstrap + .nodes + .into_iter() + .map(|node| TrustedMember { + discovery_id: node.discovery_id, + node_id: node.node_id, + admission_operation_id: node.operation_id, + host_key: key(u8::try_from(node.node_id).unwrap()), + }) + .collect(), + } +} diff --git a/container/crowdb-monitor/tests/common/node_setup.rs b/container/crowdb-monitor/tests/common/node_setup.rs index 5c999d299..d8d7de409 100644 --- a/container/crowdb-monitor/tests/common/node_setup.rs +++ b/container/crowdb-monitor/tests/common/node_setup.rs @@ -3,6 +3,12 @@ use axum::{extract::State, routing::get, Json, Router}; use crowdb_console_shared::deployment::registry::{NodeRecord, NodeRegistry, NODES_KEY}; +use crowdb_console_shared::{ + config::{NodeEntry, RackEntry, ServerEntry}, + deployment::{preparation, PreparedBootstrap}, + ops::OpContext, + ConsoleConfig, +}; use crowdb_kv::cluster::{ group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, }; @@ -39,14 +45,11 @@ impl TestAuthority { }); let kv = CrowdbKvClient::new(ClientConfig::new(vec![seed.clone()])); kv.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); + let bootstrap = bootstrap(root, &kv, &seed, &store.listen_addr().unwrap().to_string()).await; let binding = NodeBinding { node_id: 2, management_seeds: vec![seed], - bootstrap: crowdb_protocol::mgmt::SystemBootstrapIdentity { - cluster_id: uuid::Uuid::new_v4().to_string(), - operation_id: uuid::Uuid::new_v4().to_string(), - configuration_digest: "fixture".into(), - }, + bootstrap: bootstrap.identity, }; let registry = NodeRegistry { cluster_id: binding.bootstrap.cluster_id.clone(), @@ -60,18 +63,10 @@ impl TestAuthority { ssh_port: 22, ssh_user: "crowdb".into(), confirmed: true, + host_key: None, cancelled: false, }], }; - kv.put( - 0, - 0, - b"/deployment/cluster", - &serde_json::to_vec(&serde_json::json!({"identity":binding.bootstrap})).unwrap(), - None, - ) - .await - .unwrap(); kv.put(0, 0, NODES_KEY, &serde_json::to_vec(®istry).unwrap(), None) .await .unwrap(); @@ -88,6 +83,43 @@ impl TestAuthority { } } +async fn bootstrap(root: &Path, kv: &CrowdbKvClient, management: &str, rpc: &str) -> PreparedBootstrap { + let mut server = ServerEntry::new("kv-2", management); + server.node_id = Some(2); + server.rpc_url = Some(rpc.into()); + let config = ConsoleConfig { + racks: vec![RackEntry { + id: 1, + name: "owned-rack".into(), + }], + nodes: vec![NodeEntry { + id: 2, + rack_id: 1, + host: "127.0.0.1".into(), + ssh_port: 22, + ssh_user: "crowdb".into(), + ssh_credential_ref: None, + ssh_key: None, + ssh_password: None, + }], + servers: vec![server], + ..Default::default() + }; + let prepared = PreparedBootstrap::open(&root.join("fixture-bootstrap.json"), &config, &[2]).unwrap(); + kv.put( + 0, + 0, + b"/deployment/cluster", + &serde_json::to_vec(&prepared).unwrap(), + None, + ) + .await + .unwrap(); + let ctx = OpContext::new(rpc.into(), vec![management.into()], config); + preparation::reserve(&ctx, &prepared.identity).await.unwrap(); + prepared +} + impl Drop for TestAuthority { fn drop(&mut self) { self.task.abort(); diff --git a/container/crowdb-monitor/tests/common/node_setup_control.rs b/container/crowdb-monitor/tests/common/node_setup_control.rs new file mode 100644 index 000000000..38aa1acc7 --- /dev/null +++ b/container/crowdb-monitor/tests/common/node_setup_control.rs @@ -0,0 +1,64 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::mgmt::node::{NodeControl, NodeServiceAction, NodeServiceIntent}; +use std::path::Path; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +pub fn processes_once(root: &Path) -> tokio::task::JoinHandle<()> { + std::fs::create_dir_all(root).unwrap(); + let listener = tokio::net::UnixListener::bind(root.join("node-control.sock")).unwrap(); + tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + stream.read_to_end(&mut bytes).await.unwrap(); + assert!(matches!( + serde_json::from_slice::(&bytes).unwrap(), + NodeControl::Processes { .. } + )); + stream + .write_all(br#"{"ok":true,"value":{"processes":{},"service_failures":{}}}"#) + .await + .unwrap(); + }) +} + +pub async fn newer_intent( + root: &Path, + authority: &super::node_setup::TestAuthority, + action: NodeServiceAction, +) -> (String, NodeServiceIntent) { + let path = root.join("node-setup.json"); + let mut journal: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + let original = uuid::Uuid::new_v4().to_string(); + let step = journal["status"]["services"] + .as_array_mut() + .unwrap() + .iter_mut() + .find(|step| step["kind"] == "chunkdb") + .unwrap(); + step["operation_id"] = serde_json::json!(original); + std::fs::write(&path, serde_json::to_vec(&journal).unwrap()).unwrap(); + let newer = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: 2, + service_id: "chunkdb-2".into(), + kind: "chunkdb".into(), + action, + configuration: "instance_id = 9".into(), + environment: std::collections::BTreeMap::new(), + }; + authority + .kv + .put( + 0, + 0, + b"/deployment/services/2/chunkdb-2", + &serde_json::to_vec(&newer).unwrap(), + None, + ) + .await + .unwrap(); + (original, newer) +} diff --git a/container/crowdb-monitor/tests/common/node_setup_file.rs b/container/crowdb-monitor/tests/common/node_setup_file.rs new file mode 100644 index 000000000..41ca5117a --- /dev/null +++ b/container/crowdb-monitor/tests/common/node_setup_file.rs @@ -0,0 +1,94 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeSetupForTests}; +use crowdb_protocol::mgmt::node::{ + setup::{NodeSetupChoices, NodeSetupRequest, SetupStorage}, + storage::FilePreparationPhase, + NodeControl, +}; +use std::path::Path; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +pub async fn prepare( + root: &Path, + services: bool, +) -> ( + NodeSetupRequest, + super::node_setup::TestAuthority, + DeploymentProfile, +) { + let mut request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: NodeSetupChoices::default(), + }; + request.choices.storage = SetupStorage::Automatic { + capacity_bytes: 64 * 1024 * 1024, + reserve_bytes: i64::MAX as u64, + sparse: false, + }; + for service in &mut request.choices.services { + service.enabled = services; + } + std::fs::create_dir(root.join("ssh")).unwrap(); + std::fs::write( + root.join("ssh/authorized_keys"), + format!("ssh-ed25519 fixture crowdb:{}\n", request.operation_id), + ) + .unwrap(); + NodeSetupForTests::default() + .accept_for_tests(root, request.clone()) + .unwrap(); + let authority = super::node_setup::TestAuthority::start(root, &request).await; + publish_storage(&authority).await; + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.run_root = root.join("run"); + std::fs::create_dir(&profile.paths.run_root).unwrap(); + (request, authority, profile) +} + +async fn publish_storage(authority: &super::node_setup::TestAuthority) { + use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; + use crowdb_protocol::common::{NodeValue, RackValue}; + super::node_setup_groups::publish_group(authority, 1).await; + let node = &authority.registry.nodes[0]; + let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + authority.binding.management_seeds.clone(), + ))); + hardware + .add_rack( + node.rack_id, + &RackValue { + node_ids: vec![node.node_id], + ..Default::default() + }, + ) + .await + .unwrap(); + hardware + .add_node(node.rack_id, node.node_id, &NodeValue::default()) + .await + .unwrap(); +} + +pub async fn fail_file_request( + listener: tokio::net::UnixListener, + root: std::path::PathBuf, +) -> crowdb_protocol::mgmt::node::storage::FilePreparation { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + stream.read_to_end(&mut bytes).await.unwrap(); + let NodeControl::PrepareDiskFile { request } = serde_json::from_slice(&bytes).unwrap() else { + panic!("unexpected setup command"); + }; + crowdb_monitor::accept_file_for_tests(&root, &request).unwrap(); + let result = crowdb_monitor::prepare_file_for_tests(&root, &request.operation_id).unwrap(); + assert_eq!(result.phase, FilePreparationPhase::Failed); + stream.write_all(br#"{"ok":true,"value":{}}"#).await.unwrap(); + result +} diff --git a/container/crowdb-monitor/tests/common/node_setup_groups.rs b/container/crowdb-monitor/tests/common/node_setup_groups.rs new file mode 100644 index 000000000..871b34189 --- /dev/null +++ b/container/crowdb-monitor/tests/common/node_setup_groups.rs @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::node_setup::TestAuthority; +use crowdb_protocol::{ + common::GroupValue, + key::{KvGroupKey, TextKey}, +}; + +/// A component fixture publication, without certifying a native group runtime. +pub async fn publish_group(authority: &TestAuthority, group_id: u64) { + authority + .kv + .put( + 0, + 0, + KvGroupKey { + store_id: 0, + group_id, + } + .to_path() + .as_bytes(), + &serde_json::to_vec(&GroupValue { + store_id: 0, + group_id, + }) + .unwrap(), + None, + ) + .await + .unwrap(); +} diff --git a/container/crowdb-monitor/tests/common/run_root.rs b/container/crowdb-monitor/tests/common/run_root.rs new file mode 100644 index 000000000..771bf2089 --- /dev/null +++ b/container/crowdb-monitor/tests/common/run_root.rs @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_test_harness::test_dirs::{tempdir_in_test_data, TestDir}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +/// Retain test state in the workspace while bounding Unix socket addresses. +pub struct TestRunRoot { + _data: TestDir, + alias: PathBuf, + root: PathBuf, +} + +impl TestRunRoot { + pub fn new(tag: &str) -> Self { + let data = tempdir_in_test_data(tag); + // /tmp also avoids macOS's long per-user temporary directory prefix. + let alias = PathBuf::from("/tmp").join(format!("cm-run-{}", uuid::Uuid::new_v4().simple())); + fs::create_dir(data.path().join("run")).unwrap(); + std::os::unix::fs::symlink(data.path(), &alias).unwrap(); + // Keep the root itself a real directory for node identity validation. + let root = alias.join("run"); + Self { + _data: data, + alias, + root, + } + } + + pub fn path(&self) -> &Path { + &self.root + } +} + +impl Drop for TestRunRoot { + fn drop(&mut self) { + if !std::thread::panicking() { + fs::remove_file(&self.alias).unwrap(); + } + } +} diff --git a/container/crowdb-monitor/tests/common/runtime_observation.rs b/container/crowdb-monitor/tests/common/runtime_observation.rs new file mode 100644 index 000000000..3afe5565b --- /dev/null +++ b/container/crowdb-monitor/tests/common/runtime_observation.rs @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::{ + DeploymentProfile, MonitorPhase, MonitorStatus, NodeIdentity, ProcessManager, ServiceStatus, StatusStore, +}; +use crowdb_protocol::mgmt::node::{ + deployment::NodeDeploymentOverrides, NodeServiceAction, NodeServiceIntent, +}; +use std::os::unix::fs::PermissionsExt; +use std::{collections::BTreeMap, path::Path}; + +pub struct TestObservation { + pub profile: DeploymentProfile, + pub manager: ProcessManager, + pub store: StatusStore, + pub status: MonitorStatus, + pub intent: NodeServiceIntent, +} + +pub fn profile(root: &Path) -> DeploymentProfile { + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.run_root = root.join("run"); + profile.paths.log_root = root.join("log"); + profile +} + +impl TestObservation { + pub async fn start(root: &Path) -> Self { + let inputs = NodeDeploymentOverrides::default(); + crowdb_monitor::accept_node_policy(root, &inputs, &inputs).unwrap(); + let identity = NodeIdentity::load_or_create(root).unwrap(); + let profile = profile(root); + std::fs::create_dir(&profile.paths.run_root).unwrap(); + std::fs::create_dir(&profile.paths.log_root).unwrap(); + let mut manager = ProcessManager::new(profile.paths.log_root.clone(), profile.logs.clone()) + .await + .unwrap(); + let mut service = profile + .services + .iter() + .find(|service| service.id == "chunkdb") + .unwrap() + .clone(); + service.id = "chunkdb-1".into(); + service.program = "/bin/sh".into(); + service.args = vec!["-c".into(), "exec sleep 60".into()]; + let pid = manager.start(&service, &BTreeMap::new()).await.unwrap(); + let intent = NodeServiceIntent { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: 1, + service_id: service.id.clone(), + kind: "chunkdb".into(), + action: NodeServiceAction::Start, + configuration: "[server]\n".into(), + environment: BTreeMap::new(), + }; + let mut status = MonitorStatus::new(identity.uuid(), MonitorPhase::Ready); + status.services.insert( + service.id, + ServiceStatus { + operation_id: Some(intent.operation_id.clone()), + pid: Some(pid), + generation: 1, + healthy: true, + restart_attempts: 0, + }, + ); + let store = StatusStore::new(&profile.paths.run_root).unwrap(); + store.publish(&mut status).unwrap(); + let result = Self { + profile, + manager, + store, + status, + intent, + }; + result.save(true, 0, None); + result + } + + pub fn save(&self, complete: bool, attempts: u32, error: Option<&str>) { + let directory = self + .profile + .paths + .data_root + .join("services") + .join(&self.intent.service_id); + std::fs::create_dir_all(&directory).unwrap(); + std::fs::write( + directory.join("execution.json"), + serde_json::to_vec(&serde_json::json!({ + "intent":self.intent, "complete":complete, "attempts":attempts, + "pid":self.status.services[&self.intent.service_id].pid, "error":error, + })) + .unwrap(), + ) + .unwrap(); + std::fs::set_permissions( + directory.join("execution.json"), + std::fs::Permissions::from_mode(0o600), + ) + .unwrap(); + } +} diff --git a/container/crowdb-monitor/tests/common/service_dependency.rs b/container/crowdb-monitor/tests/common/service_dependency.rs new file mode 100644 index 000000000..6f96d9c93 --- /dev/null +++ b/container/crowdb-monitor/tests/common/service_dependency.rs @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_kv_client::{CrowdbKvClient, HardwareClient}; +use crowdb_protocol::{ + common::{DiskId, HwStatus, NodeValue, RackValue}, + diskdb::rpc::{DiskGroupValue, DiskValue}, +}; +use std::sync::Arc; + +pub async fn publish_online_disk(kv: Arc) { + let hardware = HardwareClient::from_shared(kv.clone()); + hardware + .add_rack( + 1, + &RackValue { + node_ids: vec![2], + ..Default::default() + }, + ) + .await + .unwrap(); + hardware + .add_node( + 1, + 2, + &NodeValue { + disk_group_ids: vec![7], + ..Default::default() + }, + ) + .await + .unwrap(); + let disk = DiskId { high: 0, low: 7 }; + hardware + .add_disk_group( + 1, + 2, + 7, + &DiskGroupValue { + disk_ids: vec![disk], + ..Default::default() + }, + ) + .await + .unwrap(); + hardware + .add_disk( + 1, + 2, + 7, + &disk, + &DiskValue { + status: HwStatus::Up as i32, + ..Default::default() + }, + ) + .await + .unwrap(); +} + +pub async fn ready_listener() -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve( + listener, + axum::Router::new().route("/ready", axum::routing::get(|| async { "ok" })), + ) + .await + .unwrap(); + }); + (address, server) +} + +pub async fn assert_initial_start_waits( + kv: &CrowdbKvClient, + profile: &crowdb_monitor::DeploymentProfile, + local: &crowdb_protocol::mgmt::node::NodeServiceIntent, + consumer: &crowdb_protocol::mgmt::node::NodeServiceIntent, +) { + for intent in [local, consumer] { + kv.put( + 0, + 0, + format!("/deployment/services/{}/{}", intent.node_id, intent.service_id).as_bytes(), + &serde_json::to_vec(intent).unwrap(), + None, + ) + .await + .unwrap(); + } + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + let mut runtime = crowdb_monitor::NodeServiceForTests::new_for_tests(profile.clone()) + .await + .unwrap(); + let error = runtime.execute_for_tests(consumer.clone()).await.unwrap_err(); + assert!( + error.to_string().contains("has not loaded"), + "initial deployment bypassed dependency readiness: {error}" + ); + assert!( + !profile + .paths + .data_root + .join("services") + .join(&consumer.service_id) + .join("execution.json") + .exists(), + "waiting for dependencies cannot consume a launch attempt" + ); +} diff --git a/container/crowdb-monitor/tests/credentials_test.rs b/container/crowdb-monitor/tests/credentials_test.rs index 3b8173031..e632bf685 100644 --- a/container/crowdb-monitor/tests/credentials_test.rs +++ b/container/crowdb-monitor/tests/credentials_test.rs @@ -37,6 +37,7 @@ fn client() -> ClientCredentials { ClientCredentials { s3_endpoint: "http://localhost:16000".into(), iceberg_endpoint: "http://localhost:8181".into(), + dataset_endpoint: None, region: "us-east-1".into(), access_key_id: "CROW123".into(), secret_access_key: "secret_123".into(), @@ -137,6 +138,28 @@ fn invalid_client_values_are_not_persisted() { assert!(!root.path().join("secrets/client.env").exists()); } +#[test] +fn dataset_client_origin_and_bearer_are_private_stable_and_validated() { + let root = TestDataRoot::new(); + let server = ServerCredentials::load_or_create(root.path()).unwrap(); + let mut client = client(); + client.dataset_endpoint = Some("http://[::1]:19093".into()); + server.persist_client(&client).unwrap(); + let original = show_client_credentials(root.path()).unwrap(); + assert!(original.contains("DATASET_URI=http://[::1]:19093\n")); + assert!(original.contains(&format!("DATASET_TOKEN={}\n", server.dataset_token()))); + assert!(!original.contains(server.s3_master_key())); + assert!(!original.contains(server.iceberg_manage_token())); + let restored = ServerCredentials::load_existing(root.path()).unwrap(); + restored.verify_client(&client).unwrap(); + restored.persist_client(&client).unwrap(); + assert_eq!(show_client_credentials(root.path()).unwrap(), original); + let path = root.path().join("secrets/client.env"); + let corrupt = original.replace(&server.dataset_token(), "invalid-token"); + fs::write(&path, corrupt).unwrap(); + assert!(show_client_credentials(root.path()).is_err()); +} + #[test] fn explicit_cli_prints_only_client_file() { let root = TestDataRoot::new(); @@ -155,3 +178,28 @@ fn explicit_cli_prints_only_client_file() { ); assert!(!String::from_utf8_lossy(&output.stdout).contains("CROWDB_S3_MASTER_KEY")); } + +#[test] +fn node_cli_uses_the_owned_console_client_file_without_exporting_node_server_secrets() { + let root = TestDataRoot::new(); + let inputs = crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides::default(); + crowdb_monitor::accept_node_policy(root.path(), &inputs, &inputs).unwrap(); + let console = root.path().join("console"); + fs::create_dir(&console).unwrap(); + let server = ServerCredentials::load_or_create(&console).unwrap(); + let mut client = client(); + client.dataset_endpoint = Some("http://127.0.0.1:19093".into()); + server.persist_client(&client).unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")) + .args(["credentials", "show", "--format", "env", "--data-root"]) + .arg(root.path()) + .output() + .unwrap(); + assert!(output.status.success()); + assert!(output.stderr.is_empty()); + assert_eq!( + output.stdout, + show_client_credentials(&console).unwrap().as_bytes() + ); + assert!(!String::from_utf8_lossy(&output.stdout).contains(server.s3_master_key())); +} diff --git a/container/crowdb-monitor/tests/file_admission_test.rs b/container/crowdb-monitor/tests/file_admission_test.rs index 9295080df..cb372ce6b 100644 --- a/container/crowdb-monitor/tests/file_admission_test.rs +++ b/container/crowdb-monitor/tests/file_admission_test.rs @@ -24,9 +24,15 @@ async fn managed_files_reserve_physical_identity_and_replay_old_receipts_without accept_file_for_tests(root.path(), &request).unwrap(); let prepared = prepare_file_for_tests(root.path(), &request.operation_id).unwrap(); let before = std::fs::metadata(&prepared.path).unwrap(); - let progress = publish_file_for_tests(root.path(), &request.operation_id) + crowdb_monitor::resume_file_operations_for_tests(root.path()) .await .unwrap(); + let progress = crowdb_monitor::file_operations_for_tests(root.path(), None, 32) + .unwrap() + .items + .into_iter() + .find(|state| state.request.operation_id == request.operation_id) + .unwrap(); assert_eq!(progress.phase, FilePreparationPhase::WaitingForDiskIO); let (high, low) = uuid::Uuid::parse_str(&request.operation_id) .unwrap() diff --git a/container/crowdb-monitor/tests/file_preparation_test.rs b/container/crowdb-monitor/tests/file_preparation_test.rs index 658f47fb2..05674e6f6 100644 --- a/container/crowdb-monitor/tests/file_preparation_test.rs +++ b/container/crowdb-monitor/tests/file_preparation_test.rs @@ -271,3 +271,18 @@ fn invalid_geometry_or_operation_path_never_accepts_intent() { } assert_eq!(fs::read_dir(&root.0).unwrap().count(), 0); } + +#[tokio::test] +async fn completed_preparation_without_admission_is_terminal_after_restart() { + let root = TestRoot::new(); + let request = request(); + accept_file_for_tests(&root.0, &request).unwrap(); + let before = prepare_file_for_tests(&root.0, &request.operation_id).unwrap(); + crowdb_monitor::resume_file_operations_for_tests(&root.0) + .await + .unwrap(); + assert_eq!( + prepare_file_for_tests(&root.0, &request.operation_id).unwrap(), + before + ); +} diff --git a/container/crowdb-monitor/tests/initialization_test.rs b/container/crowdb-monitor/tests/initialization_test.rs index 1ba7bdda9..82be71895 100644 --- a/container/crowdb-monitor/tests/initialization_test.rs +++ b/container/crowdb-monitor/tests/initialization_test.rs @@ -12,11 +12,14 @@ use crowdb_protocol::mgmt::node::{NodeControl, NodeServiceCredentials}; use serde_json::json; use tokio::io::{AsyncReadExt, AsyncWriteExt}; +#[path = "common/member_trust.rs"] +#[allow(dead_code)] +mod member_trust; + fn fixture(root: &Path) -> (DeploymentProfile, PreparedBootstrap, NodeServiceCredentials) { - let mut profile = DeploymentProfile::load( - Path::new(env!("CARGO_MANIFEST_DIR")).join("../single-node-container/profile.toml"), - ) - .unwrap(); + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); profile.paths.data_root = root.to_path_buf(); profile.paths.run_root = root.join("run"); std::fs::create_dir(&profile.paths.run_root).unwrap(); @@ -36,6 +39,7 @@ fn fixture(root: &Path) -> (DeploymentProfile, PreparedBootstrap, NodeServiceCre ssh_user: "crowdb".into(), operation_id: uuid::Uuid::new_v4().to_string(), confirmed: false, + host_key: None, cancelled: false, }; let operation = @@ -72,7 +76,7 @@ async fn accepted_initialization_survives_connection_lifetime_and_resumes_fixed_ connection.read_to_end(&mut bytes).await.unwrap(); assert!(matches!( serde_json::from_slice::(&bytes).unwrap(), - NodeControl::StartKv { node_id: 1, .. } + NodeControl::PrepareBootstrap { node_id: 1, .. } )); first.stop_for_tests().await; drop(connection); @@ -132,3 +136,50 @@ async fn altered_manifest_is_rejected_before_accepting_an_initialization() { "idle" ); } + +#[tokio::test] +async fn bootstrap_waits_for_member_trust_before_starting_a_replica() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("init-pins"); + let root = directory.path(); + let (profile, operation, credentials) = fixture(root); + std::fs::create_dir(root.join("ssh")).unwrap(); + std::fs::write(root.join("ssh/ssh_host_ed25519_key.pub"), member_trust::key(1)).unwrap(); + let listener = tokio::net::UnixListener::bind(profile.paths.run_root.join("node-control.sock")).unwrap(); + let mut runner = InitializationForTests::default(); + runner + .accept_for_tests( + &profile, + serde_json::to_value(&operation).unwrap(), + credentials, + Some(false), + ) + .unwrap(); + for stage in 0..3 { + let (mut connection, _) = tokio::time::timeout(Duration::from_secs(3), listener.accept()) + .await + .unwrap() + .unwrap(); + let mut bytes = Vec::new(); + connection.read_to_end(&mut bytes).await.unwrap(); + let request: NodeControl = serde_json::from_slice(&bytes).unwrap(); + match (stage, request) { + (0, NodeControl::PrepareBootstrap { node_id: 1, .. }) + | (2, NodeControl::StartKv { node_id: 1, .. }) => {} + (1, NodeControl::AcceptMemberTrust { trust }) => { + let trust: crowdb_console_shared::deployment::recovery::MemberTrust = + serde_json::from_value(trust).unwrap(); + trust.validate_bootstrap(&operation).unwrap(); + assert_eq!(trust.members[0].host_key, member_trust::key(1)); + } + _ => panic!("unexpected bootstrap request at stage {stage}"), + } + let reply = if stage == 2 { + br#"{"ok":false,"error":"test stops before replica execution"}"#.as_slice() + } else { + br#"{"ok":true,"value":{}}"#.as_slice() + }; + connection.write_all(reply).await.unwrap(); + connection.shutdown().await.unwrap(); + } + runner.stop_for_tests().await; +} diff --git a/container/crowdb-monitor/tests/kv_resume_test.rs b/container/crowdb-monitor/tests/kv_resume_test.rs new file mode 100644 index 000000000..077df71f2 --- /dev/null +++ b/container/crowdb-monitor/tests/kv_resume_test.rs @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/member_trust.rs"] +mod trust; + +use crowdb_monitor::{accept_member_trust_for_tests, DeploymentProfile, NodeServiceForTests, StatusStore}; +use std::{ + os::unix::fs::PermissionsExt, + sync::atomic::{AtomicBool, Ordering}, + sync::Arc, + time::Duration, +}; + +#[tokio::test] +async fn retained_quorum_startup_does_not_block_discovery_or_confuse_liveness_with_health() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("kv-resume"); + let root = directory.path(); + let trust = trust::fixture(root); + accept_member_trust_for_tests(root, &trust).unwrap(); + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("log"); + profile.paths.run_root = root.join("run"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + let program = root.join("live-process"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + for service in &mut profile.services { + service.program = program.clone(); + if service.id == "kv" { + service.probe.target = format!("http://{}/health", listener.local_addr().unwrap()); + service.probe.timeout_ms = 100; + } + } + let status = StatusStore::new(&profile.paths.run_root).unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile).await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), worker.resume_for_tests()) + .await + .expect("discovery startup cannot wait for KV quorum health") + .unwrap(); + worker.tick_for_tests().await.unwrap(); + let first = status.read(Duration::from_secs(3)).unwrap().services["kv"].clone(); + assert!(!first.healthy); + assert!(first.pid.is_some()); + assert_eq!(first.restart_attempts, 0); + for _ in 0..6 { + worker.tick_for_tests().await.unwrap(); + assert_eq!(status.read(Duration::from_secs(3)).unwrap().services["kv"], first); + } + let ready = Arc::new(AtomicBool::new(true)); + let sampled = ready.clone(); + let server = tokio::spawn(async move { + axum::serve( + listener, + axum::Router::new().route( + "/health", + axum::routing::get(move || { + let ready = sampled.clone(); + async move { + if ready.load(Ordering::Acquire) { + axum::http::StatusCode::OK + } else { + axum::http::StatusCode::SERVICE_UNAVAILABLE + } + } + }), + ), + ) + .await + .unwrap(); + }); + worker.tick_for_tests().await.unwrap(); + let recovered = status.read(Duration::from_secs(3)).unwrap().services["kv"].clone(); + assert!(recovered.healthy); + assert_eq!(recovered.pid, first.pid); + assert_eq!(recovered.restart_attempts, 0); + ready.store(false, Ordering::Release); + worker.stop_kv_for_tests().await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), worker.tick_for_tests()) + .await + .expect("restarting a retained replica cannot wait for quorum health") + .unwrap(); + let restarted = status.read(Duration::from_secs(3)).unwrap().services["kv"].clone(); + assert!(!restarted.healthy); + assert_ne!(restarted.pid, first.pid); + assert_eq!(restarted.restart_attempts, 1); + std::fs::write(root.join("ssh/ssh_host_ed25519_key.pub"), trust::key(4)).unwrap(); + assert!( + worker.resume_for_tests().await.is_err(), + "live processes cannot bypass retained key validation" + ); + server.abort(); +} diff --git a/container/crowdb-monitor/tests/liveness_test.rs b/container/crowdb-monitor/tests/liveness_test.rs index d004965b5..b8929f14d 100644 --- a/container/crowdb-monitor/tests/liveness_test.rs +++ b/container/crowdb-monitor/tests/liveness_test.rs @@ -1,43 +1,19 @@ use std::fs; -use std::path::PathBuf; use std::process::Command; use crowdb_monitor::{probe_liveness, LivenessServer}; -use uuid::Uuid; - -struct TestRunRoot(PathBuf); - -impl TestRunRoot { - fn new() -> Self { - // macOS limits Unix-domain socket paths to a short fixed buffer. Keep - // this test root under /tmp instead of the long per-user temp path. - let base = if cfg!(target_os = "macos") { - PathBuf::from("/tmp") - } else { - std::env::temp_dir() - }; - let root = base.join(format!("cm-live-{}", Uuid::new_v4().simple())); - fs::create_dir_all(&root).unwrap(); - Self(root) - } -} - -impl Drop for TestRunRoot { - fn drop(&mut self) { - if !std::thread::panicking() { - fs::remove_dir_all(&self.0).unwrap(); - } - } -} +#[path = "common/run_root.rs"] +mod run_root; +use run_root::TestRunRoot; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn liveness_requires_responsive_monitor_not_status_file() { - let root = TestRunRoot::new(); - assert!(probe_liveness(&root.0).await.is_err()); - let server = LivenessServer::start(&root.0).unwrap(); - assert!(LivenessServer::start(&root.0).is_err()); - assert!(probe_liveness(&root.0).await.is_ok()); - let path = root.0.clone(); + let root = TestRunRoot::new("monitor-liveness"); + assert!(probe_liveness(root.path()).await.is_err()); + let server = LivenessServer::start(root.path()).unwrap(); + assert!(LivenessServer::start(root.path()).is_err()); + assert!(probe_liveness(root.path()).await.is_ok()); + let path = root.path().to_path_buf(); let success = tokio::task::spawn_blocking(move || { Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")) .args(["liveness", "--run-root"]) @@ -51,24 +27,24 @@ async fn liveness_requires_responsive_monitor_not_status_file() { .unwrap(); assert!(success); drop(server); - assert!(probe_liveness(&root.0).await.is_err()); + assert!(probe_liveness(root.path()).await.is_err()); } #[tokio::test] async fn liveness_reclaims_only_a_disconnected_socket() { use std::os::unix::fs::DirBuilderExt; - let root = TestRunRoot::new(); - let directory = root.0.join("health"); + let root = TestRunRoot::new("monitor-liveness"); + let directory = root.path().join("health"); fs::DirBuilder::new().mode(0o700).create(&directory).unwrap(); let path = directory.join("monitor.sock"); let stale = std::os::unix::net::UnixListener::bind(&path).unwrap(); drop(stale); - let server = LivenessServer::start(&root.0).unwrap(); - assert!(probe_liveness(&root.0).await.is_ok()); - assert!(LivenessServer::start(&root.0).is_err()); - assert!(probe_liveness(&root.0).await.is_ok()); + let server = LivenessServer::start(root.path()).unwrap(); + assert!(probe_liveness(root.path()).await.is_ok()); + assert!(LivenessServer::start(root.path()).is_err()); + assert!(probe_liveness(root.path()).await.is_ok()); drop(server); fs::write(&path, "operator file").unwrap(); - assert!(LivenessServer::start(&root.0).is_err()); + assert!(LivenessServer::start(root.path()).is_err()); assert_eq!(fs::read_to_string(path).unwrap(), "operator file"); } diff --git a/container/crowdb-monitor/tests/member_trust_test.rs b/container/crowdb-monitor/tests/member_trust_test.rs new file mode 100644 index 000000000..df93c053a --- /dev/null +++ b/container/crowdb-monitor/tests/member_trust_test.rs @@ -0,0 +1,366 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::deployment::recovery::MemberTrust; +use crowdb_monitor::{accept_member_trust_for_tests, check_member_trust_for_tests}; +use crowdb_test_harness::test_dirs::tempdir_in_test_data; + +#[path = "common/member_trust.rs"] +mod trust; + +#[path = "common/run_root.rs"] +mod run_root; +use run_root::TestRunRoot; + +#[test] +fn duplicate_authenticated_incarnations_remove_the_route_in_any_order() { + use crowdb_protocol::mgmt::node::recovery::MemberEndpoint; + let first = MemberEndpoint { + node_id: 1, + discovery_id: uuid::Uuid::new_v4().to_string(), + boot_id: uuid::Uuid::new_v4().to_string(), + addresses: vec!["192.0.2.1".parse().unwrap()], + }; + let mut cloned = first.clone(); + cloned.boot_id = uuid::Uuid::new_v4().to_string(); + cloned.addresses = vec!["192.0.2.2".parse().unwrap()]; + for records in [vec![first.clone(), cloned.clone()], vec![cloned, first.clone()]] { + let (members, conflicts) = crowdb_monitor::unique_member_endpoints_for_tests(records); + assert!(members.is_empty()); + assert_eq!(conflicts, vec![first.discovery_id.clone()]); + } + let (members, conflicts) = + crowdb_monitor::unique_member_endpoints_for_tests(vec![first.clone(), first.clone()]); + assert_eq!(members, vec![first]); + assert!(conflicts.is_empty()); +} + +#[test] +fn bootstrap_trust_requires_every_previously_authenticated_key() { + let dir = tempdir_in_test_data("monitor-member-trust"); + let root = dir.path(); + let accepted = trust::fixture(root); + let mut changed = accepted.clone(); + changed.members[1].host_key = trust::key(4); + assert!(accept_member_trust_for_tests(root, &changed).is_err()); + assert!(!root.join("member-trust.json").exists()); + accept_member_trust_for_tests(root, &accepted).unwrap(); + check_member_trust_for_tests(root).unwrap(); + accept_member_trust_for_tests(root, &accepted).unwrap(); + assert_eq!( + MemberTrust::load(&root.join("member-trust.json")).unwrap(), + accepted + ); + let original = std::fs::read(root.join("member-trust.json")).unwrap(); + std::fs::write(root.join("ssh/known_hosts"), "").unwrap(); + assert!(accept_member_trust_for_tests(root, &accepted).is_err()); + assert_eq!(std::fs::read(root.join("member-trust.json")).unwrap(), original); +} + +#[test] +fn restart_requires_retained_trust_and_the_same_local_host_key() { + let dir = tempdir_in_test_data("monitor-member-restart"); + let root = dir.path(); + let accepted = trust::fixture(root); + assert!(check_member_trust_for_tests(root).is_err()); + accept_member_trust_for_tests(root, &accepted).unwrap(); + check_member_trust_for_tests(root).unwrap(); + std::fs::write(root.join("ssh/ssh_host_ed25519_key.pub"), trust::key(4)).unwrap(); + assert!(check_member_trust_for_tests(root).is_err()); +} + +#[test] +fn self_identity_and_retirement_cannot_be_overridden_by_a_coordinator() { + let dir = tempdir_in_test_data("monitor-member-self"); + let root = dir.path(); + let accepted = trust::fixture(root); + let mut changed = accepted.clone(); + changed.members[0].host_key = trust::key(4); + assert!(accept_member_trust_for_tests(root, &changed).is_err()); + trust::write( + &root.join(format!( + "retired-bootstrap-{}.json", + accepted.bootstrap.operation_id + )), + &true, + ); + assert!(accept_member_trust_for_tests(root, &accepted).is_err()); + assert!(!root.join("member-trust.json").exists()); +} + +#[test] +fn missing_preparation_and_corrupt_trust_fail_closed() { + let dir = tempdir_in_test_data("monitor-member-corrupt"); + let root = dir.path(); + let accepted = trust::fixture(root); + trust::write( + &root.join("accepted-node.json"), + &serde_json::json!({"node_id":1,"bootstrap":accepted.bootstrap,"prepared":false}), + ); + assert!(accept_member_trust_for_tests(root, &accepted).is_err()); + trust::write( + &root.join("accepted-node.json"), + &serde_json::json!({"node_id":1,"bootstrap":accepted.bootstrap,"prepared":true}), + ); + trust::write(&root.join("member-trust.json"), &"corrupt"); + assert!(accept_member_trust_for_tests(root, &accepted).is_err()); + assert_eq!( + std::fs::read_to_string(root.join("member-trust.json")).unwrap(), + "\"corrupt\"" + ); +} + +#[tokio::test] +async fn incomplete_endpoint_proof_does_not_rebind_unchanged_fresh_services() { + let dir = tempdir_in_test_data("monitor-partial-discovery-contacts"); + let root = dir.path(); + let _ = trust::fixture(root); + let operation = + crowdb_console_shared::deployment::PreparedBootstrap::load(&root.join("prepared-bootstrap.json")) + .unwrap(); + let seeds = ["http://192.0.2.1:10000".to_owned()]; + let binding = crowdb_protocol::mgmt::node::NodeBinding { + node_id: 1, + bootstrap: operation.identity.clone(), + management_seeds: (1..=3).map(|n| format!("http://192.0.2.{n}:10000")).collect(), + }; + trust::write(&root.join("node-binding.json"), &binding); + let original = std::fs::read(root.join("node-binding.json")).unwrap(); + // Even with no reachable Group-0 endpoint, unchanged addresses and a + // partial proof must leave accepted contacts and service operations alone. + assert!(!crowdb_monitor::rebind_recovery_services_for_tests( + root, + &operation, + 1, + "192.0.2.1", + &seeds, + false, + ) + .await + .unwrap()); + assert_eq!(std::fs::read(root.join("node-binding.json")).unwrap(), original); + assert!(!root.join("service-address.json").exists()); + assert!(!root.join("services").exists()); +} + +#[tokio::test] +async fn ssh_identity_command_is_independent_of_the_lifecycle_control_loop() { + use crowdb_protocol::mgmt::node::recovery::{MemberEndpoint, MemberRecoveryIdentity}; + let dir = TestRunRoot::new("member-identity"); + let root = dir.path(); + let trust = trust::fixture(root); + accept_member_trust_for_tests(root, &trust).unwrap(); + trust::write( + &root.join("node-endpoint.json"), + &MemberEndpoint { + discovery_id: trust.members[0].discovery_id.clone(), + node_id: 0, + boot_id: uuid::Uuid::new_v4().to_string(), + addresses: vec!["192.0.2.1".parse().unwrap()], + }, + ); + assert!( + crowdb_monitor::read_recovery_identity(root).await.is_err(), + "an offline root cannot prove live identity" + ); + let liveness = crowdb_monitor::LivenessServer::start(root).unwrap(); + let unavailable_control = root.join("unavailable-control.sock"); + let mut command = tokio::process::Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")); + command + .kill_on_drop(true) + .arg("control") + .arg("--socket") + .arg(&unavailable_control) + .arg("--data-root") + .arg(root) + .args(["--json", r#"{"command":"recovery_identity"}"#]); + let output = tokio::time::timeout(std::time::Duration::from_secs(3), command.output()) + .await + .expect("identity cannot wait for a busy lifecycle executor") + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let reply: MemberRecoveryIdentity = serde_json::from_slice(&output.stdout).unwrap(); + assert_eq!(reply.binding.bootstrap, trust.bootstrap); + assert_eq!(reply.binding.node_id, 1); + assert_eq!(reply.endpoint.discovery_id, trust.members[0].discovery_id); + drop(liveness); + assert!(crowdb_monitor::read_recovery_identity(root).await.is_err()); +} + +#[path = "common/admitted_member.rs"] +mod admitted; + +#[test] +fn confirmed_nonvoter_contacts_preserve_original_votes_and_require_exact_local_pin() { + let dir = tempdir_in_test_data("nonvoter-retained-trust"); + let root = dir.path(); + let snapshot = admitted::fixture(root); + check_member_trust_for_tests(root).unwrap(); + assert!(crowdb_monitor::member_trust_ready_for_tests(root).unwrap()); + let operation = + crowdb_console_shared::deployment::PreparedBootstrap::load(&root.join("prepared-bootstrap.json")) + .unwrap(); + let original = operation.intent.to_config(); + let contacts = snapshot.topology(&operation).unwrap(); + assert_eq!(operation.intent.members(), &[1, 2, 3]); + assert_eq!(contacts.nodes.len(), 4); + assert_eq!(contacts.stores, original.stores); + assert_eq!(contacts.groups, original.groups); + std::fs::write(root.join("ssh/ssh_host_ed25519_key.pub"), trust::key(5)).unwrap(); + assert!(check_member_trust_for_tests(root).is_err()); + assert!(crowdb_monitor::member_trust_ready_for_tests(root).is_err()); +} + +#[test] +fn pending_missing_corrupt_and_retired_nonvoter_trust_never_authorize_resume() { + for change in ["pending", "missing", "corrupt", "foreign", "cancelled", "retired"] { + let dir = tempdir_in_test_data("nonvoter-trust-fence"); + let root = dir.path(); + let mut snapshot = admitted::fixture(root); + let path = root.join("confirmed-member-trust.json"); + match change { + "pending" => { + snapshot.registry.nodes[3].confirmed = false; + snapshot.current = snapshot.initial.confirmed_registry(&snapshot.registry).unwrap(); + trust::write(&path, &snapshot); + } + "missing" => std::fs::remove_file(&path).unwrap(), + "corrupt" => std::fs::write(&path, "corrupt").unwrap(), + "foreign" => { + snapshot.current.bootstrap.cluster_id = uuid::Uuid::new_v4().to_string(); + trust::write(&path, &snapshot); + } + "cancelled" => trust::write( + &root.join(format!( + "ssh/cancelled-{}.json", + snapshot.registry.nodes[3].operation_id + )), + &true, + ), + "retired" => trust::write( + &root.join(format!( + "retired-bootstrap-{}.json", + snapshot.initial.bootstrap.operation_id + )), + &true, + ), + _ => unreachable!(), + } + assert!( + !crowdb_monitor::member_trust_ready_for_tests(root).unwrap_or(false), + "{change}" + ); + } +} + +#[tokio::test] +async fn confirmed_nonvoter_can_prove_identity_without_live_group_zero_but_pending_cannot() { + use crowdb_protocol::mgmt::node::recovery::MemberEndpoint; + let dir = TestRunRoot::new("nonvoter-ssh-recovery"); + let root = dir.path(); + let mut snapshot = admitted::fixture(root); + trust::write( + &root.join("node-endpoint.json"), + &MemberEndpoint { + discovery_id: snapshot.registry.nodes[3].discovery_id.clone(), + node_id: 0, + boot_id: uuid::Uuid::new_v4().to_string(), + addresses: vec!["192.0.2.44".parse().unwrap()], + }, + ); + let liveness = crowdb_monitor::LivenessServer::start(root).unwrap(); + let identity = crowdb_monitor::read_recovery_identity(root).await.unwrap(); + assert_eq!(identity.binding.node_id, 4); + assert_eq!( + identity.endpoint.addresses, + vec!["192.0.2.44".parse::().unwrap()] + ); + snapshot.registry.nodes[3].confirmed = false; + snapshot.current = snapshot.initial.confirmed_registry(&snapshot.registry).unwrap(); + trust::write(&root.join("confirmed-member-trust.json"), &snapshot); + assert!(crowdb_monitor::read_recovery_identity(root).await.is_err()); + drop(liveness); +} + +#[tokio::test] +async fn adding_management_seeds_preserves_unchanged_service_operations() { + let dir = tempdir_in_test_data("monitor-added-recovery-contacts"); + let root = dir.path(); + let _ = trust::fixture(root); + let operation = + crowdb_console_shared::deployment::PreparedBootstrap::load(&root.join("prepared-bootstrap.json")) + .unwrap(); + let seeds: Vec<_> = (1..=3).map(|n| format!("http://192.0.2.{n}:10000")).collect(); + let binding = crowdb_protocol::mgmt::node::NodeBinding { + node_id: 1, + bootstrap: operation.identity.clone(), + management_seeds: seeds.clone(), + }; + trust::write(&root.join("node-binding.json"), &binding); + let mut expanded = seeds; + expanded.push("http://192.0.2.4:10000".into()); + assert!(crowdb_monitor::rebind_recovery_services_for_tests( + root, + &operation, + 1, + "192.0.2.1", + &expanded, + true, + ) + .await + .expect("adding contacts cannot publish replacement service operations")); + let address: String = + serde_json::from_slice(&std::fs::read(root.join("service-address.json")).unwrap()).unwrap(); + assert_eq!(address, "192.0.2.1"); +} + +#[test] +fn a_new_admission_keeps_the_cancelled_operations_pins_and_fence() { + let dir = tempdir_in_test_data("nonvoter-readmission-trust"); + let root = dir.path(); + let mut snapshot = admitted::fixture(root); + let old_operation = snapshot.registry.nodes[3].operation_id.clone(); + let old_path = root.join(format!("admitted-member-trust-{old_operation}.json")); + let old_bytes = std::fs::read(&old_path).unwrap(); + trust::write( + &root.join(format!("ssh/cancelled-{old_operation}.json")), + &old_operation, + ); + let new_operation = uuid::Uuid::new_v4().to_string(); + snapshot.registry.nodes[3].operation_id.clone_from(&new_operation); + let mut pins = snapshot.current.clone(); + pins.members[3].admission_operation_id.clone_from(&new_operation); + pins.seal(&root.join(format!("admitted-member-trust-{new_operation}.json"))) + .unwrap(); + trust::write( + &root.join("accepted-node.json"), + &serde_json::json!({ + "node_id":4,"bootstrap":snapshot.initial.bootstrap,"prepared":true, + "admission":{"operation_id":new_operation,"management_seeds":["http://192.0.2.1:10000"]} + }), + ); + check_member_trust_for_tests(root).expect("a fresh admission must use its own immutable pins"); + assert!( + !crowdb_monitor::member_trust_ready_for_tests(root).unwrap(), + "the new operation cannot inherit the cancelled operation's confirmation" + ); + snapshot.current = pins; + trust::write(&root.join("confirmed-member-trust.json"), &snapshot); + assert!(crowdb_monitor::member_trust_ready_for_tests(root).unwrap()); + assert_eq!(std::fs::read(&old_path).unwrap(), old_bytes); + trust::write( + &root.join("accepted-node.json"), + &serde_json::json!({ + "node_id":4,"bootstrap":snapshot.initial.bootstrap,"prepared":true, + "admission":{"operation_id":old_operation,"management_seeds":["http://192.0.2.1:10000"]} + }), + ); + assert!( + check_member_trust_for_tests(root).is_err(), + "the cancelled operation remains fenced" + ); +} diff --git a/container/crowdb-monitor/tests/node_candidates_test.rs b/container/crowdb-monitor/tests/node_candidates_test.rs index 9805e6043..cb468c8c2 100644 --- a/container/crowdb-monitor/tests/node_candidates_test.rs +++ b/container/crowdb-monitor/tests/node_candidates_test.rs @@ -11,6 +11,7 @@ fn node() -> NodeAdvertisement { protocol_version: NODE_PROTOCOL_VERSION, monitor_endpoints: vec!["http://192.0.2.1:9093".into()], cluster_id: None, + formation: None, } } @@ -62,3 +63,39 @@ fn malformed_metadata_is_not_a_candidate() { assert!(!cache.observe("b".into(), peer)); assert!(cache.snapshot(&node()).is_empty()); } + +#[test] +fn scope_mode_and_initial_count_are_checked_before_a_peer_is_eligible() { + use crowdb_protocol::mgmt::node::deployment::{DeploymentMode, NodeDeploymentPolicy}; + let mut local = node(); + local.formation = Some(NodeDeploymentPolicy::default().formation()); + let mut cache = CandidateCache::default(); + let mut peer = node(); + peer.formation.clone_from(&local.formation); + cache.observe("peer".into(), peer.clone()); + assert_eq!(cache.snapshot(&local)[0].state, CandidateState::Unbound); + for policy in [ + None, + Some(crowdb_protocol::mgmt::node::deployment::FormationPolicy { + cluster: "other".into(), + ..local.formation.clone().unwrap() + }), + Some(crowdb_protocol::mgmt::node::deployment::FormationPolicy { + mode: DeploymentMode::Production, + ..local.formation.clone().unwrap() + }), + Some(crowdb_protocol::mgmt::node::deployment::FormationPolicy { + initial_voters: 3, + ..local.formation.clone().unwrap() + }), + ] { + peer.formation = policy; + assert!(cache.observe("peer".into(), peer.clone())); + assert_eq!(cache.snapshot(&local)[0].state, CandidateState::Incompatible); + } + peer.formation = Some(crowdb_protocol::mgmt::node::deployment::FormationPolicy { + version: 99, + ..local.formation.unwrap() + }); + assert!(!cache.observe("invalid".into(), peer)); +} diff --git a/container/crowdb-monitor/tests/node_cleanup_test.rs b/container/crowdb-monitor/tests/node_cleanup_test.rs new file mode 100644 index 000000000..d878c5e7c --- /dev/null +++ b/container/crowdb-monitor/tests/node_cleanup_test.rs @@ -0,0 +1,146 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::{cleanup_node_for_tests, DeploymentProfile, ProcessManager}; +use crowdb_protocol::mgmt::SystemBootstrapIdentity; +use crowdb_test_harness::test_dirs::tempdir_in_test_data; +use std::{os::unix::fs::PermissionsExt, path::Path}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; + +#[tokio::test] +async fn retired_cleanup_restarts_its_local_worker_after_a_failed_deletion() { + let directory = tempdir_in_test_data("monitor-retired-cleanup"); + let root = directory.path(); + let bootstrap = SystemBootstrapIdentity { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + configuration_digest: "a".repeat(64), + }; + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("logs"); + let program = root.join("test-worker"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + profile + .services + .iter_mut() + .find(|service| service.id == "kv") + .unwrap() + .program = program; + let accepted = serde_json::json!({"node_id":1,"bootstrap":bootstrap,"prepared":true}); + std::fs::write( + root.join("accepted-node.json"), + serde_json::to_vec(&accepted).unwrap(), + ) + .unwrap(); + std::fs::set_permissions( + root.join("accepted-node.json"), + std::fs::Permissions::from_mode(0o600), + ) + .unwrap(); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + let mut manager = ProcessManager::new(profile.paths.log_root.clone(), profile.logs.clone()) + .await + .unwrap(); + let contacts = root.join("confirmed-member-trust.json"); + let retained_contacts = serde_json::to_vec(&serde_json::json!({"bootstrap":bootstrap})).unwrap(); + std::fs::write(&contacts, &retained_contacts).unwrap(); + let retired = root.join(format!("retired-bootstrap-{}.json", bootstrap.operation_id)); + assert!( + cleanup_node_for_tests(&profile, &mut manager, bootstrap.clone(), false) + .await + .is_err() + ); + let mut wrong = bootstrap.clone(); + wrong.cluster_id = uuid::Uuid::new_v4().to_string(); + assert!(cleanup_node_for_tests(&profile, &mut manager, wrong, true) + .await + .is_err()); + assert!(!retired.exists()); + assert!(!manager.owns("kv")); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:10000").await.unwrap(); + let endpoint = tokio::spawn(serve_deletions(listener, bootstrap.clone(), retired.clone())); + assert!( + cleanup_node_for_tests(&profile, &mut manager, bootstrap.clone(), true) + .await + .is_err() + ); + assert!(retired.exists()); + assert!(root.join("accepted-node.json").exists()); + assert_eq!(std::fs::read(&contacts).unwrap(), retained_contacts); + assert!(!manager.owns("kv")); + cleanup_node_for_tests(&profile, &mut manager, bootstrap.clone(), true) + .await + .unwrap(); + endpoint.await.unwrap(); + assert!(retired.exists()); + assert!(!root.join("accepted-node.json").exists()); + assert!( + !contacts.exists(), + "retired cluster contact pins cannot poison a later admission" + ); + assert!(root.join("node-cleaned.json").exists()); + assert!(!manager.owns("kv")); + cleanup_node_for_tests(&profile, &mut manager, bootstrap, true) + .await + .unwrap(); +} + +async fn serve_deletions( + listener: tokio::net::TcpListener, + expected: SystemBootstrapIdentity, + witness: std::path::PathBuf, +) { + let mut deletions = 0; + while deletions < 2 { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + let mut buffer = [0; 4096]; + let body = loop { + let count = stream.read(&mut buffer).await.unwrap(); + assert!(count > 0); + bytes.extend_from_slice(&buffer[..count]); + if let Some(index) = bytes.windows(4).position(|value| value == b"\r\n\r\n") { + let headers = String::from_utf8_lossy(&bytes[..index]); + let length: usize = headers + .lines() + .find_map(|line| { + line.to_ascii_lowercase() + .strip_prefix("content-length: ")? + .parse() + .ok() + }) + .unwrap_or(0); + if bytes.len() >= index + 4 + length { + break bytes[index + 4..].to_vec(); + } + } + }; + let deletion = bytes.starts_with(b"POST /system/cleanup "); + let status = if deletion { + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(value["bootstrap"], serde_json::to_value(&expected).unwrap()); + assert_eq!(value["confirm_delete_system_store"], true); + assert!(witness.exists()); + deletions += 1; + if deletions == 1 { + "503 Service Unavailable" + } else { + "200 OK" + } + } else { + assert!(bytes.starts_with(b"GET /health ")); + "200 OK" + }; + stream + .write_all( + format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n").as_bytes(), + ) + .await + .unwrap(); + } +} diff --git a/container/crowdb-monitor/tests/node_dispatch_test.rs b/container/crowdb-monitor/tests/node_dispatch_test.rs new file mode 100644 index 000000000..6c43e71dd --- /dev/null +++ b/container/crowdb-monitor/tests/node_dispatch_test.rs @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::NodeDispatchForTests; +use std::time::Duration; +use tokio::net::{UnixListener, UnixStream}; + +#[path = "common/run_root.rs"] +mod run_root; + +#[tokio::test] +async fn queued_control_and_overdue_health_both_make_progress() { + let root = run_root::TestRunRoot::new("node-dispatch"); + let socket = root.path().join("control.sock"); + let listener = UnixListener::bind(&socket).unwrap(); + let mut queued = Vec::new(); + for _ in 0..16 { + queued.push(UnixStream::connect(&socket).await.unwrap()); + } + let mut heartbeat = tokio::time::interval(Duration::from_secs(1)); + let mut dispatch = NodeDispatchForTests::default(); + let mut controls = 0; + let mut health = 0; + for _ in 0..16 { + heartbeat.reset_at(tokio::time::Instant::now() - Duration::from_secs(1)); + if dispatch.next(&listener, &mut heartbeat).await.unwrap().is_some() { + controls += 1; + } else { + health += 1; + } + } + assert!(controls >= 8, "overdue health starved queued control: {controls}"); + assert!( + health >= 8, + "control traffic starved service reconciliation: {health}" + ); +} diff --git a/container/crowdb-monitor/tests/node_health_test.rs b/container/crowdb-monitor/tests/node_health_test.rs new file mode 100644 index 000000000..f2086bee8 --- /dev/null +++ b/container/crowdb-monitor/tests/node_health_test.rs @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::{node_health_for_tests, DeploymentProfile, MonitorPhase, ProcessManager}; +use std::{ + collections::BTreeMap, + path::Path, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Duration, +}; + +#[tokio::test] +async fn web_health_uses_its_probe_and_recovers_without_restarting_a_live_process() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-web-health"); + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.path().to_path_buf(); + profile.paths.run_root = root.path().join("run"); + profile.paths.log_root = root.path().join("log"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + let ready = Arc::new(AtomicBool::new(false)); + let observed = Arc::clone(&ready); + let router = axum::Router::new().route( + "/healthz", + axum::routing::get(move || { + let ready = Arc::clone(&observed); + async move { + if ready.load(Ordering::Acquire) { + axum::http::StatusCode::OK + } else { + axum::http::StatusCode::SERVICE_UNAVAILABLE + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + let mut web = profile + .services + .iter() + .find(|service| service.id == "web") + .unwrap() + .clone(); + web.program = "/bin/sleep".into(); + web.args = vec!["60".into()]; + web.probe.target = format!("http://{address}/healthz"); + let mut manager = ProcessManager::new(profile.paths.log_root.clone(), profile.logs.clone()) + .await + .unwrap(); + let pid = manager.start(&web, &BTreeMap::new()).await.unwrap(); + let unhealthy = node_health_for_tests(&profile, &web, &mut manager).await.unwrap(); + ready.store(true, Ordering::Release); + let recovered = node_health_for_tests(&profile, &web, &mut manager).await.unwrap(); + let retained_pid = manager.pid("web"); + manager.stop("web", Duration::from_secs(2)).await.unwrap(); + server.abort(); + assert!(!unhealthy.services["web"].healthy); + assert_eq!(unhealthy.phase, MonitorPhase::Restarting); + assert!(recovered.services["web"].healthy); + assert_eq!(recovered.phase, MonitorPhase::Ready); + assert_eq!(unhealthy.services["web"].pid, Some(pid)); + assert_eq!(recovered.services["web"].pid, Some(pid)); + assert_eq!(retained_pid, Some(pid)); + assert_eq!(recovered.services["web"].restart_attempts, 0); +} diff --git a/container/crowdb-monitor/tests/node_management_inputs_test.rs b/container/crowdb-monitor/tests/node_management_inputs_test.rs new file mode 100644 index 000000000..a02b18179 --- /dev/null +++ b/container/crowdb-monitor/tests/node_management_inputs_test.rs @@ -0,0 +1,71 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::{run_node, serve_node_management, DiscoveryConfig, NodeRuntimeConfig}; + +#[tokio::test] +async fn invalid_seeds_fail_before_identity_creation_or_listener_binding_without_exposing_credentials() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("invalid-discovery-seeds"); + let occupied = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let bind = occupied.local_addr().unwrap(); + let mut cases = vec![ + vec!["not-an-origin".into()], + vec!["https://127.0.0.1".into()], + vec!["http://operator:private-token@127.0.0.1".into()], + vec!["http://127.0.0.1/node".into()], + vec!["http://127.0.0.1/?private-token".into()], + vec!["http://127.0.0.1/#private-token".into()], + ]; + cases.push(vec!["http://127.0.0.1".into(); 65]); + for seeds in cases { + let config = DiscoveryConfig { + seeds, + interfaces: vec!["lo".into()], + addresses: vec![bind.ip()], + monitor_port: bind.port(), + cluster_id: None, + }; + let error = serve_node_management(root.path(), bind, &config, "test-host".into(), async {}) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("discovery seeds"), "{error}"); + assert!(!error.contains("physical host"), "{error}"); + assert!( + !error.contains("operator") && !error.contains("private-token"), + "{error}" + ); + let runtime = NodeRuntimeConfig { + profile: root.path().join("absent-profile.toml"), + bind, + discovery: config, + physical_host_id: "test-host".into(), + }; + let error = run_node(runtime).await.unwrap_err().to_string(); + assert!(error.contains("discovery seeds"), "{error}"); + assert!(!error.contains("private-token"), "{error}"); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); + } +} + +#[test] +fn container_preflight_rejects_seed_credentials_before_accepting_policy() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("invalid-seed-preflight"); + for command in ["accept-policy", "run"] { + let output = std::process::Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")) + .args([command, "--profile"]) + .arg(root.path().join("absent-profile.toml")) + .env_remove("CROWDB_NODE_POLICY_FILE") + .env( + "CROWDB_DISCOVERY_SEEDS", + "http://operator:private-token@127.0.0.1", + ) + .output() + .unwrap(); + assert!(!output.status.success()); + let diagnostic = String::from_utf8_lossy(&output.stderr); + assert!(diagnostic.contains("discovery seeds"), "{diagnostic}"); + assert!(!diagnostic.contains("private-token"), "{diagnostic}"); + assert_eq!(std::fs::read_dir(root.path()).unwrap().count(), 0); + } +} diff --git a/container/crowdb-monitor/tests/node_management_test.rs b/container/crowdb-monitor/tests/node_management_test.rs index 1e5698381..865b180fe 100644 --- a/container/crowdb-monitor/tests/node_management_test.rs +++ b/container/crowdb-monitor/tests/node_management_test.rs @@ -11,6 +11,74 @@ use crowdb_protocol::mgmt::node::{CandidateSnapshot, NodeHandshake}; use tokio::sync::oneshot; use uuid::Uuid; +#[tokio::test] +async fn management_advertises_accepted_formation_without_storage_preparation() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-management-policy"); + let inputs = crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides { + cluster: Some("discovery-scope".into()), + initial_voters: Some(3), + ..Default::default() + }; + let policy = crowdb_monitor::accept_node_policy( + root.path(), + &crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides::default(), + &inputs, + ) + .unwrap(); + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let bind = listener.local_addr().unwrap(); + drop(listener); + let config = DiscoveryConfig { + seeds: vec![], + interfaces: vec!["lo".into()], + addresses: vec![bind.ip()], + monitor_port: bind.port(), + cluster_id: None, + }; + let (stop, stopped) = oneshot::channel(); + let data_root = root.path().to_path_buf(); + let server = tokio::spawn(async move { + serve_node_management(&data_root, bind, &config, "shared-host".into(), async { + let _ = stopped.await; + }) + .await + }); + let client = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(1)) + .build() + .unwrap(); + let handshake = tokio::time::timeout(Duration::from_secs(3), async { + loop { + assert!( + !server.is_finished(), + "management exited before policy advertisement" + ); + if let Ok(reply) = client.get(format!("http://{bind}/node")).send().await { + break reply + .error_for_status() + .unwrap() + .json::() + .await + .unwrap(); + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(handshake.advertisement.formation, Some(policy.formation())); + assert_eq!(handshake.advertisement.cluster_id, None); + assert!(!root.path().join("disks").exists()); + assert!(!root.path().join("accepted-node.json").exists()); + stop.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(5), server) + .await + .unwrap() + .unwrap() + .unwrap(); +} + #[tokio::test] async fn management_exposes_persistent_identity_and_requires_physical_host() { let root = std::env::temp_dir().join(format!("crowdb-node-management-{}", Uuid::new_v4())); @@ -188,6 +256,7 @@ async fn test_seed() -> (String, String, tokio::task::JoinHandle<()>, oneshot::S protocol_version: NODE_PROTOCOL_VERSION, monitor_endpoints: vec![seed.clone()], cluster_id: None, + formation: None, }, physical_host_id: "seed-only-host".into(), rack_hint: None, diff --git a/container/crowdb-monitor/tests/node_policy_test.rs b/container/crowdb-monitor/tests/node_policy_test.rs new file mode 100644 index 000000000..eeca7391d --- /dev/null +++ b/container/crowdb-monitor/tests/node_policy_test.rs @@ -0,0 +1,181 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::fs; +use std::os::unix::fs::{symlink, PermissionsExt}; + +use crowdb_monitor::{accept_node_policy, read_node_policy, BootstrapSession, LivenessServer, NodeIdentity}; +use crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides; + +#[path = "common/run_root.rs"] +mod run_root; + +#[test] +fn accepted_policy_survives_restart_without_defaults_or_identity_effects() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("accepted-node-policy"); + let omitted = NodeDeploymentOverrides::default(); + let inputs = NodeDeploymentOverrides { + cluster: Some("durable-cluster".into()), + initial_voters: Some(3), + ..Default::default() + }; + let accepted = accept_node_policy(root.path(), &omitted, &inputs).unwrap(); + let before = fs::read(root.path().join("node-policy.json")).unwrap(); + assert_eq!( + accept_node_policy(root.path(), &omitted, &omitted).unwrap(), + accepted + ); + assert_eq!(read_node_policy(root.path()).unwrap(), Some(accepted)); + assert_eq!(fs::read(root.path().join("node-policy.json")).unwrap(), before); + assert!(!root.path().join("node-identity").exists()); + assert!(!root.path().join("disks").exists()); + assert_eq!( + fs::metadata(root.path().join("node-policy.json")) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); +} + +#[test] +fn incompatible_overrides_preserve_policy_and_retained_identity_bytes() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-conflict"); + let omitted = NodeDeploymentOverrides::default(); + accept_node_policy(root.path(), &omitted, &omitted).unwrap(); + let identity = NodeIdentity::load_or_create(root.path()).unwrap(); + let before = fs::read(root.path().join("node-policy.json")).unwrap(); + for inputs in [ + NodeDeploymentOverrides { + initial_voters: Some(3), + ..Default::default() + }, + NodeDeploymentOverrides { + cluster: Some("foreign".into()), + ..Default::default() + }, + NodeDeploymentOverrides { + version: Some(99), + ..Default::default() + }, + ] { + assert!(accept_node_policy(root.path(), &omitted, &inputs).is_err()); + assert_eq!(fs::read(root.path().join("node-policy.json")).unwrap(), before); + assert_eq!(NodeIdentity::load_or_create(root.path()).unwrap(), identity); + } + assert!(BootstrapSession::open(root.path(), b"profile", b"config", &["kv"]).is_err()); + assert!(!root.path().join("bootstrap").exists()); +} + +#[test] +fn missing_identity_of_an_accepted_deployment_cannot_generate_a_replacement() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-missing-identity"); + let omitted = NodeDeploymentOverrides::default(); + accept_node_policy(root.path(), &omitted, &omitted).unwrap(); + fs::write(root.path().join("accepted-node.json"), b"retained bootstrap").unwrap(); + assert!(accept_node_policy(root.path(), &omitted, &omitted).is_err()); + assert!(!root.path().join("node-identity").exists()); + assert_eq!( + fs::read(root.path().join("accepted-node.json")).unwrap(), + b"retained bootstrap" + ); +} + +#[test] +fn obsolete_and_unrecognized_populated_roots_fail_before_new_state() { + for name in [ + "startup-mode.json", + "node-identity", + "accepted-node.json", + "operator-data", + ] { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-populated"); + fs::write(root.path().join(name), b"retained bytes").unwrap(); + let inputs = NodeDeploymentOverrides::default(); + assert!(accept_node_policy(root.path(), &inputs, &inputs).is_err()); + assert_eq!(fs::read(root.path().join(name)).unwrap(), b"retained bytes"); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 1); + } +} + +#[test] +fn corrupt_unknown_or_symlinked_policy_is_never_replaced() { + for bytes in [b"corrupt".as_slice(), br#"{"version":99}"#.as_slice()] { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-invalid"); + let path = root.path().join("node-policy.json"); + fs::write(&path, bytes).unwrap(); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let inputs = NodeDeploymentOverrides::default(); + assert!(accept_node_policy(root.path(), &inputs, &inputs).is_err()); + assert_eq!(fs::read(path).unwrap(), bytes); + assert!(!root.path().join("node-identity").exists()); + } + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-symlink"); + let target = root.path().join("operator-data"); + fs::write(&target, b"preserve").unwrap(); + symlink(&target, root.path().join("node-policy.json")).unwrap(); + let inputs = NodeDeploymentOverrides::default(); + assert!(accept_node_policy(root.path(), &inputs, &inputs).is_err()); + assert_eq!(fs::read(target).unwrap(), b"preserve"); +} + +#[test] +fn generated_ssh_directory_is_allowed_but_automatic_storage_never_adopts_files() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-preparation"); + fs::create_dir(root.path().join("ssh")).unwrap(); + fs::write(root.path().join("ssh/id_ed25519"), b"existing key").unwrap(); + fs::create_dir(root.path().join("disks")).unwrap(); + fs::write(root.path().join("disks/foreign.img"), b"retained media").unwrap(); + let inputs = NodeDeploymentOverrides::default(); + assert!(accept_node_policy(root.path(), &inputs, &inputs).is_err()); + assert_eq!( + fs::read(root.path().join("disks/foreign.img")).unwrap(), + b"retained media" + ); + fs::remove_file(root.path().join("disks/foreign.img")).unwrap(); + accept_node_policy(root.path(), &inputs, &inputs).unwrap(); + assert_eq!( + fs::read(root.path().join("ssh/id_ed25519")).unwrap(), + b"existing key" + ); +} + +#[test] +fn concurrent_incompatible_publishers_accept_exactly_one_policy() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-policy-race"); + let outcomes = std::thread::scope(|scope| { + let start = |cluster: &'static str| { + let omitted = NodeDeploymentOverrides::default(); + accept_node_policy( + root.path(), + &omitted, + &NodeDeploymentOverrides { + cluster: Some(cluster.into()), + ..Default::default() + }, + ) + }; + let first = scope.spawn(move || start("first")); + let second = scope.spawn(move || start("second")); + [first.join().unwrap(), second.join().unwrap()] + }); + assert_eq!(outcomes.iter().filter(|outcome| outcome.is_ok()).count(), 1); + assert_eq!( + Some(outcomes.into_iter().find_map(Result::ok).unwrap()), + read_node_policy(root.path()).unwrap() + ); + assert!(!root.path().join("node-identity").exists()); +} + +#[tokio::test] +async fn root_guard_rejects_another_monitor_even_with_a_private_run_directory() { + let root = run_root::TestRunRoot::new("node-policy-owner"); + let inputs = NodeDeploymentOverrides::default(); + accept_node_policy(root.path(), &inputs, &inputs).unwrap(); + let owner = LivenessServer::start(root.path()).unwrap(); + assert!(LivenessServer::start(root.path()).is_err()); + assert!(!root.path().join("node-identity").exists()); + drop(owner); + let _replacement = LivenessServer::start(root.path()).unwrap(); +} diff --git a/container/crowdb-monitor/tests/node_profile_test.rs b/container/crowdb-monitor/tests/node_profile_test.rs new file mode 100644 index 000000000..2089dd209 --- /dev/null +++ b/container/crowdb-monitor/tests/node_profile_test.rs @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::DeploymentProfile; +use std::{collections::BTreeMap, path::Path}; + +fn catalog() -> DeploymentProfile { + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap() +} + +#[test] +fn node_catalog_has_no_prescribed_members_media_or_rendered_service_configuration() { + let profile = catalog(); + profile.validate_node_catalog().unwrap(); + assert!(profile.nodes.is_empty() && profile.groups.is_empty() && profile.disks.is_empty()); + assert!(profile + .services + .iter() + .all(|service| service.args.is_empty() && service.config_template.is_none())); + let web = profile + .services + .iter() + .find(|service| service.id == "web") + .unwrap(); + assert!( + web.dependencies.is_empty(), + "management starts before cluster preparation" + ); + let endpoints: BTreeMap<_, _> = profile + .public_endpoints + .iter() + .map(|endpoint| (endpoint.id.as_str(), endpoint.port)) + .collect(); + assert_eq!( + endpoints, + BTreeMap::from([("web", 9090), ("s3", 9091), ("iceberg", 9092), ("dataset", 9093)]) + ); +} + +#[test] +fn incomplete_catalog_and_preset_or_partial_topology_are_rejected() { + let mut missing = catalog(); + missing.services.pop(); + assert!(missing.validate_node_catalog().is_err()); + let mut unknown = catalog(); + unknown.services.last_mut().unwrap().id = "unknown".into(); + assert!(unknown.validate_node_catalog().is_err()); + let mut partial = catalog(); + partial.nodes.push(crowdb_monitor::NodeProfile { + node_id: 1, + rack_id: 1, + }); + assert!(partial.validate().is_err()); + let fixture = DeploymentProfile::load( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../single-node-container/profile.toml"), + ) + .unwrap(); + fixture.validate().unwrap(); + assert!(fixture.validate_node_catalog().is_err()); +} + +#[test] +fn obsolete_startup_modes_are_absent_from_monitor_arguments() { + for mode in ["single", "manual"] { + let output = std::process::Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")) + .args(["run", "--mode", mode]) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("unexpected argument '--mode'")); + } +} diff --git a/container/crowdb-monitor/tests/node_run_inputs_test.rs b/container/crowdb-monitor/tests/node_run_inputs_test.rs new file mode 100644 index 000000000..f7e0ba356 --- /dev/null +++ b/container/crowdb-monitor/tests/node_run_inputs_test.rs @@ -0,0 +1,243 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::{fs, path::Path, process::Command}; + +use crowdb_monitor::accept_node_policy; +use crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides; + +fn profile(root: &Path) -> std::path::PathBuf { + let install = root.join("install"); + fs::create_dir_all(install.join("data")).unwrap(); + let template = Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"); + let text = fs::read_to_string(template) + .unwrap() + .replace("/opt/crowdb", install.to_str().unwrap()); + let path = root.join("profile.toml"); + fs::write(&path, text).unwrap(); + path +} + +fn command(profile: &Path) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")); + command.args(["run", "--profile"]).arg(profile); + for name in [ + "CROWDB_NODE_POLICY_FILE", + "CROWDB_DEPLOYMENT_MODE", + "CROWDB_CLUSTER", + "CROWDB_INITIAL_VOTERS", + "CROWDB_PHYSICAL_HOST_ID", + "CROWDB_S3_PUBLIC_URI", + "CROWDB_ICEBERG_PUBLIC_URI", + "CROWDB_DATASET_PUBLIC_URI", + ] { + command.env_remove(name); + } + command +} + +#[test] +fn public_uri_flags_override_environment_before_acceptance_and_changed_restarts_fail() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("public-origin"); + let profile = profile(root.path()); + let output = command(&profile) + .arg("--s3-public-uri") + .arg("http://127.0.0.1:19091") + .env("CROWDB_S3_PUBLIC_URI", "http://user:password@invalid") + .args(["--initial-voters", "2"]) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("initial voters")); + let mut accept = Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")); + accept + .args(["accept-policy", "--profile"]) + .arg(&profile) + .env("CROWDB_S3_PUBLIC_URI", "http://127.0.0.1:19091") + .env("CROWDB_ICEBERG_PUBLIC_URI", "http://127.0.0.1:19092") + .env("CROWDB_DATASET_PUBLIC_URI", "http://127.0.0.1:19093"); + let output = accept.output().unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + let data = root.path().join("install/data"); + let before = fs::read(data.join("node-policy.json")).unwrap(); + let policy = crowdb_monitor::read_node_policy(&data).unwrap().unwrap(); + assert_eq!( + policy.public_access.dataset.as_deref(), + Some("http://127.0.0.1:19093") + ); + let output = command(&profile) + .arg("--dataset-public-uri") + .arg("http://127.0.0.1:29093") + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("accepted node policy")); + assert_eq!(fs::read(data.join("node-policy.json")).unwrap(), before); + assert_eq!(fs::read_dir(data).unwrap().count(), 1); +} + +#[test] +fn invalid_voter_override_fails_before_node_identity_and_storage_effects() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-cli-invalid-voters"); + let profile = profile(root.path()); + let output = command(&profile) + .args(["--initial-voters", "2"]) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("initial voters must be one or three")); + assert_eq!(fs::read_dir(root.path().join("install/data")).unwrap().count(), 0); +} + +#[test] +fn explicit_flags_override_environment_and_file_without_rewriting_accepted_inputs() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-cli-input-precedence"); + let profile = profile(root.path()); + let data = root.path().join("install/data"); + let inputs = NodeDeploymentOverrides { + cluster: Some("retained".into()), + initial_voters: Some(3), + ..Default::default() + }; + accept_node_policy(&data, &NodeDeploymentOverrides::default(), &inputs).unwrap(); + let before = fs::read(data.join("node-policy.json")).unwrap(); + let config = root.path().join("node.toml"); + fs::write(&config, "cluster = 'file-cluster'\ninitial_voters = 1\n").unwrap(); + let output = command(&profile) + .arg("--deployment-config") + .arg(config) + .env("CROWDB_CLUSTER", "environment-cluster") + .env("CROWDB_INITIAL_VOTERS", "1") + .args([ + "--cluster", + "retained", + "--initial-voters", + "3", + "--physical-host-id", + "", + ]) + .output() + .unwrap(); + assert!(!output.status.success()); + let error = String::from_utf8_lossy(&output.stderr); + assert!( + error.contains("physical host identity must contain 1 to 256 bytes"), + "{error}" + ); + assert_eq!(fs::read(data.join("node-policy.json")).unwrap(), before); + assert!(!data.join("node-identity").exists()); +} + +#[test] +fn production_cannot_fall_back_to_local_host_identity() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-cli-production"); + let profile = profile(root.path()); + let output = command(&profile) + .env("CROWDB_DEPLOYMENT_MODE", "production") + .output() + .unwrap(); + assert!(!output.status.success()); + assert!( + String::from_utf8_lossy(&output.stderr).contains("production deployment requires physical-host-id") + ); + assert_eq!(fs::read_dir(root.path().join("install/data")).unwrap().count(), 0); +} + +#[test] +fn unknown_file_version_and_corrupt_inputs_do_not_acquire_policy() { + for content in ["version = 99\n", "invalid = true\n", "not TOML"] { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-cli-config-invalid"); + let profile = profile(root.path()); + let config = root.path().join("node.toml"); + fs::write(&config, content).unwrap(); + let output = command(&profile) + .env("CROWDB_NODE_POLICY_FILE", config) + .output() + .unwrap(); + assert!(!output.status.success()); + assert_eq!(fs::read_dir(root.path().join("install/data")).unwrap().count(), 0); + } +} + +#[test] +fn container_preflight_seals_policy_without_generating_keys_identity_or_storage() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-cli-preflight"); + let profile = profile(root.path()); + let invoke = |voters: Option<&str>| { + let mut command = Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")); + command.args(["accept-policy", "--profile"]).arg(&profile); + for name in [ + "CROWDB_NODE_POLICY_FILE", + "CROWDB_DEPLOYMENT_MODE", + "CROWDB_CLUSTER", + "CROWDB_INITIAL_VOTERS", + "CROWDB_PHYSICAL_HOST_ID", + ] { + command.env_remove(name); + } + if let Some(voters) = voters { + command.args(["--initial-voters", voters]); + } + command.output().unwrap() + }; + let output = invoke(Some("3")); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.stdout, b"local\n"); + let data = root.path().join("install/data"); + assert_eq!(fs::read_dir(&data).unwrap().count(), 1); + let before = fs::read(data.join("node-policy.json")).unwrap(); + assert!(invoke(None).status.success()); + assert!(!invoke(Some("1")).status.success()); + assert_eq!(fs::read(data.join("node-policy.json")).unwrap(), before); + assert_eq!(fs::read_dir(data).unwrap().count(), 1); +} + +#[test] +fn container_preflight_returns_retained_production_mode_without_a_local_fallback() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-cli-preflight-production"); + let profile = profile(root.path()); + let config = root.path().join("node.toml"); + fs::write(&config, "mode = 'production'\ninitial_voters = 3\n").unwrap(); + let output = Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")) + .args(["accept-policy", "--profile"]) + .arg(&profile) + .arg("--deployment-config") + .arg(&config) + .args(["--physical-host-id", "real-host"]) + .env_remove("CROWDB_DEPLOYMENT_MODE") + .env_remove("CROWDB_CLUSTER") + .env_remove("CROWDB_INITIAL_VOTERS") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.stdout, b"production\n"); + let output = Command::new(env!("CARGO_BIN_EXE_crowdb-monitor")) + .args(["accept-policy", "--profile"]) + .arg(&profile) + .args(["--physical-host-id", "real-host"]) + .env_remove("CROWDB_DEPLOYMENT_MODE") + .env_remove("CROWDB_CLUSTER") + .env_remove("CROWDB_INITIAL_VOTERS") + .env_remove("CROWDB_NODE_POLICY_FILE") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(output.stdout, b"production\n"); + assert_eq!(fs::read_dir(root.path().join("install/data")).unwrap().count(), 1); +} diff --git a/container/crowdb-monitor/tests/node_service_dependencies_test.rs b/container/crowdb-monitor/tests/node_service_dependencies_test.rs new file mode 100644 index 000000000..52fd81e65 --- /dev/null +++ b/container/crowdb-monitor/tests/node_service_dependencies_test.rs @@ -0,0 +1,174 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "../src/node/durable.rs"] +mod durable; + +use crowdb_monitor::{DeploymentProfile, NodeServiceForTests}; +use crowdb_protocol::mgmt::{ + node::{NodeBinding, NodeServiceAction, NodeServiceIntent}, + SystemBootstrapIdentity, +}; +use std::{collections::BTreeMap, path::Path}; + +#[tokio::test] +async fn a_local_chunk_kv_listener_cannot_admit_access_before_catalog_authority_is_available() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("access-dependency-gate"); + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.path().to_path_buf(); + profile + .services + .iter_mut() + .find(|service| service.id == "access") + .unwrap() + .dependencies = vec!["chunk-kv".into()]; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let router = axum::Router::new().route("/ready", axum::routing::get(|| async { "ok" })); + let server = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + let binding = NodeBinding { + node_id: 1, + bootstrap: SystemBootstrapIdentity { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + configuration_digest: "digest".into(), + }, + management_seeds: vec!["http://127.0.0.1:1".into()], + }; + durable::write(&root.path().join("node-binding.json"), &binding).unwrap(); + let binding: NodeBinding = durable::read(&root.path().join("node-binding.json")) + .unwrap() + .unwrap(); + let local = NodeServiceIntent { + cluster_id: binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: 1, + service_id: "chunk-kv-1".into(), + kind: "chunk-kv".into(), + action: NodeServiceAction::Start, + configuration: format!("http_listen_addr = \"{address}\"\n"), + environment: BTreeMap::new(), + }; + let access = NodeServiceIntent { + kind: "access".into(), + service_id: "access-1".into(), + configuration: String::new(), + ..local.clone() + }; + let readiness = NodeServiceForTests::dependencies_ready_for_tests(&profile, &access, &[local]).await; + server.abort(); + assert!( + readiness.is_err(), + "a local idle listener does not prove routed catalog reads can succeed" + ); + assert!(!root.path().join("services/access-1/execution.json").exists()); +} + +#[path = "common/node_setup.rs"] +mod node_setup; +#[path = "common/service_dependency.rs"] +mod service_dependency; + +#[tokio::test] +async fn healthy_diskdb_listener_must_publish_the_registered_disks_ownership_before_consumers_start() { + use crowdb_kv_client::ServiceRegistryClient; + use crowdb_protocol::mgmt::node::setup::{NodeSetupChoices, NodeSetupRequest}; + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("diskdb-route-gate"); + let request = NodeSetupRequest { + version: 1, + discovery_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root.path(), &request).await; + assert_eq!(authority.registry.nodes[0].node_id, 2); + authority.kv.refresh_topology().await.unwrap(); + let kv = std::sync::Arc::new(crowdb_kv_client::CrowdbKvClient::new( + crowdb_kv_client::ClientConfig::new(authority.binding.management_seeds.clone()), + )); + service_dependency::publish_online_disk(kv.clone()).await; + let (address, server) = service_dependency::ready_listener().await; + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); + profile.paths.data_root = root.path().to_path_buf(); + profile.paths.log_root = root.path().join("log"); + profile + .services + .iter_mut() + .find(|service| service.id == "chunkdb") + .unwrap() + .dependencies = vec!["diskdb".into()]; + let local = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: 2, + service_id: "diskdb-2".into(), + kind: "diskdb".into(), + action: NodeServiceAction::Start, + configuration: format!("[server]\nhttp_listen_addr = \"{address}\"\n"), + environment: BTreeMap::new(), + }; + let consumer = NodeServiceIntent { + kind: "chunkdb".into(), + service_id: "chunkdb-2".into(), + configuration: String::new(), + ..local.clone() + }; + let registry = ServiceRegistryClient::from_shared(kv.clone()); + registry + .register_diskdb(2, "127.0.0.1:7102", &[], &[]) + .await + .unwrap(); + assert!( + NodeServiceForTests::dependencies_ready_for_tests(&profile, &consumer, std::slice::from_ref(&local)) + .await + .is_err(), + "a ready listener with no published owner cannot route an allocation" + ); + registry + .heartbeat_diskdb(2, "127.0.0.1:7102", &[99], &[]) + .await + .unwrap(); + assert!(NodeServiceForTests::dependencies_ready_for_tests( + &profile, + &consumer, + std::slice::from_ref(&local) + ) + .await + .is_err()); + registry + .heartbeat_diskdb(2, "127.0.0.1:7102", &[7], &[]) + .await + .unwrap(); + assert!( + NodeServiceForTests::dependencies_ready_for_tests(&profile, &consumer, std::slice::from_ref(&local)) + .await + .is_err(), + "an owned group without its registered disk loaded cannot allocate" + ); + service_dependency::assert_initial_start_waits(&kv, &profile, &local, &consumer).await; + registry + .heartbeat_diskdb( + 2, + "127.0.0.1:7102", + &[7], + &[crowdb_protocol::common::DiskGroupUsageSummary { + disk_group_id: 7, + usage_known: true, + disk_count: 1, + ..Default::default() + }], + ) + .await + .unwrap(); + NodeServiceForTests::dependencies_ready_for_tests(&profile, &consumer, &[local]) + .await + .unwrap(); + server.abort(); + assert!(!root.path().join("services/chunkdb-2/execution.json").exists()); +} diff --git a/container/crowdb-monitor/tests/node_service_policy_test.rs b/container/crowdb-monitor/tests/node_service_policy_test.rs new file mode 100644 index 000000000..035e9004f --- /dev/null +++ b/container/crowdb-monitor/tests/node_service_policy_test.rs @@ -0,0 +1,47 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::{DeploymentProfile, NodeServiceForTests}; +use crowdb_protocol::mgmt::node::{ + deployment::{DeploymentMode, NodeDeploymentOverrides}, + NodeServiceAction, NodeServiceIntent, +}; + +#[tokio::test] +async fn production_service_intents_cannot_activate_local_placement_or_single_copy_streams() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("production-service-policy"); + let inputs = NodeDeploymentOverrides { + mode: Some(DeploymentMode::Production), + ..NodeDeploymentOverrides::default() + }; + crowdb_monitor::accept_node_policy(root.path(), &inputs, &NodeDeploymentOverrides::default()).unwrap(); + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.path().to_path_buf(); + profile.paths.log_root = root.path().join("log"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + let mut services = NodeServiceForTests::new_for_tests(profile).await.unwrap(); + for (kind, configuration) in [ + ("chunkdb", "[deployment]\nmode='local'"), + ("access", "[deployment]\nmode='local'"), + ("chunk-kv", "[storage]\nstream_mirror_copies=1"), + ] { + let error = services + .execute_for_tests(NodeServiceIntent { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: 2, + service_id: format!("{kind}-2"), + kind: kind.into(), + action: NodeServiceAction::Start, + configuration: configuration.into(), + environment: std::collections::BTreeMap::new(), + }) + .await + .unwrap_err(); + assert!(error.to_string().contains("production node policy"), "{error}"); + assert!(!root.path().join("services").exists()); + } +} diff --git a/container/crowdb-monitor/tests/node_service_readiness_test.rs b/container/crowdb-monitor/tests/node_service_readiness_test.rs new file mode 100644 index 000000000..2a84e6cdc --- /dev/null +++ b/container/crowdb-monitor/tests/node_service_readiness_test.rs @@ -0,0 +1,222 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/node_setup.rs"] +mod node_setup; + +use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeServiceForTests}; +use crowdb_protocol::mgmt::node::{setup::NodeSetupRequest, NodeServiceAction, NodeServiceIntent}; +use std::os::unix::fs::PermissionsExt; + +#[tokio::test] +async fn readiness_timeout_preserves_the_live_process_and_start_budget() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("service-ready"); + let root = directory.path(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: crowdb_protocol::mgmt::node::setup::NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root, &request).await; + assert_eq!(authority.registry.nodes[0].node_id, 2); + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("log"); + profile.paths.run_root = root.join("run"); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + let program = root.join("starting-service"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + let child = profile + .services + .iter_mut() + .find(|service| service.id == "chunkdb") + .unwrap(); + child.program = program; + // This probe test owns a dummy child; the dependency gate is covered below. + child.dependencies = vec!["kv".into()]; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: 2, + service_id: "chunkdb-2".into(), + kind: "chunkdb".into(), + action: NodeServiceAction::Start, + configuration: format!( + "[server]\nhttp_listen_addr = '{}'\n", + listener.local_addr().unwrap() + ), + environment: std::collections::BTreeMap::default(), + }; + let key = b"/deployment/services/2/chunkdb-2"; + authority + .kv + .put(0, 0, key, &serde_json::to_vec(&intent).unwrap(), None) + .await + .unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile).await.unwrap(); + let error = worker.execute_for_tests(intent.clone()).await.unwrap_err(); + assert!(error.to_string().contains("readiness failed"), "{error}"); + let state = root.join("services/chunkdb-2/execution.json"); + let mut pending: serde_json::Value = serde_json::from_slice(&std::fs::read(&state).unwrap()).unwrap(); + assert_eq!(pending["attempts"], 1); + let pid = pending["pid"].as_u64().expect("live process remains observable"); + // The last permitted launch can still be starting. Its live process must + // remain observable without authorizing a sixth launch. + pending["attempts"] = serde_json::json!(5); + std::fs::write(&state, serde_json::to_vec(&pending).unwrap()).unwrap(); + assert!(worker.pending_for_tests(&intent).unwrap()); + assert_eq!( + worker.failures_for_tests().unwrap()["chunkdb-2"]["exhausted"], + false + ); + let pending_error = tokio::time::timeout( + std::time::Duration::from_secs(2), + worker.execute_for_tests(intent.clone()), + ) + .await + .expect("a continuing readiness observation must not occupy another full startup window") + .unwrap_err(); + assert!( + pending_error.to_string().contains("readiness failed"), + "{pending_error}" + ); + let observed: serde_json::Value = serde_json::from_slice(&std::fs::read(&state).unwrap()).unwrap(); + assert_eq!(observed["pid"], pid); + assert_eq!(observed["attempts"], 5); + let server = tokio::spawn(async move { + axum::serve( + listener, + axum::Router::new().route("/ready", axum::routing::get(|| async { "ready" })), + ) + .await + .unwrap(); + }); + let ready = worker.execute_for_tests(intent.clone()).await.unwrap(); + assert_eq!( + ready["pid"], pid, + "observing readiness must not restart the process" + ); + assert!(!worker.pending_for_tests(&intent).unwrap()); + intent.action = NodeServiceAction::Stop; + intent.operation_id = uuid::Uuid::new_v4().to_string(); + authority + .kv + .put(0, 0, key, &serde_json::to_vec(&intent).unwrap(), None) + .await + .unwrap(); + worker.execute_for_tests(intent).await.unwrap(); + server.abort(); +} + +#[tokio::test] +async fn recovery_waits_for_live_dependencies_without_consuming_launches() { + // Use the real Group-0 authority and process manager for recovery. + + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("service-dependencies"); + let data = directory.path().join("data"); + std::fs::create_dir_all(&data).unwrap(); + let root = data.as_path(); + let request = NodeSetupRequest { + version: 1, + discovery_id: NodeIdentity::load_or_create(root).unwrap().uuid().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: crowdb_protocol::mgmt::node::setup::NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root, &request).await; + let mut profile = DeploymentProfile::load( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), + ) + .unwrap(); + profile.paths.data_root = root.to_path_buf(); + profile.paths.log_root = root.join("log"); + profile.paths.install_root = directory.path().to_path_buf(); + profile.paths.bin_root = directory.path().join("bin"); + profile.paths.template_root = directory.path().join("templates"); + profile.paths.run_root = directory.path().join("run"); + std::fs::create_dir_all(&profile.paths.bin_root).unwrap(); + std::fs::create_dir_all(&profile.paths.log_root).unwrap(); + std::fs::create_dir_all(&profile.paths.run_root).unwrap(); + let program = profile.paths.bin_root.join("starting-service"); + std::fs::write(&program, "#!/bin/sh\nexec sleep 60\n").unwrap(); + std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o700)).unwrap(); + for service in &mut profile.services { + service.program = program.clone(); + if service.id == "chunkdb" { + service.dependencies = vec!["diskdb".into()]; + } + } + let mut listeners = Vec::new(); + let mut intents = Vec::new(); + for kind in ["diskdb", "chunkdb"] { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let intent = NodeServiceIntent { + cluster_id: authority.binding.bootstrap.cluster_id.clone(), + operation_id: uuid::Uuid::new_v4().to_string(), + node_id: authority.binding.node_id, + service_id: format!("{kind}-2"), + kind: kind.into(), + action: NodeServiceAction::Start, + configuration: format!( + "[server]\nhttp_listen_addr = '{}'\n", + listener.local_addr().unwrap() + ), + environment: std::collections::BTreeMap::default(), + }; + let key = format!("/deployment/services/2/{}", intent.service_id); + authority + .kv + .put(0, 0, key.as_bytes(), &serde_json::to_vec(&intent).unwrap(), None) + .await + .unwrap(); + listeners.push(listener); + intents.push(intent); + } + profile.validate().unwrap(); + let mut worker = NodeServiceForTests::new_for_tests(profile).await.unwrap(); + worker.reconcile_for_tests().await.unwrap(); + assert!( + !root.join("services/chunkdb-2/execution.json").exists(), + "waiting for a dependency must not launch a process or consume its budget" + ); + let servers: Vec<_> = listeners + .into_iter() + .map(|listener| { + tokio::spawn(async move { + axum::serve( + listener, + axum::Router::new().route("/ready", axum::routing::get(|| async { "ready" })), + ) + .await + .unwrap(); + }) + }) + .collect(); + worker.reconcile_for_tests().await.unwrap(); + let execution: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("services/chunkdb-2/execution.json")).unwrap()) + .unwrap(); + assert_eq!(execution["complete"], true); + assert_eq!(execution["intent"]["operation_id"], intents[1].operation_id); + assert!(execution["pid"].is_number()); + for mut intent in intents { + intent.action = NodeServiceAction::Stop; + intent.operation_id = uuid::Uuid::new_v4().to_string(); + let key = format!("/deployment/services/2/{}", intent.service_id); + authority + .kv + .put(0, 0, key.as_bytes(), &serde_json::to_vec(&intent).unwrap(), None) + .await + .unwrap(); + } + worker.reconcile_for_tests().await.unwrap(); + for server in servers { + server.abort(); + } +} diff --git a/container/crowdb-monitor/tests/node_setup_file_failure_test.rs b/container/crowdb-monitor/tests/node_setup_file_failure_test.rs new file mode 100644 index 000000000..4b18186c2 --- /dev/null +++ b/container/crowdb-monitor/tests/node_setup_file_failure_test.rs @@ -0,0 +1,124 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::NodeSetupForTests; +use crowdb_protocol::mgmt::node::setup::{NodeSetupPhase, SetupStepPhase}; +use node_setup_file::fail_file_request; +use std::path::Path; + +#[path = "common/node_setup.rs"] +mod node_setup; +#[path = "common/node_setup_file.rs"] +mod node_setup_file; +#[path = "common/node_setup_groups.rs"] +mod node_setup_groups; + +#[tokio::test] +async fn failed_file_is_terminal_and_explicit_retry_preserves_its_operation_and_geometry() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("file-fail"); + let root = directory.path(); + let (request, _authority, profile) = node_setup_file::prepare(root, false).await; + let socket = profile.paths.run_root.join("node-control.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let worker = tokio::spawn(fail_file_request(listener, root.to_path_buf())); + NodeSetupForTests::advance_for_tests(profile.clone()) + .await + .unwrap(); + let original = worker.await.unwrap(); + NodeSetupForTests::advance_for_tests(profile.clone()) + .await + .unwrap(); + let failed = NodeSetupForTests::status_for_tests(root).unwrap().unwrap(); + assert_eq!(failed.phase, NodeSetupPhase::Failed); + assert_eq!(failed.disks[0].phase, SetupStepPhase::Failed); + assert!(failed.disks[0] + .detail + .as_ref() + .unwrap() + .contains("insufficient backing space")); + assert!(failed.error.unwrap().contains("explicit retry")); + assert!(!Path::new(&original.path).exists()); + let operation = original.request.operation_id.clone(); + let path = root.join(format!("disk-file-{operation}.json")); + let before = std::fs::read(&path).unwrap(); + NodeSetupForTests::advance_for_tests(profile.clone()) + .await + .unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), before); + let status = NodeSetupForTests::default() + .accept_for_tests(root, request) + .unwrap(); + assert_eq!(status.phase, NodeSetupPhase::Accepted); + std::fs::remove_file(&socket).unwrap(); + let listener = tokio::net::UnixListener::bind(socket).unwrap(); + let worker = tokio::spawn(fail_file_request(listener, root.to_path_buf())); + NodeSetupForTests::advance_for_tests(profile.clone()) + .await + .unwrap(); + let retried = worker.await.unwrap(); + assert_eq!(retried, original); + let journal: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("node-setup.json")).unwrap()).unwrap(); + assert!(journal["retry_disks"].as_array().unwrap().is_empty()); + NodeSetupForTests::advance_for_tests(profile).await.unwrap(); + assert_eq!( + NodeSetupForTests::status_for_tests(root).unwrap().unwrap().phase, + NodeSetupPhase::Failed + ); + assert!(!retried.request.sparse); + assert!(!Path::new(&retried.path).exists()); +} + +#[tokio::test] +async fn storage_consumers_wait_for_disk_registration_after_prerequisite_services_are_ready() { + use crowdb_protocol::mgmt::node::{setup::SetupServiceKind, NodeControl}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("setup-order"); + let root = directory.path(); + let (_request, _authority, profile) = node_setup_file::prepare(root, true).await; + let path = root.join("node-setup.json"); + let mut state: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + for step in state["status"]["services"].as_array_mut().unwrap() { + if ["paxos-kv", "diskdb", "diskio"].contains(&step["kind"].as_str().unwrap()) { + step["phase"] = serde_json::json!("ready"); + } + } + std::fs::write(path, serde_json::to_vec(&state).unwrap()).unwrap(); + let listener = tokio::net::UnixListener::bind(profile.paths.run_root.join("node-control.sock")).unwrap(); + let data = root.to_path_buf(); + let worker = tokio::spawn(async move { + for _ in 0..2 { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + stream.read_to_end(&mut bytes).await.unwrap(); + let response = match serde_json::from_slice::(&bytes).unwrap() { + NodeControl::PrepareDiskFile { request } => { + crowdb_monitor::accept_file_for_tests(&data, &request).unwrap(); + serde_json::json!({}) + } + NodeControl::Processes { .. } => serde_json::json!({"processes":{},"service_failures":{}}), + _ => panic!("unexpected setup control"), + }; + stream + .write_all(&serde_json::to_vec(&serde_json::json!({"ok":true,"value":response})).unwrap()) + .await + .unwrap(); + } + }); + NodeSetupForTests::advance_for_tests(profile).await.unwrap(); + worker.await.unwrap(); + let status = NodeSetupForTests::status_for_tests(root).unwrap().unwrap(); + assert_eq!(status.phase, NodeSetupPhase::Running); + assert_ne!(status.disks[0].phase, SetupStepPhase::Ready); + for kind in [ + SetupServiceKind::Chunkdb, + SetupServiceKind::ChunkKv, + SetupServiceKind::AccessServer, + ] { + let step = status.services.iter().find(|step| step.kind == kind).unwrap(); + assert!( + step.operation_id.is_none(), + "storage is still unregistered: {step:?}" + ); + } +} diff --git a/container/crowdb-monitor/tests/node_setup_supplied_test.rs b/container/crowdb-monitor/tests/node_setup_supplied_test.rs index 785af196b..dec0d8507 100644 --- a/container/crowdb-monitor/tests/node_setup_supplied_test.rs +++ b/container/crowdb-monitor/tests/node_setup_supplied_test.rs @@ -4,9 +4,8 @@ use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; use crowdb_monitor::{DeploymentProfile, NodeIdentity, NodeSetupForTests}; use crowdb_protocol::{ - common::{BindMapValue, GroupValue, NodeValue, RackValue}, + common::{BindMapValue, NodeValue, RackValue}, diskdb::rpc::DiskGroupValue, - key::{KvGroupKey, TextKey}, mgmt::node::{ setup::{ NodeSetupChoices, NodeSetupPhase, NodeSetupRequest, SetupMedium, SetupStepPhase, SetupStorage, @@ -20,6 +19,8 @@ use crowdb_protocol::{ use std::{fs, os::unix::fs::PermissionsExt, path::Path}; #[path = "common/node_setup.rs"] mod node_setup; +#[path = "common/node_setup_groups.rs"] +mod node_setup_groups; struct TestSetup { root: crowdb_test_harness::test_dirs::TestDir, @@ -84,7 +85,7 @@ impl TestSetup { ) .unwrap(); let mut profile = DeploymentProfile::load( - Path::new(env!("CARGO_MANIFEST_DIR")).join("../single-node-container/profile.toml"), + Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml"), ) .unwrap(); profile.paths.data_root = root.path().to_path_buf(); @@ -136,26 +137,7 @@ impl TestSetup { .await .unwrap(); hardware.set_bind(1, 2, 7, 0, 1).await.unwrap(); - authority - .kv - .put( - 0, - 0, - KvGroupKey { - store_id: 0, - group_id: 1, - } - .to_path() - .as_bytes(), - &serde_json::to_vec(&GroupValue { - store_id: 0, - group_id: 1, - }) - .unwrap(), - None, - ) - .await - .unwrap(); + node_setup_groups::publish_group(&authority, 1).await; authority } @@ -330,7 +312,7 @@ async fn failed_medium_keeps_successful_siblings_and_only_explicit_retry_dispatc assert_eq!(observer.await.unwrap(), vec![requests[1].clone()]); let state: serde_json::Value = serde_json::from_slice(&fs::read(test.root.path().join("node-setup.json")).unwrap()).unwrap(); - assert!(state["retry_supplied"].as_array().unwrap().is_empty()); + assert!(state["retry_disks"].as_array().unwrap().is_empty()); } #[tokio::test] diff --git a/container/crowdb-monitor/tests/node_setup_test.rs b/container/crowdb-monitor/tests/node_setup_test.rs index fa37e4fc5..7b856a88b 100644 --- a/container/crowdb-monitor/tests/node_setup_test.rs +++ b/container/crowdb-monitor/tests/node_setup_test.rs @@ -6,6 +6,10 @@ use crowdb_protocol::mgmt::node::setup::{NodeSetupPhase, NodeSetupRequest, Setup use std::path::Path; #[path = "common/node_setup.rs"] mod node_setup; +#[path = "common/node_setup_control.rs"] +mod node_setup_control; +#[path = "common/node_setup_groups.rs"] +mod node_setup_groups; fn request(root: &Path) -> NodeSetupRequest { NodeSetupRequest { @@ -26,6 +30,34 @@ fn authorize(root: &Path, request: &NodeSetupRequest) { .unwrap(); } +#[test] +fn accepted_deployment_choices_cannot_be_overridden_by_admission_or_a_changed_setup_journal() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-setup-sealed-policy"); + let root = directory.path(); + let inputs = crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides::default(); + let policy = crowdb_monitor::accept_node_policy(root, &inputs, &inputs).unwrap(); + let mut intent = request(root); + intent.choices = policy.setup; + authorize(root, &intent); + let mut conflicting = intent.clone(); + conflicting.choices.storage = SetupStorage::Diskless; + let mut setup = NodeSetupForTests::default(); + let before = std::fs::read(root.join("node-policy.json")).unwrap(); + let error = setup.accept_for_tests(root, conflicting.clone()).unwrap_err(); + assert!(error.to_string().contains("accepted deployment policy")); + assert!(!root.join("node-setup.json").exists()); + assert!(!root.join("disks").exists()); + setup.accept_for_tests(root, intent).unwrap(); + let path = root.join("node-setup.json"); + let mut journal: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + journal["request"]["choices"] = serde_json::to_value(conflicting.choices).unwrap(); + let corrupt = serde_json::to_vec(&journal).unwrap(); + std::fs::write(&path, &corrupt).unwrap(); + assert!(NodeSetupForTests::status_for_tests(root).is_err()); + assert_eq!(std::fs::read(path).unwrap(), corrupt); + assert_eq!(std::fs::read(root.join("node-policy.json")).unwrap(), before); +} + #[test] fn setup_requires_target_identity_and_authenticated_admission_then_replays_read_only() { let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-setup-accept"); @@ -143,22 +175,51 @@ async fn unbound_setup_survives_a_worker_restart_without_creating_disks_or_servi async fn diskless_setup_requires_the_exact_confirmed_admission_and_does_not_start_services() { let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("node-setup-diskless"); let root = directory.path(); - let mut intent = request(root); - intent.choices.storage = SetupStorage::Diskless; - for service in &mut intent.choices.services { + let mut choices = crowdb_protocol::mgmt::node::deployment::NodeDeploymentPolicy::default().setup; + choices.storage = SetupStorage::Diskless; + for service in &mut choices.services { service.enabled = false; } + let inputs = crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides { + storage: Some(choices.storage.clone()), + services: Some(choices.services.clone()), + ..Default::default() + }; + crowdb_monitor::accept_node_policy( + root, + &inputs, + &crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides::default(), + ) + .unwrap(); + let mut intent = request(root); + intent.choices = choices; authorize(root, &intent); NodeSetupForTests::default() .accept_for_tests(root, intent.clone()) .unwrap(); let mut authority = node_setup::TestAuthority::start(root, &intent).await; + let preparation_key = crowdb_console_shared::deployment::preparation::DEFAULT_PREPARATION_KEY; + authority.kv.delete(0, 0, preparation_key, None).await.unwrap(); let profile = profile(root); NodeSetupForTests::advance_for_tests(profile.clone()) .await .unwrap(); let status = NodeSetupForTests::status_for_tests(root).unwrap().unwrap(); assert_eq!(status.phase, NodeSetupPhase::Ready); + assert!(matches!( + authority + .kv + .get( + 0, + 0, + preparation_key, + crowdb_kv_client::ReadMode::Linearizable, + None + ) + .await + .unwrap(), + crowdb_kv_client::GetOutcome::NotFound, + )); assert_eq!(status.node_id, Some(2)); assert_eq!( status.cluster_id.as_ref(), @@ -226,10 +287,7 @@ async fn explicit_cleanup_retires_only_the_matching_setup_and_never_replays_it() #[tokio::test] async fn default_file_waits_for_metadata_then_reuses_one_durable_request_and_group() { use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; - use crowdb_protocol::{ - common::{GroupValue, NodeValue, RackValue}, - key::{KvGroupKey, TextKey}, - }; + use crowdb_protocol::common::{NodeValue, RackValue}; // Keep the socket pathname short enough for CI checkout prefixes. let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("setup"); let root = directory.path(); @@ -256,6 +314,7 @@ async fn default_file_waits_for_metadata_then_reuses_one_durable_request_and_gro .await .unwrap(); hardware.add_node(1, 2, &NodeValue::default()).await.unwrap(); + node_setup_groups::publish_group(&authority, 8).await; let mut profile = profile(root); profile.paths.run_root = root.join("run"); NodeSetupForTests::advance_for_tests(profile.clone()) @@ -266,26 +325,7 @@ async fn default_file_waits_for_metadata_then_reuses_one_durable_request_and_gro assert!(waiting.error.unwrap().contains("ordinary metadata group")); assert!(hardware.list_disk_groups_on_node(1, 2).await.unwrap().is_empty()); assert!(!root.join("disks").exists()); - authority - .kv - .put( - 0, - 0, - KvGroupKey { - store_id: 0, - group_id: 1, - } - .to_path() - .as_bytes(), - &serde_json::to_vec(&GroupValue { - store_id: 0, - group_id: 1, - }) - .unwrap(), - None, - ) - .await - .unwrap(); + node_setup_groups::publish_group(&authority, 1).await; std::fs::create_dir(&profile.paths.run_root).unwrap(); let listener = tokio::net::UnixListener::bind(profile.paths.run_root.join("node-control.sock")).unwrap(); let observe = tokio::spawn(observe_file_requests(listener)); @@ -314,15 +354,153 @@ async fn default_file_waits_for_metadata_then_reuses_one_durable_request_and_gro assert_eq!((binding.store_id, binding.group_id), (0, 1)); } +#[tokio::test] +async fn a_conflicting_reserved_disk_group_binding_cannot_redirect_default_storage() { + use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient}; + use crowdb_protocol::{ + common::{NodeValue, RackValue}, + diskdb::rpc::DiskGroupValue, + }; + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("setup-binding"); + let root = directory.path(); + let mut intent = request(root); + for service in &mut intent.choices.services { + service.enabled = false; + } + authorize(root, &intent); + NodeSetupForTests::default() + .accept_for_tests(root, intent.clone()) + .unwrap(); + let authority = node_setup::TestAuthority::start(root, &intent).await; + node_setup_groups::publish_group(&authority, 1).await; + node_setup_groups::publish_group(&authority, 8).await; + let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new( + authority.binding.management_seeds.clone(), + ))); + hardware + .add_rack( + 1, + &RackValue { + node_ids: vec![2], + ..RackValue::default() + }, + ) + .await + .unwrap(); + hardware + .add_node( + 1, + 2, + &NodeValue { + disk_group_ids: vec![1], + ..NodeValue::default() + }, + ) + .await + .unwrap(); + hardware + .add_disk_group(1, 2, 1, &DiskGroupValue::default()) + .await + .unwrap(); + hardware.set_bind(1, 2, 1, 0, 8).await.unwrap(); + NodeSetupForTests::advance_for_tests(profile(root)).await.unwrap(); + let observed = NodeSetupForTests::status_for_tests(root).unwrap().unwrap(); + assert!(observed.error.unwrap().contains("conflicting metadata binding")); + assert!(!root.join("disks").exists()); + let journal: serde_json::Value = + serde_json::from_slice(&std::fs::read(root.join("node-setup.json")).unwrap()).unwrap(); + assert!(journal["file"].is_null()); + assert_eq!(hardware.get_bind(1, 2, 1).await.unwrap().unwrap().group_id, 8); + assert_eq!(hardware.list_disk_groups_on_node(1, 2).await.unwrap().len(), 1); +} + +#[tokio::test] +async fn normal_setup_defers_disks_and_service_commands_until_owned_metadata_is_available() { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("setup-metadata-gate"); + let root = directory.path(); + let inputs = crowdb_protocol::mgmt::node::deployment::NodeDeploymentOverrides::default(); + let policy = crowdb_monitor::accept_node_policy(root, &inputs, &inputs).unwrap(); + let mut intent = request(root); + intent.choices = policy.setup; + authorize(root, &intent); + NodeSetupForTests::default() + .accept_for_tests(root, intent.clone()) + .unwrap(); + let _authority = node_setup::TestAuthority::start(root, &intent).await; + NodeSetupForTests::advance_for_tests(profile(root)).await.unwrap(); + let observed = NodeSetupForTests::status_for_tests(root).unwrap().unwrap(); + assert_eq!(observed.phase, NodeSetupPhase::Running); + assert!(observed.error.unwrap().contains("live kv-server")); + assert!(observed + .services + .iter() + .all(|step| step.phase == crowdb_protocol::mgmt::node::setup::SetupStepPhase::Pending)); + assert!(observed.disks.is_empty()); + assert!(!root.join("disks").exists()); +} + fn profile(root: &Path) -> DeploymentProfile { - let mut profile = DeploymentProfile::load( - Path::new(env!("CARGO_MANIFEST_DIR")).join("../single-node-container/profile.toml"), - ) - .unwrap(); + let mut profile = + DeploymentProfile::load(Path::new(env!("CARGO_MANIFEST_DIR")).join("../oci-image/node-profile.toml")) + .unwrap(); profile.paths.data_root = root.to_path_buf(); profile } +#[tokio::test] +async fn recreated_setup_never_replaces_later_stop_delete_restart_or_reconfiguration() { + use crowdb_protocol::mgmt::node::setup::{SetupServiceKind, SetupStepPhase}; + use crowdb_protocol::mgmt::node::NodeServiceAction; + for action in [ + NodeServiceAction::Stop, + NodeServiceAction::Delete, + NodeServiceAction::Restart, + NodeServiceAction::Start, + ] { + let directory = crowdb_test_harness::test_dirs::tempdir_in_test_data("intent"); + let root = directory.path(); + let mut request = request(root); + request.choices.storage = SetupStorage::Diskless; + for service in &mut request.choices.services { + service.enabled = service.kind == SetupServiceKind::Chunkdb; + } + authorize(root, &request); + NodeSetupForTests::default() + .accept_for_tests(root, request.clone()) + .unwrap(); + let authority = node_setup::TestAuthority::start(root, &request).await; + let (original, newer) = node_setup_control::newer_intent(root, &authority, action).await; + let mut profile = profile(root); + profile.paths.run_root = root.join("run"); + let observe = node_setup_control::processes_once(&profile.paths.run_root); + NodeSetupForTests::advance_for_tests(profile.clone()) + .await + .unwrap(); + observe.await.unwrap(); + let status = NodeSetupForTests::status_for_tests(root).unwrap().unwrap(); + let step = status + .services + .iter() + .find(|step| step.kind == SetupServiceKind::Chunkdb) + .unwrap(); + assert_eq!(step.phase, SetupStepPhase::Warning); + assert_eq!(step.operation_id.as_deref(), Some(original.as_str())); + assert!(step.detail.as_ref().unwrap().contains("operator command")); + // Historical plan completion is not a health assertion. Recreation and + // exact acceptance cannot turn this newer command into the old Start. + NodeSetupForTests::default() + .accept_for_tests(root, request) + .unwrap(); + NodeSetupForTests::advance_for_tests(profile).await.unwrap(); + assert_eq!( + crowdb_console_shared::deployment::services::current(&authority.kv, 2, "chunkdb-2") + .await + .unwrap(), + Some(newer) + ); + } +} + async fn observe_file_requests( listener: tokio::net::UnixListener, ) -> Vec { @@ -364,12 +542,14 @@ async fn service_failure_budget_survives_restart_and_yields_to_a_new_operator_co profile.paths.log_root = root.join("log"); std::fs::create_dir_all(&profile.paths.log_root).unwrap(); profile.paths.run_root = root.join("run"); - profile + let child = profile .services .iter_mut() .find(|service| service.id == "chunkdb") - .unwrap() - .program = root.join("absent-binary"); + .unwrap(); + child.program = root.join("absent-binary"); + // Isolate launch-budget accounting from the separately tested dependency gate. + child.dependencies = vec!["kv".into()]; let mut intent = NodeServiceIntent { cluster_id: authority.binding.bootstrap.cluster_id.clone(), operation_id: uuid::Uuid::new_v4().to_string(), diff --git a/container/crowdb-monitor/tests/preview_run_test.rs b/container/crowdb-monitor/tests/preview_run_test.rs index 824f1b6a9..a34043157 100644 --- a/container/crowdb-monitor/tests/preview_run_test.rs +++ b/container/crowdb-monitor/tests/preview_run_test.rs @@ -2,7 +2,7 @@ use std::fs; use std::os::unix::fs::symlink; use std::path::{Path, PathBuf}; -use crowdb_monitor::{run_preview, DeploymentProfile, MonitorPhase, StatusStore}; +use crowdb_monitor::{run_preview_for_tests, DeploymentProfile, MonitorPhase, StatusStore}; use uuid::Uuid; struct TestRoot(PathBuf); @@ -77,7 +77,7 @@ impl Drop for TestRoot { async fn failed_child_never_marks_preview_ready_and_template_drift_fails_closed() { let root = TestRoot::new(); let profile = root.profile_path(); - assert!(run_preview(&profile).await.is_err()); + assert!(run_preview_for_tests(&profile).await.is_err()); let status = StatusStore::open(&root.0.join("run")) .unwrap() .read(std::time::Duration::from_secs(10)) @@ -93,7 +93,7 @@ async fn failed_child_never_marks_preview_ready_and_template_drift_fails_closed( let template = root.0.join("templates/kv.toml"); fs::write(&template, format!("{}\n", fs::read_to_string(&template).unwrap())).unwrap(); let before = fs::read(root.0.join("data/bootstrap/manifest.json")).unwrap(); - assert!(run_preview(&profile).await.is_err()); + assert!(run_preview_for_tests(&profile).await.is_err()); assert_eq!( fs::read(root.0.join("data/bootstrap/manifest.json")).unwrap(), before @@ -105,7 +105,7 @@ async fn nonempty_uninitialized_root_is_not_adopted() { let root = TestRoot::new(); let profile = root.profile_path(); fs::write(root.0.join("data/foreign"), b"unrelated").unwrap(); - assert!(run_preview(&profile).await.is_err()); + assert!(run_preview_for_tests(&profile).await.is_err()); assert!(!root.0.join("data/bootstrap").exists()); assert!(!root.0.join("data/secrets").exists()); } @@ -129,21 +129,7 @@ async fn invalid_release_selection_fails_before_writing_bootstrap_state() { } profile.validate().unwrap(); fs::write(&path, toml::to_string(&profile).unwrap()).unwrap(); - assert!(run_preview(&path).await.is_err()); - let result = crowdb_monitor::run_single_node(crowdb_monitor::NodeRuntimeConfig { - profile: path.clone(), - bind: "127.0.0.1:0".parse().unwrap(), - discovery: crowdb_monitor::DiscoveryConfig { - seeds: Vec::new(), - interfaces: vec!["lo".into()], - addresses: vec!["127.0.0.1".parse().unwrap()], - monitor_port: 9095, - cluster_id: None, - }, - physical_host_id: "selection-test".into(), - }) - .await; - assert!(result.is_err()); + assert!(run_preview_for_tests(&path).await.is_err()); assert!(!root.0.join("data/node-identity").exists()); assert!(!root.0.join("data/bootstrap").exists()); assert!(!root.0.join("data/secrets").exists()); @@ -158,7 +144,7 @@ async fn kv_subset_supervises_only_kv_and_does_not_render_or_create_storage() { let mut profile = DeploymentProfile::load(&path).unwrap(); profile.services.retain(|service| service.id == "kv"); fs::write(&path, toml::to_string(&profile).unwrap()).unwrap(); - assert!(run_preview(&path).await.is_err()); + assert!(run_preview_for_tests(&path).await.is_err()); let status = StatusStore::open(&root.0.join("run")) .unwrap() .read(std::time::Duration::from_secs(10)) diff --git a/container/crowdb-monitor/tests/preview_selection_test.rs b/container/crowdb-monitor/tests/preview_selection_test.rs index 3a91dda94..d4caa108e 100644 --- a/container/crowdb-monitor/tests/preview_selection_test.rs +++ b/container/crowdb-monitor/tests/preview_selection_test.rs @@ -3,7 +3,7 @@ use crowdb_monitor::{ disk_step_names, hardware_step_names, iceberg_step_names, kv_step_names, logical_step_names, - preview_step_names, s3_step_names, DeploymentProfile, + preview_step_names_for_tests, s3_step_names, DeploymentProfile, }; use std::path::Path; @@ -26,7 +26,7 @@ fn complete_profile_preserves_the_durable_bootstrap_order() { .chain(s3_step_names().map(str::to_owned)) .chain(iceberg_step_names().map(str::to_owned)) .collect(); - assert_eq!(preview_step_names(&profile).unwrap(), expected); + assert_eq!(preview_step_names_for_tests(&profile).unwrap(), expected); } #[test] @@ -48,7 +48,7 @@ fn kv_and_storage_subsets_exclude_unselected_bootstrap_contracts() { expected.extend(hardware_step_names()); } expected.extend(logical_step_names().map(str::to_owned)); - assert_eq!(preview_step_names(&profile).unwrap(), expected); + assert_eq!(preview_step_names_for_tests(&profile).unwrap(), expected); } } @@ -57,7 +57,7 @@ fn unknown_services_and_missing_runtime_dependency_edges_are_rejected() { let mut unknown = profile(); unknown.services.last_mut().unwrap().id = "unknown".into(); unknown.validate().unwrap(); - assert!(preview_step_names(&unknown).is_err()); + assert!(preview_step_names_for_tests(&unknown).is_err()); let mut missing = profile(); missing .services @@ -67,7 +67,7 @@ fn unknown_services_and_missing_runtime_dependency_edges_are_rejected() { .dependencies .clear(); missing.validate().unwrap(); - assert!(preview_step_names(&missing).is_err()); + assert!(preview_step_names_for_tests(&missing).is_err()); let mut absent = profile(); absent .services @@ -79,7 +79,7 @@ fn unknown_services_and_missing_runtime_dependency_edges_are_rejected() { .unwrap() .dependencies = vec!["kv".into()]; absent.validate().unwrap(); - assert!(preview_step_names(&absent).is_err()); + assert!(preview_step_names_for_tests(&absent).is_err()); } #[test] @@ -91,7 +91,7 @@ fn incompatible_management_seed_and_durable_root_fail_preflight() { .unwrap() .env .insert("CROWDB_MANAGEMENT_SEEDS".into(), "http://127.0.0.1:1".into()); - assert!(preview_step_names(&seed).is_err()); + assert!(preview_step_names_for_tests(&seed).is_err()); let mut root = profile(); let kv = root .services @@ -100,5 +100,5 @@ fn incompatible_management_seed_and_durable_root_fail_preflight() { .unwrap(); let index = kv.args.iter().position(|argument| argument == "--root").unwrap(); kv.args[index + 1] = "/tmp/foreign-kv".into(); - assert!(preview_step_names(&root).is_err()); + assert!(preview_step_names_for_tests(&root).is_err()); } diff --git a/container/crowdb-monitor/tests/recovery_owners_test.rs b/container/crowdb-monitor/tests/recovery_owners_test.rs new file mode 100644 index 000000000..872532656 --- /dev/null +++ b/container/crowdb-monitor/tests/recovery_owners_test.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/node_setup.rs"] +mod node_setup; + +use crowdb_console_shared::{ops::OpContext, ConsoleConfig}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use crowdb_protocol::{ + common::ReplicaValue, + kv_membership::GroupMember, + mgmt::node::{ + recovery::RecoveryGroups, + setup::{NodeSetupChoices, NodeSetupRequest}, + }, +}; +use std::sync::Arc; + +#[tokio::test] +async fn installing_membership_authorizes_catchup_routes_before_replica_promotion() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("joining-recovery-routes"); + let request = NodeSetupRequest { + version: 1, + discovery_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + choices: NodeSetupChoices::default(), + }; + let authority = node_setup::TestAuthority::start(root.path(), &request).await; + authority.kv.refresh_topology().await.unwrap(); + assert_eq!(authority.registry.nodes[0].node_id, 2); + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new( + authority.binding.management_seeds.clone(), + ))); + let ctx = OpContext::with_shared_client_preserving_hint( + kv, + &authority.binding.management_seeds, + ConsoleConfig::default(), + ); + let original = GroupMember { + node_id: 2, + replica_id: 1, + endpoint: "127.0.0.1:10100".into(), + voting: true, + }; + ctx.sysmd() + .add_replica(&ReplicaValue { + store_id: 0, + group_id: 0, + node_id: original.node_id, + replica_id: original.replica_id, + voting: true, + endpoint: original.endpoint.clone(), + ..Default::default() + }) + .await + .unwrap(); + let initial = ctx + .membership() + .create(0, 0, vec![original.clone()]) + .await + .unwrap(); + ctx.membership().complete(&initial).await.unwrap(); + let joining = GroupMember { + node_id: 4, + replica_id: 601, + endpoint: "127.0.0.1:10200".into(), + voting: true, + }; + let installing = ctx + .membership() + .begin_change(0, 0, 1, vec![original.clone(), joining]) + .await + .unwrap(); + assert_eq!(ctx.sysmd().list_all_replicas().await.unwrap().len(), 1); + let snapshot = captured(root.path(), &ctx, &authority.binding.bootstrap, 4).await; + assert_eq!( + snapshot.groups.len(), + 1, + "the joining replica needs recovery routes before catch-up can finish" + ); + assert_eq!( + snapshot.groups[0] + .members + .iter() + .map(|member| (member.node_id, member.replica_id)) + .collect::>(), + vec![(2, 1), (4, 601)] + ); + let snapshot = captured(root.path(), &ctx, &authority.binding.bootstrap, 2).await; + assert!( + snapshot.groups[0] + .members + .iter() + .any(|member| member.replica_id == 601), + "existing voters must be able to send catch-up records to the learner" + ); + + ctx.membership().complete(&installing).await.unwrap(); + let removing = ctx + .membership() + .begin_change(0, 0, 2, vec![original]) + .await + .unwrap(); + let snapshot = captured(root.path(), &ctx, &authority.binding.bootstrap, 4).await; + assert_eq!( + snapshot.groups.len(), + 1, + "retiring members remain known until installation completes" + ); + ctx.membership().complete(&removing).await.unwrap(); + let snapshot = captured(root.path(), &ctx, &authority.binding.bootstrap, 4).await; + assert!( + snapshot.groups.is_empty(), + "completed removal revokes recovery membership" + ); +} + +async fn captured( + root: &std::path::Path, + ctx: &OpContext, + bootstrap: &crowdb_protocol::mgmt::SystemBootstrapIdentity, + node: u64, +) -> RecoveryGroups { + crowdb_monitor::capture_recovery_owners_for_tests(root, ctx, bootstrap, node) + .await + .unwrap(); + serde_json::from_slice(&std::fs::read(root.join("recovery-groups.json")).unwrap()).unwrap() +} diff --git a/container/crowdb-monitor/tests/rpc_probe_socket_lifecycle_test.rs b/container/crowdb-monitor/tests/rpc_probe_socket_lifecycle_test.rs new file mode 100644 index 000000000..7916afb53 --- /dev/null +++ b/container/crowdb-monitor/tests/rpc_probe_socket_lifecycle_test.rs @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#![cfg(target_os = "linux")] + +use crowdb_monitor::{ProbeExecutor, ProbeKind, ProbeProfile, RestartProfile, ServiceProfile}; +use crowdb_rpc_ffi::RpcServer; +use std::{collections::BTreeMap, path::PathBuf, time::Duration}; + +fn descriptors() -> usize { + std::fs::read_dir("/proc/self/fd").unwrap().count() +} + +#[tokio::test] +async fn repeated_idle_rpc_probes_reclaim_descriptors_while_the_executor_stays_alive() { + let probes = ProbeExecutor::new(true).unwrap(); + let server = RpcServer::new(None); + server.listen("127.0.0.1", 0).unwrap(); + server.start(); + let service = ServiceProfile { + id: "socket-lifecycle".into(), + program: PathBuf::from("/bin/true"), + args: vec![], + env: BTreeMap::new(), + dependencies: vec![], + fence_listeners: vec![], + config_template: None, + probe: ProbeProfile { + kind: ProbeKind::RpcPing, + target: format!("127.0.0.1:{}", server.port()), + bearer_env: None, + timeout_ms: 1000, + failure_threshold: 1, + }, + additional_probes: vec![], + restart: RestartProfile { + max_attempts: 1, + backoff_base_ms: 1, + backoff_max_ms: 1, + stable_after_ms: 60_000, + }, + }; + let before = descriptors(); + for _ in 0..128 { + probes.probe_service(&service, &BTreeMap::new()).await.unwrap(); + } + let result = tokio::time::timeout(Duration::from_secs(3), async { + while descriptors() > before + 8 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; + assert!( + result.is_ok(), + "128 completed probes leaked idle connections: before={before}, after={}", + descriptors() + ); + probes.probe_service(&service, &BTreeMap::new()).await.unwrap(); + server.stop(); +} diff --git a/container/crowdb-monitor/tests/runtime_observation_test.rs b/container/crowdb-monitor/tests/runtime_observation_test.rs new file mode 100644 index 000000000..d4c332420 --- /dev/null +++ b/container/crowdb-monitor/tests/runtime_observation_test.rs @@ -0,0 +1,113 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_monitor::runtime_observation_for_tests; +use crowdb_protocol::mgmt::{cluster::DependencyState, node::NodeServiceAction}; +use std::time::Duration; + +#[path = "common/runtime_observation.rs"] +mod runtime_observation; +use runtime_observation::TestObservation; + +#[tokio::test] +async fn health_requires_the_executed_operation_and_current_owned_process() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("runtime-health"); + let mut fixture = TestObservation::start(root.path()).await; + let report = runtime_observation_for_tests(&fixture.profile, &mut fixture.manager).unwrap(); + assert_eq!(report.services[0].state, DependencyState::Ready); + fixture.save(false, 1, None); + assert_eq!( + runtime_observation_for_tests(&fixture.profile, &mut fixture.manager) + .unwrap() + .services[0] + .state, + DependencyState::Initializing + ); + fixture.intent.operation_id = uuid::Uuid::new_v4().to_string(); + fixture.save(true, 0, None); + let report = runtime_observation_for_tests(&fixture.profile, &mut fixture.manager).unwrap(); + assert_eq!(report.services[0].state, DependencyState::Initializing); + assert_eq!( + report.services[0].operation_id.as_ref(), + Some(&fixture.intent.operation_id) + ); + fixture.status.services.get_mut("chunkdb-1").unwrap().operation_id = + Some(fixture.intent.operation_id.clone()); + fixture.store.publish(&mut fixture.status).unwrap(); + assert_eq!( + runtime_observation_for_tests(&fixture.profile, &mut fixture.manager) + .unwrap() + .services[0] + .state, + DependencyState::Ready + ); + fixture + .manager + .stop("chunkdb-1", Duration::from_secs(1)) + .await + .unwrap(); + assert_eq!( + runtime_observation_for_tests(&fixture.profile, &mut fixture.manager) + .unwrap() + .services[0] + .state, + DependencyState::Initializing + ); + fixture.intent.action = NodeServiceAction::Stop; + fixture.save(true, 0, None); + assert_eq!( + runtime_observation_for_tests(&fixture.profile, &mut fixture.manager) + .unwrap() + .services[0] + .state, + DependencyState::Stopped + ); +} + +#[tokio::test] +async fn stale_health_and_terminal_failures_are_distinct_and_do_not_export_error_contents() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("runtime-failure"); + let mut fixture = TestObservation::start(root.path()).await; + let path = fixture.profile.paths.run_root.join("status/monitor.json"); + let mut stale = serde_json::to_value(&fixture.status).unwrap(); + stale["updated_at_ms"] = serde_json::json!(0); + std::fs::write(path, serde_json::to_vec(&stale).unwrap()).unwrap(); + assert_eq!( + runtime_observation_for_tests(&fixture.profile, &mut fixture.manager) + .unwrap() + .services[0] + .state, + DependencyState::Initializing + ); + fixture.save(false, 5, Some("private-error-sentinel")); + let report = runtime_observation_for_tests(&fixture.profile, &mut fixture.manager).unwrap(); + assert_eq!(report.services[0].state, DependencyState::Initializing); + assert!(!serde_json::to_string(&report) + .unwrap() + .contains("private-error-sentinel")); + fixture + .manager + .stop("chunkdb-1", Duration::from_secs(1)) + .await + .unwrap(); + let report = runtime_observation_for_tests(&fixture.profile, &mut fixture.manager).unwrap(); + assert_eq!(report.services[0].state, DependencyState::Failed); + assert!(!serde_json::to_string(&report) + .unwrap() + .contains("private-error-sentinel")); +} + +#[tokio::test] +async fn observation_never_creates_a_missing_identity_or_status_directory() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("observation-readonly"); + let profile = runtime_observation::profile(root.path()); + std::fs::create_dir(&profile.paths.log_root).unwrap(); + let mut manager = + crowdb_monitor::ProcessManager::new(profile.paths.log_root.clone(), profile.logs.clone()) + .await + .unwrap(); + assert!(runtime_observation_for_tests(&profile, &mut manager).is_err()); + assert!(!root.path().join("node-identity").exists()); + assert!(!profile.paths.run_root.exists()); + assert!(!root.path().join("node-policy.json").exists()); +} diff --git a/container/crowdb-monitor/tests/status_test.rs b/container/crowdb-monitor/tests/status_test.rs index f7cb42a0c..f7d0afd23 100644 --- a/container/crowdb-monitor/tests/status_test.rs +++ b/container/crowdb-monitor/tests/status_test.rs @@ -53,6 +53,7 @@ fn readiness_requires_fresh_ready_snapshot() { status.services.insert( "kv".into(), ServiceStatus { + operation_id: None, pid: Some(123), generation: 1, healthy: true, diff --git a/container/oci-image/Dockerfile b/container/oci-image/Dockerfile index 7df423b44..cf1bd9e4d 100644 --- a/container/oci-image/Dockerfile +++ b/container/oci-image/Dockerfile @@ -16,7 +16,6 @@ RUN --mount=type=bind,source=.,target=/staged,ro \ && test "$(cat /staged/VERSION)" = "$PREVIEW_VERSION" \ && mkdir -p /opt/crowdb/etc /opt/crowdb/data /opt/crowdb/run \ && cp -a /staged/bin /staged/lib /staged/ui /opt/crowdb/ \ - && cp -a /staged/templates /opt/crowdb/etc/ \ && cp /staged/profile.toml /opt/crowdb/etc/profile.toml \ && cp /staged/entrypoint.sh /opt/crowdb/bin/entrypoint \ && chown -R crowdb:crowdb /opt/crowdb/data /opt/crowdb/run \ @@ -37,13 +36,12 @@ LABEL org.opencontainers.image.title="CROWDB Node" \ org.opencontainers.image.version="$PREVIEW_VERSION" \ org.crowdb.runtime.sha256="$RUNTIME_SHA256" \ org.crowdb.debug-info="$DEBUG_INFO" \ - org.crowdb.startup.default="manual" -ENV CROWDB_STARTUP_MODE="manual" \ - PATH="/opt/crowdb/bin:${PATH}" \ + org.crowdb.startup.default="local" +ENV PATH="/opt/crowdb/bin:${PATH}" \ LD_LIBRARY_PATH="/opt/crowdb/lib" \ CROWDB_RUNTIME_ROOT="/opt/crowdb/run" USER root VOLUME ["/opt/crowdb/data"] -EXPOSE 9090 9091 9092 9095 2222 +EXPOSE 9090 9091 9092 9093 9095 2222 STOPSIGNAL SIGTERM ENTRYPOINT ["/usr/bin/tini", "--", "/opt/crowdb/bin/entrypoint"] diff --git a/container/oci-image/README.md b/container/oci-image/README.md index 06ca49379..4bf82d37a 100644 --- a/container/oci-image/README.md +++ b/container/oci-image/README.md @@ -11,13 +11,13 @@ socket or containerd service. Running the image is a separate operation. ## Build and import ```bash -pixi run image-build --output target/crowdb.oci.tar -pixi run image-inspect target/crowdb.oci.tar --receipt +pixi run oci-build --output target/crowdb.oci.tar +pixi run --manifest-path container/oci-image/pixi.toml inspect target/crowdb.oci.tar --receipt # Optional import into the system Docker image store: -pixi run image-import target/crowdb.oci.tar --image crowdb-node:dev +pixi run --manifest-path container/oci-image/pixi.toml import target/crowdb.oci.tar --image crowdb-node:dev ``` -`image-build` compiles/stages with the repository environment, then selects the +`oci-build` compiles/stages with the repository environment, then selects the locked standalone builder environment. `--staged` reuses an existing `target/container-runtime`. Unsupported platforms fail before compilation. Only Linux amd64 is enabled; macOS will compile inside a Linux VM backend. @@ -33,7 +33,7 @@ Both rootless build/import and the explicit rootful recipe are supported. If rootless prerequisites are unavailable, an administrator can explicitly run: ```bash -pixi run stage-single-node-container +pixi run bash container/oci-image/stage.sh pixi install --locked --manifest-path container/oci-image/pixi.toml pixi run sudo "$(command -v pixi)" run --as-is \ --manifest-path container/oci-image/pixi.toml build --privileged \ @@ -61,37 +61,213 @@ processes and forwards stop signals to the monitor; the monitor retains service shutdown ordering and child exit ownership. No runtime-specific `--init` flag is needed. Do not replace the image entrypoint with the monitor for normal nodes. -## Startup policies +## Local cluster shortcuts -The image defaults to `CROWDB_STARTUP_MODE=manual`: discovery, SSH and UI start. -Operators authenticate and admit candidates into the cluster. Auto join defaults -to enabled: three eligible admitted nodes are ordered by UUID, and the smallest -UUID's monitor initializes Store 0 / Group 0 with those three voters. Membership -is then fixed; later admitted nodes do not automatically become Group-0 voters. -Disable Auto join to submit initialization through the UI instead. Accepted -initialization continues in the monitor after the UI closes; inspect its progress -or retry the same operation after a reported failure. +Build and load once, then run a named cluster: -Storage preparation is separate from effective Group-0 disk membership. Configure -and verify each disk's placement, metadata binding, owner and DiskIO availability. -A single admitted node in standard mode does not imply the single-node policy. +```bash +pixi run oci-build --load +pixi run oci-run --cluster demo +# Explicit equivalent of oci-run: +pixi run oci-run-1c --cluster demo +# Use a different name and port range for a three-container cluster: +pixi run oci-run-3c --cluster demo-three --port-base 9190 +pixi run oci-stop --cluster demo-three +pixi run oci-run-3c --cluster demo-three +pixi run oci-clean --cluster demo-three +# Recreate containers and bridge with the original volumes and accepted policy: +pixi run oci-run-3c --cluster demo-three +# Explicitly delete the owned data too: +pixi run oci-clean --cluster demo-three --delete-data +``` + +Run never compiles, downloads or pulls an image. It accepts `--image`, `--cpus`, +`--memory-mib` and `--port-base` on first creation, and retains those inputs. +A different node count or conflicting input requires another cluster name. +Ports for each node are UI/S3/Iceberg/Dataset at base/base+1/base+2/base+3; +subsequent members add 10 and 20. All published ports bind to `127.0.0.1`. +Names default to `local`; global stop/clean requires explicit `--all`. + +Host receipts live in `.crowdb-runtime/persistent/oci-clusters/`. The shortcuts +verify the Docker daemon/context and exact ownership labels before effects. +Clean stops and removes owned containers and unused owned bridges. It keeps +volumes and restart inputs unless `--delete-data` is explicit. Image cleanup +only considers dangling CROWDB images, preserving retained cluster references +and images used by any running or stopped container. Attached or relabelled +networks and volumes referenced elsewhere are rejected. No global prune runs. +These host scripts invoke Docker directly and require no CLI binary or application +target binding. Explicit data deletion also removes a matching optional CLI +target binding; a binding with different ownership is rejected before deletion. + +One node and three nodes use the same editable monitor lifecycle. Local mode +prepares test storage and all access services. An incomplete three-node start +waits for peers; it never falls back to singleton bootstrap. Each node retains +its identity, SSH keys, cluster binding and accepted setup in its own volume. +Discovery uses mDNS/DNS-SD; no DNS server installation or peer list is required. +Advertisements supply hints; pinned SSH identity verifies new member addresses. + +## Common CLI lifecycle and application targets + +The CLI can create the same local Docker clusters and wait for complete +application readiness. It shares host ownership receipts with the shortcuts; +neither entry point implements a separate bootstrap or storage deployment. -Explicit `CROWDB_STARTUP_MODE=single` creates virtual disks and initializes the -existing development profile automatically. UI editing and backend mutations -are disabled. Both policies use the same image. Existing single-node development -helpers select `single` explicitly. Persisted bootstrap/admission records reject -incompatible policy changes; do not change modes on a populated data directory. +```bash +pixi run crowdb-cli --cluster demo cluster start +pixi run crowdb-cli --cluster demo-three cluster start --nodes 3 --port-base 9190 +pixi run crowdb-cli --cluster demo-three cluster check --wait 180 --json +pixi run crowdb-cli --cluster demo-three cluster stop +pixi run crowdb-cli --cluster demo-three cluster start +pixi run crowdb-cli --cluster demo-three s3 bucket put example +pixi run crowdb-cli --cluster demo-three s3 object put example hello.txt --text hello +pixi run crowdb-cli --management-endpoint http://127.0.0.1:9190 s3 object get example hello.txt +pixi run oci-clean --cluster demo-three --delete-data +``` -For a local automatic Docker container: +Use the built `crowdb-cli` executable on PATH, or substitute its absolute path +after `pixi run`. Start defaults to cluster `local`, one node, 1 CPU, 1536 MiB, +and console port 9090. It accepts `--image`, `--cpus`, `--memory-mib` and +`--port-base` before sealing the first launch inputs. Subsequent starts verify +those inputs and resume existing state. `--wait` bounds readiness observation; +an unmet check returns 1 and leaves accepted work resumable. + +For production, `cluster start --network host` runs one node on this physical +host. Select `--runtime docker` or `--runtime containerd`; containerd also accepts +`--runtime-endpoint` and `--snapshotter native|overlayfs`. Python 3.11 or newer +and the selected runtime client must be installed on the host. Load the immutable +node image first; this command never builds or pulls it. ```bash -pixi run docker run -d --name crowdb-single --cpus 2 --memory 1g \ - -e CROWDB_STARTUP_MODE=single -p 9090:9090 -p 9091:9091 -p 9092:9092 \ - --mount type=volume,source=crowdb-data,target=/opt/crowdb/data \ +pixi run crowdb-cli --cluster production cluster start --runtime containerd \ + --network host --snapshotter native --image sha256: \ + --nodes 3 --cpus 2 --memory-mib 2048 --data-root /srv/crowdb/data \ + --physical-host-id --interface eth0 --management-address 192.0.2.10 \ + --node-policy-file /srv/crowdb/node-policy.toml \ + --password-file /srv/crowdb/private-password --device /dev/nvme1n1 +pixi run crowdb-cli --cluster production cluster stop +pixi run crowdb-cli --cluster production cluster start +pixi run oci-host-clean --cluster production +``` + +The existing data directory must be writable by UID 10001. The node policy +must select production mode, the same cluster and initial voter count, supplied +media and reachable public protocol URIs. `--nodes 3` selects the three-voter +formation policy; run one node on each host. Host networking uses port 9090 and +rejects `--port-base`. The management address must belong to the selected local +interface. Disk files use `--disk-file`; files and devices are explicit existing +inputs and are never created by the CLI. + +The CLI seals a private host receipt before creating a container. Start/resume +and stop verify the recorded daemon, Unix endpoint, context/namespace, owner, +image, resource limits, mounts and startup inputs. Resume preserves the accepted +policy and media identities; foreign same-name containers are rejected. +`oci-host-clean` works without the CLI binary and removes only that verified +container. It preserves data, raw devices, files, credentials and both launch +and application receipts, so the next start can recreate the container. + +Status/check and application commands need only management HTTP access. A named +CLI target pins the observed cluster UUID; explicit management endpoints must +agree with that UUID and with one another. An unavailable selected endpoint +does not fall back to localhost. S3 commands retrieve the selected cluster's +published S3 endpoint and credentials, then issue signed native S3 requests. +They require no Docker socket or local deployment directory. HTTP diagnostics +redact authorization and session tokens. The CLI has no S3-specific cluster +lifecycle or container cleanup command. + +Workloads attach to the same selected cluster. They never start, stop, reset or +provision it. Native KV/storage clients additionally need network reachability +to the advertised RPC addresses; HTTP-only port forwarding is insufficient. + +```bash +pixi run crowdb-cli --cluster demo-three bench kv prepare --store 0 --group 1 --keys 100 +pixi run crowdb-cli --cluster demo-three bench s3 mix --operations 100 --dataset-objects 8 +pixi run crowdb-cli --cluster demo-three bench diskdb mix --duration-secs 2 --concurrency 1 +pixi run crowdb-cli --cluster demo-three bench chunkdb allocate --duration-secs 2 --copy-count 1 +pixi run crowdb-cli --cluster demo-three bench chunkio write --objects 1 --duration-secs 2 \ + --mirror-copies 1 --block-size 1048576 --chunk-size 16777216 --prefetch-chunks 1 +``` + +The local storage policy accepts 1 MiB mirror strips with one or two copies. +Use the deployment's supported layout explicitly; a benchmark never changes +that policy. DiskDB checks current disk-group geometry and accounting units +instead of requiring a fixed group/disk count. `bench kv prepare` writes only +its declared keys. S3 workloads use a fresh bucket per invocation and delete +only that workload's objects/bucket. `--max-dataset-bytes` bounds the prepared +dataset; benchmark output does not claim remote memory usage or backend type. + +## Direct Docker and Compose + +A direct single-container cluster needs a volume and four published ports: + +```bash +pixi run docker run -d --name crowdb-one --cpus 1 --memory 1536m \ + -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:9091 \ + -p 127.0.0.1:9092:9092 -p 127.0.0.1:9093:9093 \ + --mount type=volume,source=crowdb-one-data,target=/opt/crowdb/data \ crowdb-node:dev -pixi run docker exec crowdb-single crowdb-monitor readiness ``` +For three nodes, create a dedicated bridge and execute these commands separately +in any order. No node addresses or membership list are supplied: + +```bash +pixi run docker network create crowdb-demo +pixi run docker run -d --name crowdb-demo-1 --network crowdb-demo --cpus 1 --memory 1536m \ + -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ + -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9091 \ + -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9092 \ + -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9093 \ + -p 127.0.0.1:9090:9090 -p 127.0.0.1:9091:9091 \ + -p 127.0.0.1:9092:9092 -p 127.0.0.1:9093:9093 \ + --mount type=volume,source=crowdb-demo-1-data,target=/opt/crowdb/data crowdb-node:dev +pixi run docker run -d --name crowdb-demo-2 --network crowdb-demo --cpus 1 --memory 1536m \ + -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ + -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9101 \ + -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9102 \ + -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9103 \ + -p 127.0.0.1:9100:9090 -p 127.0.0.1:9101:9091 \ + -p 127.0.0.1:9102:9092 -p 127.0.0.1:9103:9093 \ + --mount type=volume,source=crowdb-demo-2-data,target=/opt/crowdb/data crowdb-node:dev +pixi run docker run -d --name crowdb-demo-3 --network crowdb-demo --cpus 1 --memory 1536m \ + -e CROWDB_CLUSTER=demo -e CROWDB_INITIAL_VOTERS=3 \ + -e CROWDB_S3_PUBLIC_URI=http://127.0.0.1:9111 \ + -e CROWDB_ICEBERG_PUBLIC_URI=http://127.0.0.1:9112 \ + -e CROWDB_DATASET_PUBLIC_URI=http://127.0.0.1:9113 \ + -p 127.0.0.1:9110:9090 -p 127.0.0.1:9111:9091 \ + -p 127.0.0.1:9112:9092 -p 127.0.0.1:9113:9093 \ + --mount type=volume,source=crowdb-demo-3-data,target=/opt/crowdb/data crowdb-node:dev +``` + +Equivalent Compose examples are [compose.yaml](compose.yaml) and +[compose-three.yaml](compose-three.yaml): + +```bash +pixi run docker compose -f container/oci-image/compose.yaml up -d +pixi run docker compose -f container/oci-image/compose.yaml down +pixi run docker compose -f container/oci-image/compose-three.yaml up -d +pixi run docker compose -f container/oci-image/compose-three.yaml down +``` + +`down` removes the Compose containers and network, preserving named volumes. +Set `CROWDB_IMAGE` to select the loaded image and `CROWDB_CLUSTER` to isolate +another project. Change the port variables shown in each file before starting +another project concurrently. Keep accepted public origins unchanged on reuse. +Plain Docker and Compose resources have their own ownership; Pixi clean does +not adopt them. Use Docker/Compose stop/remove/down for those resources and +remove each dedicated bridge after its containers are removed. + +Read progress and credentials explicitly with standard Docker tools: + +```bash +pixi run docker logs crowdb-one +pixi run docker exec crowdb-one crowdb-monitor credentials show --format env +``` + +The credential command prints S3, Iceberg and Dataset client values and their +public origins. Keep that output private. Setup completion is historical; +protocol availability requires current services and storage to be ready. +See [local storage and client details](../single-node-container/README.md). + ## Linux host deployment Prepare a persistent directory owned by UID/GID 10001 and a private password file @@ -107,7 +283,7 @@ explicit configuration, inherited by services as `TOKIO_WORKER_THREADS`. ```bash -pixi run image-run --runtime docker \ +pixi run --manifest-path container/oci-image/pixi.toml -e runtime run --runtime docker --cluster production --initial-voters 3 \ --image docker.io/crowdb/crowdb-node@sha256: \ --name crowdb-node --network host --cpus 2 --memory-mib 1024 \ --data-root /var/lib/crowdb --physical-host-id host-one --interface eno1 \ @@ -115,18 +291,27 @@ pixi run image-run --runtime docker \ ``` Host networking uses port 9090 directly; no `-p` mapping is needed. Fixed listener -ports mean one node per host. Use `--seed host:9095` when multicast is unavailable. +ports mean one node per host. Use `--seed http://host:9095` when multicast is unavailable. Explicit `--device /dev/...` permits individual block devices; the host must grant UID/GID 10001 access. The launcher checks device type and effective read/write permissions, actual cgroup v2 CPU/memory limits and the host network namespace. -A failed check removes only the container created by that launch. Data remains. +A failed check retains the owned container and its data for diagnosis and explicit +cleanup. A startup error does not authorize removing runtime or storage state. Physical hardware I/O must also be validated for the selected deployment. +Use `--node-policy-file /absolute/path/node.toml` to pass the normal node's TOML +startup inputs, including explicit supplied-media setup and public protocol URIs. +The launcher rejects symlinked, oversized, unreadable or malformed files before +runtime creation. It mounts the file read-only; the monitor validates its complete +semantics and seals accepted policy before identity, admission or storage setup. +The file must be readable by container UID/GID 10001 and remain available for +recreation with the same accepted inputs. + For containerd, install the separate locked runtime environment: ```bash pixi install --locked --manifest-path container/oci-image/pixi.toml -e runtime -pixi run image-run --runtime containerd \ +pixi run --manifest-path container/oci-image/pixi.toml -e runtime run --runtime containerd --cluster production --initial-voters 3 \ --address /run/crowdb-containerd/containerd.sock \ --image docker.io/crowdb/crowdb-node@sha256: \ --name crowdb-node --network host --cpus 2 --memory-mib 1024 \ @@ -135,7 +320,8 @@ pixi run image-run --runtime containerd \ ``` The host administrator starts containerd separately. Initial support is rootful -Linux, cgroup v2, namespace `crowdb`, and host networking. `k8s.io` is rejected. +Linux, cgroup v2 and host networking. The cluster name maps to the containerd +namespace `crowdb-`; runtime namespace is not a separate cluster selector. The default snapshotter is overlayfs; use `--snapshotter native` where overlayfs is unavailable. No Docker daemon, runtime socket mount or CNI plugin is needed. The tested tool versions are containerd 2.4.1, nerdctl 2.4.1 and runc 1.4.3. @@ -157,8 +343,9 @@ Never mount a Docker or containerd socket into the node to manage its storage. Standard startup prepares the common disk directory for UID/GID 10001. It does not recursively change the ownership or mode of supplied files or devices. -- With no supplied inputs, standard node setup defaults to one 16-GiB managed - file with preallocation and a 1-GiB host reserve. It is created in the retained +- With no supplied inputs, Local node setup defaults to one 16-GiB sparse + managed file and a 1-GiB host reserve. Production automatic storage uses + preallocation and rejects sparse allocation. It is created in the retained data directory. Additional Create File Disk operations use generated paths in that same directory. Sparse allocation is an explicit advanced choice; failed preallocation remains a failure. @@ -178,11 +365,12 @@ not recursively change the ownership or mode of supplied files or devices. while retaining its bytes, ID, geometry and allocation metadata. Rejoin uses that retained identity Offline; explicit validated Online restores participation. Removal does not evacuate data or establish recovery/redundancy. -- Explicit single mode retains the development profile's four sparse 16-GiB - files and one-voter Store-0/Group-0 initialization. It is not inferred from - seeing only one standard node and cannot replace an accepted standard mode. - Supplied external media require manual mode. Single startup keeps its initial - root pristine until bootstrap accepts the manifest. +- The accepted deployment policy explicitly selects Local or Production and one + or three initial voters. Local permits simulated placement with zero physical + host failure tolerance. Production requires configured credentials and real + placement; a single production voter cannot satisfy protected ChunkDB + placement. Supplied or diskless storage is an explicit policy choice. Retained + policy, identity and bootstrap inputs cannot be replaced by image defaults. ## Daemon and node lifecycle @@ -202,17 +390,11 @@ daemon; containerd's restart plugin and `unless-stopped` policy recover exited nodes. Monitor supervises the child services. Readiness failure alone never restarts or reinitializes the whole node. -```bash -pixi run image-ctl --runtime containerd --address /run/crowdb-containerd/containerd.sock logs crowdb-node -pixi run image-ctl --runtime containerd --address /run/crowdb-containerd/containerd.sock readiness crowdb-node -pixi run image-ctl --runtime containerd --address /run/crowdb-containerd/containerd.sock stop crowdb-node -pixi run image-ctl --runtime containerd --address /run/crowdb-containerd/containerd.sock remove crowdb-node -``` - -The lifecycle helper accepts launcher-owned containers only; `remove` requires a -stopped node and does not remove data or images. Recreate with `image-run` and the -same root/host identity to retain UUID, SSH keys and cluster binding. `restart` -and `liveness` are also available. Use `--runtime docker` for the same operations. +Use standard runtime tools to inspect, read logs, execute probes, stop and remove +nodes. For containerd select its address and namespace on `nerdctl`; for Docker +use its selected context. Removing a stopped container does not delete its +external data. Host networking uses one node per host with the fixed listener +ports; RDMA uses this host-network path. ## CI and acceptance @@ -222,8 +404,7 @@ matching successful image digests allow publication. The protected publish job copies the graph with digest preservation, verifies the remote digest, signs it, and attaches SPDX SBOM and SLSA provenance before updating `latest`. The canonical repository is `crowdb/crowdb-node`; the `crowdb/crowdb-iceberg`, `crowdb/crowdb-s3` and `crowdb/crowdb-dataset` repositories -are single-node usage aliases of that image. Their examples explicitly set -`CROWDB_STARTUP_MODE=single`; image names alone do not select startup mode. +are single-node usage aliases of that image. Image names do not select a different lifecycle or UI permission policy. The Dataset alias uses the same image. Dataset access runs through the `server access dataset serve` entry point with the versioned REST API and Dataset-specific chunk storage. No second Docker image is published. @@ -234,19 +415,20 @@ Workflow execution and live registry signing require a release dispatch. pixi run test-oci-image pixi run --skip-deps test-single-node-container pixi run test-container-e2e --profile bridge-admission --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 -pixi run test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 -pixi run test-container-e2e --profile containerd-cluster --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 +pixi run sudo "$(command -v pixi)" run --as-is test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 +pixi run test-container-e2e --profile containerd-cluster --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 8192 # One disposable privileged Linux host, with native containerd and no Docker socket inside: -pixi run test-container-e2e --profile containerd-host --concurrency 1 --cpus 3 --memory-mib 2048 \ +pixi run sudo "$(command -v pixi)" run --as-is test-container-e2e --profile containerd-host --concurrency 1 --cpus 3 --memory-mib 3072 \ --archive "$PWD/target/crowdb.oci.tar" ``` Developer-only `tests/build-sandbox.py` and `tests/runtime-sandbox.py` use system Docker to provide disposable privileged Linux hosts. Inside those hosts the builder/runtime tests have no Docker CLI/socket. They verify the isolated process -contract, not a bare-metal installation. The runtime gate exercises both modes, -Group-0 admission/bootstrap, enforced resources, probes, SIGTERM and persisted -identity/key recovery. Three isolated Linux namespace hosts additionally cover +contract, not a bare-metal installation. The native host gate exercises production metadata lifecycle separately from +Local full-protocol readiness and retained recreation. Production metadata alone +is not whole-cluster Ready. Enforced resources, host networking and retained +identity/key checks apply to both policies. Three isolated Linux namespace hosts additionally cover discovery, SSH admission, Group-0 quorum and member recovery. This simulation is not a physical multi-host network qualification. Existing Docker tests cover application writes and crash recovery. Local registry transfer retained the exact diff --git a/container/oci-image/compose-three.yaml b/container/oci-image/compose-three.yaml new file mode 100644 index 000000000..af59582c7 --- /dev/null +++ b/container/oci-image/compose-three.yaml @@ -0,0 +1,58 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +name: crowdb-${CROWDB_CLUSTER:-local-three} +x-node: &node + image: ${CROWDB_IMAGE:-crowdb-node:dev} + pull_policy: never + cpus: 1 + mem_limit: 1536m +services: + node1: + <<: *node + environment: + CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local-three} + CROWDB_INITIAL_VOTERS: "3" + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_S3_PORT:-9091} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-9092} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093} + ports: + - "127.0.0.1:${CROWDB_NODE1_UI_PORT:-9090}:9090" + - "127.0.0.1:${CROWDB_NODE1_S3_PORT:-9091}:9091" + - "127.0.0.1:${CROWDB_NODE1_ICEBERG_PORT:-9092}:9092" + - "127.0.0.1:${CROWDB_NODE1_DATASET_PORT:-9093}:9093" + volumes: + - node1-data:/opt/crowdb/data + node2: + <<: *node + environment: + CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local-three} + CROWDB_INITIAL_VOTERS: "3" + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_S3_PORT:-9101} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-9102} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103} + ports: + - "127.0.0.1:${CROWDB_NODE2_UI_PORT:-9100}:9090" + - "127.0.0.1:${CROWDB_NODE2_S3_PORT:-9101}:9091" + - "127.0.0.1:${CROWDB_NODE2_ICEBERG_PORT:-9102}:9092" + - "127.0.0.1:${CROWDB_NODE2_DATASET_PORT:-9103}:9093" + volumes: + - node2-data:/opt/crowdb/data + node3: + <<: *node + environment: + CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local-three} + CROWDB_INITIAL_VOTERS: "3" + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_S3_PORT:-9111} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-9112} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113} + ports: + - "127.0.0.1:${CROWDB_NODE3_UI_PORT:-9110}:9090" + - "127.0.0.1:${CROWDB_NODE3_S3_PORT:-9111}:9091" + - "127.0.0.1:${CROWDB_NODE3_ICEBERG_PORT:-9112}:9092" + - "127.0.0.1:${CROWDB_NODE3_DATASET_PORT:-9113}:9093" + volumes: + - node3-data:/opt/crowdb/data +volumes: + node1-data: + node2-data: + node3-data: diff --git a/container/oci-image/compose.yaml b/container/oci-image/compose.yaml new file mode 100644 index 000000000..7ad711291 --- /dev/null +++ b/container/oci-image/compose.yaml @@ -0,0 +1,24 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +name: crowdb-${CROWDB_CLUSTER:-local} +services: + node: + image: ${CROWDB_IMAGE:-crowdb-node:dev} + pull_policy: never + cpus: 1 + mem_limit: 1536m + environment: + CROWDB_CLUSTER: ${CROWDB_CLUSTER:-local} + CROWDB_INITIAL_VOTERS: "1" + CROWDB_S3_PUBLIC_URI: http://127.0.0.1:${CROWDB_S3_PORT:-9091} + CROWDB_ICEBERG_PUBLIC_URI: http://127.0.0.1:${CROWDB_ICEBERG_PORT:-9092} + CROWDB_DATASET_PUBLIC_URI: http://127.0.0.1:${CROWDB_DATASET_PORT:-9093} + ports: + - "127.0.0.1:${CROWDB_UI_PORT:-9090}:9090" + - "127.0.0.1:${CROWDB_S3_PORT:-9091}:9091" + - "127.0.0.1:${CROWDB_ICEBERG_PORT:-9092}:9092" + - "127.0.0.1:${CROWDB_DATASET_PORT:-9093}:9093" + volumes: + - node-data:/opt/crowdb/data +volumes: + node-data: diff --git a/container/oci-image/docker-local.py b/container/oci-image/docker-local.py new file mode 100644 index 000000000..483580622 --- /dev/null +++ b/container/oci-image/docker-local.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Docker-only local cluster shortcuts, independent of crowdb-cli.""" + +import argparse +import sys + +from docker_local_cleanup import clean, image_garbage, selected, stop +from docker_local_launch import accepted, run +from docker_local_state import Docker, check_daemon, record_path + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("run", "stop", "clean")) + scope = parser.add_mutually_exclusive_group() + scope.add_argument("--cluster") + scope.add_argument("--all", action="store_true", help="explicitly select every recorded local cluster") + parser.add_argument("--nodes", type=int, choices=(1, 3)) + parser.add_argument("--image") + parser.add_argument("--port-base", type=int) + parser.add_argument("--cpus", type=float) + parser.add_argument("--memory-mib", type=int) + parser.add_argument("--delete-data", action="store_true", help="clean also deletes owned persistent volumes") + args = parser.parse_args() + if args.operation == "run" and (args.all or args.delete_data): + parser.error("run selects one cluster and preserves its data") + if args.operation != "run" and any(value is not None for value in + (args.nodes, args.image, args.port_base, args.cpus, args.memory_mib)): + parser.error("launch inputs apply only to run") + if args.delete_data and args.operation != "clean": + parser.error("--delete-data applies only to clean") + args.cluster = args.cluster or "local" + record_path(args.cluster) + docker = Docker() + if args.operation == "run": + run(docker, accepted(docker, args)) + return + records = selected(args.cluster, args.all) + for record in records: + check_daemon(docker, record) + for record in records: + if args.operation == "stop": + stop(docker, record) + else: + clean(docker, record, args.delete_data) + if args.operation == "clean": + image_garbage(docker, runtime=records[0]["runtime"] if records else docker.identity()) + + +if __name__ == "__main__": + try: + main() + except (OSError, ValueError, RuntimeError, KeyError, TypeError) as error: + print(str(error), file=sys.stderr) + sys.exit(1) diff --git a/container/oci-image/docker_local_cleanup.py b/container/oci-image/docker_local_cleanup.py new file mode 100644 index 000000000..bf2512237 --- /dev/null +++ b/container/oci-image/docker_local_cleanup.py @@ -0,0 +1,101 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Scoped Docker stop and cleanup; retained volumes and receipts are the default.""" + +import json +import stat + +from docker_local_state import RECORDS, check_daemon, load, record_path, require_owned + + +def selected(cluster, all_clusters, directory=RECORDS): + if all_clusters: + if directory.is_symlink(): + raise ValueError("cluster receipt directory must not be a symlink") + return [load(path.stem, directory) for path in sorted(directory.glob("*.json"))] + record = load(cluster, directory) + if record is None: + raise ValueError("cluster has no local Docker receipt; use Docker tools for direct containers") + return [record] + + +def current(docker, record, kind, identity): + item = json.loads(docker.call(kind, "inspect", identity))[0] + require_owned(item, record, kind) + check_daemon(docker, record) + return item + + +def stop(docker, record): + check_daemon(docker, record) + for item in docker.owned(record, "container"): + item = current(docker, record, "container", item["Id"]) + if item["State"]["Running"]: + docker.call("stop", "--time", "20", item["Id"]) + + +def clean(docker, record, delete_data=False, directory=RECORDS): + binding = owned_binding(record, directory) if delete_data else None + stop(docker, record) + for item in docker.owned(record, "container"): + item = current(docker, record, "container", item["Id"]) + if item["State"]["Running"]: + raise ValueError("container restarted during clean; retry after stopping its owner") + docker.call("rm", item["Id"]) + for item in docker.owned(record, "network"): + item = current(docker, record, "network", item["Id"]) + if item.get("Containers"): + raise ValueError("owned bridge still has attached containers; refusing network deletion") + docker.call("network", "rm", item["Id"]) + if delete_data: + for item in docker.owned(record, "volume"): + item = current(docker, record, "volume", item["Name"]) + if docker.call("ps", "-aq", "--filter", "volume=" + item["Name"]): + raise ValueError("owned volume is referenced by another container") + docker.call("volume", "rm", item["Name"]) + if binding: + # Keep ownership retryable if removing the binding fails. + binding.unlink() + record_path(record["cluster"], directory).unlink() + print(f"Cleaned containers and bridges for {record['cluster']}; " + + ("deleted owned data volumes" if delete_data else "preserved data volumes and restart inputs")) + + +def owned_binding(record, directory): + path = directory.parent / "clusters" / record["cluster"] / "target.json" + if any(parent.is_symlink() for parent in (path, *path.parents)): + raise ValueError("cluster target binding must not contain symlinks") + if not path.exists(): + return None + metadata = path.stat() + if not stat.S_ISREG(metadata.st_mode) or metadata.st_mode & 0o077 or metadata.st_size > 32768: + raise ValueError("cluster target binding must be a bounded private regular file") + target = json.loads(path.read_text()) + expected = {"owner": record["owner"], "selection": { + "runtime": "docker", "context": record["runtime"]["context"], "endpoint": record["runtime"]["endpoint"]}} + if target.get("version") != 1 or target.get("name") != record["cluster"] or target.get("runtime") != expected: + raise ValueError("cluster target binding differs from owned runtime inputs") + return path + + +def image_garbage(docker, directory=RECORDS, runtime=None): + # Only dangling CROWDB images are garbage. Receipts retain image identity + # even after their last container is cleaned; a stopped foreign user also + # protects an image. Docker's non-force removal handles reference races. + runtime = docker.identity() if runtime is None else runtime + if docker.identity() != runtime: + raise ValueError("Docker endpoint/context changed before image cleanup") + retained = {record["image_id"] for record in selected(None, True, directory)} + candidates = docker.call("image", "ls", "-q", "--filter", "dangling=true", "--filter", + "label=org.opencontainers.image.source=https://github.com/buzzcrow/crowdb").splitlines() + for identity in candidates: + item = json.loads(docker.call("image", "inspect", identity))[0] + if item.get("Config", {}).get("Labels", {}).get("org.opencontainers.image.source") != "https://github.com/buzzcrow/crowdb": + continue + if item["Id"] in retained or docker.call("ps", "-aq", "--filter", "ancestor=" + item["Id"]): + continue + if item.get("RepoTags") or item.get("RepoDigests"): + continue + if docker.identity() != runtime: + raise ValueError("Docker endpoint/context changed during image cleanup") + docker.call("image", "rm", item["Id"]) diff --git a/container/oci-image/docker_local_launch.py b/container/oci-image/docker_local_launch.py new file mode 100644 index 000000000..be02c5f4c --- /dev/null +++ b/container/oci-image/docker_local_launch.py @@ -0,0 +1,113 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Direct Docker resource creation; the OCI node owns all cluster preparation.""" + +import json +import math +import uuid + +from docker_local_state import CLUSTER, OWNER, check_daemon, load, require_owned, seal + + +def arguments(record): + return ["--label", f"{CLUSTER}={record['cluster']}", "--label", f"{OWNER}={record['owner']}"] + + +def ensure(docker, record, kind, name): + names = {item.get("Name", "").lstrip("/"): item for item in docker.objects(kind)} + if name in names: + require_owned(names[name], record, kind) + return names[name] + check_daemon(docker, record) + docker.call(kind, "create", *arguments(record), name) + item = json.loads(docker.call(kind, "inspect", name))[0] + require_owned(item, record, kind) + return item + + +def accepted(docker, args): + previous = load(args.cluster) + if previous: + check_daemon(docker, previous) + for key, requested in (("nodes", args.nodes), ("port_base", args.port_base), + ("image", args.image), ("cpus", args.cpus), ("memory_mib", args.memory_mib)): + if requested is not None and previous[key] != requested: + raise ValueError(f"{key} differs from retained cluster inputs") + return previous + image = args.image or "crowdb-node:dev" + try: + metadata = json.loads(docker.call("image", "inspect", image))[0] + except RuntimeError as error: + raise ValueError("image is not loaded; run pixi run oci-build and load its OCI archive first") from error + if (metadata["Os"], metadata["Architecture"]) != ("linux", "amd64"): + raise ValueError("local node image must be Linux amd64") + if metadata.get("Config", {}).get("Labels", {}).get("org.opencontainers.image.source") != "https://github.com/buzzcrow/crowdb": + raise ValueError("image is not a CROWDB node image") + record = {"version": 1, "cluster": args.cluster, "owner": str(uuid.uuid4()), "runtime": docker.identity(), + "nodes": args.nodes if args.nodes is not None else 1, "port_base": args.port_base if args.port_base is not None else 9090, "image": image, + "image_id": metadata["Id"], "cpus": args.cpus if args.cpus is not None else 1, + "memory_mib": args.memory_mib if args.memory_mib is not None else 1536} + if record["nodes"] not in (1, 3) or not 1 <= record["port_base"] <= 65512: + raise ValueError("local cluster requires one or three nodes and a valid port range") + if not math.isfinite(record["cpus"]) or record["cpus"] <= 0 or record["memory_mib"] < 512: + raise ValueError("CPU must be positive and memory at least 512 MiB") + seal(record) + return record + + +def verify_container(item, record, index, volume, network): + require_owned(item, record, "container") + host = item["HostConfig"] + base = record["port_base"] + (index - 1) * 10 + ports = {f"{9090+offset}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(base+offset)}] + for offset in range(4)} + expected_environment = {"CROWDB_CLUSTER": record["cluster"], "CROWDB_INITIAL_VOTERS": str(record["nodes"])} + for offset, protocol in enumerate((None, "S3", "ICEBERG", "DATASET")): + if protocol: + expected_environment[f"CROWDB_{protocol}_PUBLIC_URI"] = f"http://127.0.0.1:{base+offset}" + environment = dict(value.split("=", 1) for value in item["Config"].get("Env", []) if "=" in value) + mounts = [mount for mount in item["Mounts"] if mount["Destination"] == "/opt/crowdb/data"] + if (item["Image"] != record["image_id"] or host["NanoCpus"] != round(record["cpus"] * 1_000_000_000) + or host["Memory"] != record["memory_mib"] * 1024 * 1024 + or host.get("PortBindings") != ports or host.get("NetworkMode") != network + or len(mounts) != 1 or mounts[0].get("Type") != "volume" or mounts[0].get("Name") != volume + or not mounts[0].get("RW") + or any(environment.get(key) != value for key, value in expected_environment.items())): + raise ValueError("retained container resources, endpoints, image or deployment inputs differ") + + +def run(docker, record): + check_daemon(docker, record) + # Inspect the sealed local image ID before resource effects; run never pulls. + docker.call("image", "inspect", record["image_id"]) + network = "crowdb-" + record["cluster"] + bridge = ensure(docker, record, "network", network) + if bridge.get("Driver") != "bridge": + raise ValueError("local cluster requires its owned Docker bridge") + containers = {item["Name"].lstrip("/"): item for item in docker.objects("container")} + for index in range(1, record["nodes"] + 1): + name = f"{network}-{index}" + volume = name + "-data" + ensure(docker, record, "volume", volume) + if name in containers: + item = containers[name] + verify_container(item, record, index, volume, network) + if not item["State"]["Running"]: + check_daemon(docker, record) + docker.call("start", item["Id"]) + continue + base = record["port_base"] + (index - 1) * 10 + command = ["run", "-d", "--pull", "never", "--name", name, *arguments(record), + "--network", network, "--cpus", record["cpus"], "--memory", f"{record['memory_mib']}m", + "--mount", f"type=volume,source={volume},target=/opt/crowdb/data", + "-e", "CROWDB_CLUSTER=" + record["cluster"], "-e", f"CROWDB_INITIAL_VOTERS={record['nodes']}"] + for offset, protocol in enumerate((None, "S3", "ICEBERG", "DATASET")): + command += ["-p", f"127.0.0.1:{base+offset}:{9090+offset}"] + if protocol: + command += ["-e", f"CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{base+offset}"] + check_daemon(docker, record) + identity = docker.call(*command, record["image_id"]) + item = json.loads(docker.call("container", "inspect", identity))[0] + verify_container(item, record, index, volume, network) + print(f"Started {record['nodes']} node(s) for {record['cluster']}; UI http://127.0.0.1:{record['port_base']}/") + print("Node monitors continue discovery and preparation; inspect progress in the UI or with docker logs.") diff --git a/container/oci-image/docker_local_state.py b/container/oci-image/docker_local_state.py new file mode 100644 index 000000000..0605f0357 --- /dev/null +++ b/container/oci-image/docker_local_state.py @@ -0,0 +1,128 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Private host receipts and exact Docker resource ownership for local clusters.""" + +import json +import math +import os +from pathlib import Path +import re +import subprocess +import stat +import uuid + +CLUSTER = "org.crowdb.cluster" +OWNER = "org.crowdb.cluster.owner" +ROOT = Path(__file__).resolve().parents[2] +RECORDS = Path(os.environ.get("CROWDB_RUNTIME_ROOT", ROOT / ".crowdb-runtime")) / "persistent/oci-clusters" + + +class Docker: + def call(self, *args): + result = subprocess.run(["docker", *map(str, args)], text=True, capture_output=True, timeout=60) + if result.returncode: + raise RuntimeError(f"docker {args[0]} failed: {result.stderr.strip()}") + return result.stdout.strip() + + def identity(self): + context = self.call("context", "show") + details = json.loads(self.call("context", "inspect", context))[0] + return {"daemon": self.call("info", "--format", "{{.ID}}"), "context": context, + "endpoint": os.environ.get("DOCKER_HOST") or details["Endpoints"]["docker"]["Host"]} + + def objects(self, kind, filters=()): + args = ("ps", "-aq") if kind == "container" else (kind, "ls", "-q") + for condition in filters: + args += ("--filter", condition) + ids = self.call(*args).splitlines() + return json.loads(self.call(kind, "inspect", *ids)) if ids else [] + + def owned(self, record, kind): + filters = (f"label={CLUSTER}={record['cluster']}", f"label={OWNER}={record['owner']}") + return [item for item in self.objects(kind, filters) if labels(item, kind).get(CLUSTER) == record["cluster"] + and labels(item, kind).get(OWNER) == record["owner"] + and item.get("Name", "").lstrip("/") in resource_names(record, kind)] + + +def labels(item, kind): + return (item.get("Config", {}).get("Labels") if kind in ("container", "image") else item.get("Labels")) or {} + + +def check_daemon(docker, record): + if docker.identity() != record["runtime"]: + raise ValueError("Docker endpoint/context differs from the retained cluster receipt") + + +def record_path(cluster, directory=RECORDS): + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,47}", cluster): + raise ValueError("cluster must be 1–48 ASCII letters, digits, dots, underscores or hyphens, starting with a letter or digit") + if any(parent.is_symlink() for parent in (directory, *directory.parents)): + raise ValueError("cluster receipt directory must not be a symlink") + return directory / (cluster + ".json") + + +def load(cluster, directory=RECORDS): + path = record_path(cluster, directory) + if path.is_symlink(): + raise ValueError("cluster receipt must not be a symlink") + if not path.exists(): + return None + metadata = path.stat() + if not stat.S_ISREG(metadata.st_mode) or metadata.st_mode & 0o077 or metadata.st_size > 1024 * 1024: + raise ValueError("cluster receipt must be a bounded private regular file") + record = json.loads(path.read_text()) + if record.get("version") != 1 or record.get("cluster") != cluster: + raise ValueError("invalid cluster receipt") + required = {"version", "cluster", "owner", "runtime", "nodes", "port_base", "image", "image_id", "cpus", "memory_mib"} + if set(record) != required or not isinstance(record["runtime"], dict): + raise ValueError("invalid cluster receipt fields") + owner = uuid.UUID(record["owner"]) + if owner.int == 0 or str(owner) != record["owner"]: + raise ValueError("invalid cluster receipt owner") + if (set(record["runtime"]) != {"daemon", "context", "endpoint"} + or not all(isinstance(value, str) and value for value in record["runtime"].values()) + or record["nodes"] not in (1, 3) or not isinstance(record["port_base"], int) + or not 1 <= record["port_base"] <= 65512 + or not isinstance(record["cpus"], (float, int)) or not math.isfinite(record["cpus"]) or record["cpus"] <= 0 + or not isinstance(record["memory_mib"], int) or record["memory_mib"] < 512 + or not all(isinstance(record[key], str) and record[key] for key in ("image", "image_id"))): + raise ValueError("invalid retained cluster inputs") + return record + + +def seal(record, directory=RECORDS): + path = record_path(record["cluster"], directory) + directory.mkdir(parents=True, exist_ok=True, mode=0o700) + temporary = directory / ("." + uuid.uuid4().hex) + try: + with open(temporary, "x", opener=lambda path, flags: os.open(path, flags, 0o600)) as stream: + json.dump(record, stream, indent=2) + stream.flush() + os.fsync(stream.fileno()) + try: + os.link(temporary, path) + descriptor = os.open(directory, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + except FileExistsError: + if load(record["cluster"], directory) != record: + raise ValueError("another operation already accepted different cluster inputs") from None + finally: + temporary.unlink(missing_ok=True) + + +def resource_names(record, kind): + prefix = "crowdb-" + record["cluster"] + if kind == "network": + return {prefix} + suffix = "-data" if kind == "volume" else "" + return {f"{prefix}-{index}{suffix}" for index in range(1, record["nodes"] + 1)} + + +def require_owned(item, record, kind): + expected = {CLUSTER: record["cluster"], OWNER: record["owner"]} + if (any(labels(item, kind).get(key) != value for key, value in expected.items()) + or item.get("Name", "").lstrip("/") not in resource_names(record, kind)): + raise ValueError(f"refusing foreign {kind} ownership") diff --git a/container/oci-image/host-clean.py b/container/oci-image/host-clean.py new file mode 100644 index 000000000..21546fbbd --- /dev/null +++ b/container/oci-image/host-clean.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Remove only the recorded host container, preserving disks, data and receipts.""" + +import argparse +import os +from pathlib import Path + +from host_cleanup import clean, selected + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--cluster', required=True) + parser.add_argument('--runtime-root', type=Path, + default=Path(os.environ.get('CROWDB_RUNTIME_ROOT', + Path(__file__).resolve().parents[2] / '.crowdb-runtime'))) + args = parser.parse_args() + try: + clean(selected(args.cluster, args.runtime_root)) + except (OSError, ValueError, RuntimeError, KeyError, TypeError) as error: + parser.error(str(error)) + print(f'Removed owned host container for {args.cluster}; preserved data, media and receipts') + + +if __name__ == '__main__': + main() diff --git a/container/oci-image/host_cleanup.py b/container/oci-image/host_cleanup.py new file mode 100644 index 000000000..a1b3bce20 --- /dev/null +++ b/container/oci-image/host_cleanup.py @@ -0,0 +1,75 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Independent cleanup of a host container, retaining media and launch receipts.""" + +import json +import os +from pathlib import Path +import re +import stat +import uuid + +from host_runtime import HostRuntime, require_owned + + +def read_private(path): + if any(part.is_symlink() for part in (path, *path.parents)): + raise ValueError('host receipt path cannot contain symlinks') + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + except FileNotFoundError: + return None + try: + metadata = os.fstat(descriptor) + if (not stat.S_ISREG(metadata.st_mode) or metadata.st_mode & 0o077 + or metadata.st_uid != os.geteuid() or metadata.st_size > 1024 * 1024): + raise ValueError('host receipt must be an owned bounded private regular file') + content = os.read(descriptor, 1024 * 1024 + 1) + finally: + os.close(descriptor) + if len(content) > 1024 * 1024: + raise ValueError('host receipt exceeds its bound') + return json.loads(content) + + +def selected(cluster, root): + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.-]{0,47}', cluster): + raise ValueError('invalid cluster name') + record = read_private(root / 'persistent/oci-hosts' / (cluster + '.json')) + if record is None: + raise ValueError('no accepted host ownership receipt') + owner = uuid.UUID(record['owner']) + if (record['version'] != 1 or record['cluster'] != cluster + or record['name'] != 'crowdb-' + cluster + '-host' + or str(owner) != record['owner'] or owner.int == 0 + or record['options']['network'] != 'host' + or record['options']['runtime'] not in ('docker', 'containerd')): + raise ValueError('invalid accepted host ownership') + bridge = read_private(root / 'persistent/oci-clusters' / (cluster + '.json')) + if bridge is not None: + raise ValueError('conflicting host and bridge ownership receipts') + binding = read_private(root / 'persistent/clusters' / cluster / 'target.json') + if binding is not None and (binding.get('version') != 1 or binding.get('name') != cluster + or binding.get('management_endpoints') != record['origins'] + or binding.get('runtime') != dict(selection=record['runtime']['selection'], owner=record['owner'])): + raise ValueError('application target conflicts with accepted host ownership') + return record + + +def clean(record): + runtime = HostRuntime(record['options'], record['cluster']) + runtime.check(record) + current = runtime.named(record['name']) + if current is None: + return + require_owned(current, record) + identity = current['Id'] + if current['State']['Running']: + runtime.check(record) + runtime.invoke('stop', identity) + current = runtime.inspect(identity) + require_owned(current, record) + if current['State']['Running']: + raise ValueError('owned container is still running; refusing removal') + runtime.check(record) + runtime.invoke('rm', identity) diff --git a/container/oci-image/host_inputs.py b/container/oci-image/host_inputs.py new file mode 100644 index 000000000..117ce7bb4 --- /dev/null +++ b/container/oci-image/host_inputs.py @@ -0,0 +1,102 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Reuse the ordinary host launch validator before accepting runtime ownership.""" + +import hashlib +import ipaddress +import json +import os +from pathlib import Path +from types import SimpleNamespace +import tomllib + +import run_node +from runtime import node_policy_file +from storage import inputs, validate_retained_inputs +from host_runtime import output + + +class Parser: + @staticmethod + def error(message): + raise ValueError(message) + + +def options(request, previous): + proposed = request['options'] + if previous: + accepted = previous['options'] + for key, value in proposed.items(): + if value is not None and value != [] and value != accepted.get(key): + raise ValueError('host launch arguments differ from accepted inputs: ' + key) + return dict(accepted) + current = {'runtime': 'docker', 'network': 'host', 'snapshotter': 'overlayfs', + 'device': [], 'disk_file': [], 'seed': [], 'runtime_endpoint': None, + 'password_file': None} + current.update({key: value for key, value in proposed.items() if value is not None}) + for key in ('image', 'cpus', 'memory_mib', 'data_root', 'physical_host_id', 'interface', + 'management_address', 'node_policy_file'): + if not current.get(key): + raise ValueError('new host launch requires --' + key.replace('_', '-')) + if current['network'] != 'host': + raise ValueError('this host lifecycle requires explicit host networking') + if current['runtime'] not in ('docker', 'containerd'): + raise ValueError('unsupported host runtime') + if current['runtime'] == 'docker' and proposed.get('snapshotter') is not None: + raise ValueError('--snapshotter requires containerd') + policy_path = node_policy_file(SimpleNamespace(node_policy_file=Path(current['node_policy_file']))) + with policy_path.open('rb') as source: + content = source.read(32769) + if len(content) > 32768: + raise ValueError('node policy exceeds its bound') + try: + policy = tomllib.loads(content.decode('utf-8')) + except (UnicodeError, tomllib.TOMLDecodeError): + raise ValueError('node policy is not valid UTF-8 TOML') from None + current['nodes'] = current.get('nodes') or policy.get('initial_voters', 3) + if (policy.get('version') != 1 or policy.get('mode') != 'production' + or policy.get('cluster') != request['cluster'] + or policy.get('initial_voters') != current['nodes']): + raise ValueError('node policy formation differs from the selected production host launch') + return current + + +def arguments(record): + values = dict(record['options']) + for key in ('data_root', 'node_policy_file', 'password_file'): + if values.get(key): + values[key] = Path(values[key]) + for key in ('device', 'disk_file'): + values[key] = [Path(path) for path in values[key]] + values.update(name=record['name'], cluster=record['cluster'], initial_voters=values['nodes'], + ui_port=None, cpu_shares=None, worker_threads=None, owner_token=record['owner']) + return SimpleNamespace(**values) + + +def management(options): + address = ipaddress.ip_address(options['management_address']) + if address.is_unspecified or address.is_multicast: + raise ValueError('management address must identify this host interface') + interfaces = json.loads(output(['ip', '-json', 'address', 'show', 'dev', options['interface']])) + if str(address) not in {value['local'] for interface in interfaces + for value in interface.get('addr_info', [])}: + raise ValueError('management address is not assigned to the selected interface') + return 'http://' + (f'[{address}]' if address.version == 6 else str(address)) + ':9090' + + +def validate(record, running): + args = arguments(record) + for key in ('data_root', 'node_policy_file', 'password_file'): + value = getattr(args, key) + if value and any(path.is_symlink() for path in (value, *value.parents)): + raise ValueError('host launch paths cannot contain symlinks') + policy = node_policy_file(args) + secret = args.password_file + if secret and secret.stat().st_size > 4096: + raise ValueError('SSH password file exceeds its bound') + run_node.validate(args, Parser(), check_ports=not running) + disks = inputs(args) + validate_retained_inputs(args.data_root, disks) + hashes = {'node_policy': hashlib.sha256(policy.read_bytes()).hexdigest(), + 'password': hashlib.sha256(secret.read_bytes()).hexdigest() if secret else None} + return disks, hashes, [management(record['options'])] diff --git a/container/oci-image/host_lifecycle.py b/container/oci-image/host_lifecycle.py new file mode 100644 index 000000000..063bd7059 --- /dev/null +++ b/container/oci-image/host_lifecycle.py @@ -0,0 +1,102 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Host runtime effects consume a previously sealed private ownership receipt.""" + +import contextlib +import json +import sys +import uuid + +from host_inputs import arguments, options, validate +from host_runtime import HostRuntime, require_owned + + +def prepare(request): + previous = request.get('record') + record = previous or {'version': 1, 'cluster': request['cluster'], + 'owner': str(uuid.uuid4()), 'name': 'crowdb-' + request['cluster'] + '-host'} + if record['version'] != 1 or record['cluster'] != request['cluster']: + raise ValueError('host receipt identity differs from the selected cluster') + record = dict(record, options=options(request, previous)) + runtime = HostRuntime(record['options'], record['cluster']) + identity = runtime.identity() + if previous and identity != record['runtime']: + raise ValueError('runtime identity differs from the accepted host receipt') + current = runtime.named(record['name']) + if current: + if not previous: + raise ValueError('same-name host container has no accepted ownership receipt') + require_owned(current, record) + disks, hashes, origins = validate(record, bool(current and current['State']['Running'])) + reference = record['options']['image'] + image = runtime.inspect(reference.rsplit('@', 1)[-1] if runtime.name == 'containerd' + else reference, image=True) + if (image.get('Os') != 'linux' or image.get('Architecture') != 'amd64' + or image.get('Config', {}).get('Labels', {}).get('org.opencontainers.image.source') + != 'https://github.com/buzzcrow/crowdb'): + raise ValueError('host launch requires an already loaded Linux amd64 CROWDB node image') + runtime_image = (record['options']['image'].rsplit('@', 1)[-1] + if runtime.name == 'containerd' else image['Id']) + current_record = dict(record, runtime=identity, runtime_image=runtime_image, + image_id=image['Id'], disks=disks, file_hashes=hashes, origins=origins) + if previous and current_record != previous: + raise ValueError('host image, policy, credentials, media or address differ from accepted inputs') + return current_record + + +def verify(runtime, current, record): + require_owned(current, record) + if current['State']['Running']: + runtime.verify(current['Id'], record['options']['cpus'], record['options']['memory_mib'], True) + + +def run(request): + record = prepare(request) + if request.get('record') != record: + raise ValueError('seal accepted host inputs before requesting runtime effects') + runtime = HostRuntime(record['options'], record['cluster']) + runtime.check(record) + current = runtime.named(record['name']) + if current: + verify(runtime, current, record) + if not current['State']['Running']: + runtime.check(record) + runtime.invoke('start', current['Id']) + else: + args = arguments(record) + args.image = record['runtime_image'] + runtime.check(record) + # Runtime failure diagnostics belong on stderr; stdout remains JSON. + with contextlib.redirect_stdout(sys.stderr): + runtime.create(args) + current = runtime.inspect(record['name']) + verify(runtime, current, record) + return {'container_id': current['Id']} + + +def stop(request): + record = request['record'] + runtime = HostRuntime(record['options'], record['cluster']) + runtime.check(record) + current = runtime.named(record['name']) + if current: + require_owned(current, record) + if current['State']['Running']: + runtime.check(record) + runtime.invoke('stop', current['Id']) + return {'data_preserved': True} + + +def main(): + request = json.load(sys.stdin) + action = request['operation'] + handler = {'prepare': prepare, 'run': run, 'stop': stop}[action] + print(json.dumps(handler(request))) + + +if __name__ == '__main__': + try: + main() + except (OSError, ValueError, RuntimeError, KeyError, TypeError) as error: + print(str(error), file=sys.stderr) + sys.exit(2) diff --git a/container/oci-image/host_runtime.py b/container/oci-image/host_runtime.py new file mode 100644 index 000000000..11af08074 --- /dev/null +++ b/container/oci-image/host_runtime.py @@ -0,0 +1,135 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Pinned host-runtime commands and exact owned-container inspection.""" + +import json +import os +import shutil +import subprocess +from pathlib import Path + +from runtime import Runtime, OWNER + + +def output(command): + result = subprocess.run(command, text=True, capture_output=True, timeout=60) + if result.returncode: + raise RuntimeError(result.stderr.strip() or 'runtime command failed') + return result.stdout.strip() + + +def executable(name): + binary = shutil.which(name) + if binary is None: + prefix = os.environ.get('CONDA_PREFIX') + candidate = Path(prefix) / 'bin' / name if prefix else None + if candidate and candidate.is_file(): + binary = str(candidate) + if binary is None: + raise ValueError(f'{name} is required on PATH; activate the OCI runtime Pixi environment') + return str(Path(binary).resolve()) + + +class HostRuntime(Runtime): + def __init__(self, options, cluster): + self.name = options['runtime'] + self.namespace = 'crowdb-' + cluster + self.snapshotter = options['snapshotter'] + self.binary = executable('docker' if self.name == 'docker' else 'nerdctl') + if self.name == 'docker': + self.context = output([self.binary, 'context', 'show']) + details = json.loads(output([self.binary, 'context', 'inspect', self.context]))[0] + self.address = (options.get('runtime_endpoint') or os.environ.get('DOCKER_HOST') + or details['Endpoints']['docker']['Host']) + if not self.address.startswith('unix://'): + raise ValueError('host lifecycle requires a local Unix Docker endpoint') + self.command = [self.binary, '--host', self.address] + else: + self.context = self.namespace + self.address = options.get('runtime_endpoint') or '/run/containerd/containerd.sock' + if not Path(self.address).is_absolute(): + raise ValueError('containerd address must be an absolute Unix socket path') + self.command = [self.binary, '--address', self.address, '--namespace', self.namespace, + '--snapshotter', self.snapshotter] + + def identity(self): + info = json.loads(self.invoke('info', '--format', '{{json .}}')) + daemon = info.get('ID') + if not isinstance(daemon, str) or not daemon: + raise ValueError('runtime did not report its stable daemon identity') + if self.name == 'containerd' and info.get('Driver') != self.snapshotter: + raise ValueError('containerd did not apply the requested snapshotter') + if self.name == 'docker' and output([self.binary, 'context', 'show']) != self.context: + raise ValueError('Docker context changed during host launch') + return {'selection': {'runtime': self.name, 'endpoint': self.address, 'context': self.context}, + 'daemon': daemon, 'snapshotter': self.snapshotter, 'binary': self.binary} + + def check(self, record): + if self.identity() != record['runtime']: + raise ValueError('runtime daemon, endpoint, context or snapshotter differs from the host receipt') + + def named(self, name): + identities = self.invoke('ps', '-aq').splitlines() + if len(identities) > 4096: + raise ValueError('runtime inventory exceeds the host inspection bound') + for identity in identities: + current = self.inspect(identity) + if current.get('Name', '').lstrip('/') == name: + return current + return None + + +def require_owned(current, record): + if (current.get('Config', {}).get('Labels', {}).get(OWNER) != record['owner'] + or current.get('Config', {}).get('Labels', {}).get('org.crowdb.cluster') != record['cluster'] + or current.get('Name', '').lstrip('/') != record['name']): + raise ValueError('same-name host container is foreign; refusing lifecycle effects') + if current.get('Image') != record['runtime_image']: + raise ValueError('owned host container image differs from its accepted artifact') + host = current.get('HostConfig', {}) + if host.get('NetworkMode') != 'host': + raise ValueError('owned host container no longer uses host networking') + period, quota = host.get('CpuPeriod', 0), host.get('CpuQuota', 0) + nanocpus = host.get('NanoCpus', 0) + limits = ([nanocpus / 1_000_000_000] if nanocpus > 0 else []) + if period > 0 and quota > 0: + limits.append(quota / period) + if (not limits or nanocpus < 0 or any(abs(limit - record['options']['cpus']) > 0.00001 for limit in limits) + or host.get('Memory') != record['options']['memory_mib'] * 1024 * 1024): + raise ValueError('owned host container resource limits differ from accepted inputs') + expected = {'/opt/crowdb/data': (record['options']['data_root'], True), + '/run/crowdb-node-policy.toml': (record['options']['node_policy_file'], False)} + if record['options'].get('password_file'): + expected['/run/crowdb-ssh-password'] = (record['options']['password_file'], False) + for disk in record['disks']: + expected[disk['path']] = (disk['source'], True) + mounts = {mount['Destination']: mount for mount in current.get('Mounts', [])} + for target, (source, writable) in expected.items(): + mount = mounts.get(target) + # Raw devices are represented by HostConfig.Devices rather than Mounts. + device = next((device for device in (current.get('HostConfig', {}).get('Devices') or []) + if device.get('PathInContainer') == target), None) + if device is not None: + if device.get('PathOnHost') != source or device.get('CgroupPermissions') != 'rw': + raise ValueError('owned raw-device mapping differs from its accepted input') + elif (mount is None or mount.get('Source') != source or mount.get('RW') != writable): + raise ValueError('owned host container mount differs from its accepted input') + environment = dict(item.split('=', 1) for item in current.get('Config', {}).get('Env', []) + if '=' in item) + options = record['options'] + for key, value in {'CROWDB_CLUSTER': record['cluster'], 'CROWDB_INITIAL_VOTERS': options['nodes'], + 'CROWDB_DEPLOYMENT_MODE': 'production', + 'CROWDB_PHYSICAL_HOST_ID': options['physical_host_id'], + 'CROWDB_MANAGEMENT_INTERFACE': options['interface'], + 'CROWDB_NODE_POLICY_FILE': '/run/crowdb-node-policy.toml'}.items(): + if environment.get(key) != str(value): + raise ValueError('owned host container startup policy differs from its accepted input') + declaration = [{key: disk[key] for key in ('backend', 'path', 'backing_identity', 'resource_identity')} + for disk in record['disks']] + if json.loads(environment.get('CROWDB_DISK_INPUTS', 'null')) != declaration: + raise ValueError('owned host container media declaration differs from accepted inputs') + if environment.get('CROWDB_DISCOVERY_SEEDS', '') != ','.join(options['seed']): + raise ValueError('owned host container discovery inputs differ from accepted inputs') + expected_password = '/run/crowdb-ssh-password' if options.get('password_file') else None + if environment.get('CROWDB_SSH_PASSWORD_FILE') != expected_password: + raise ValueError('owned host container credential input differs from accepted inputs') diff --git a/container/oci-image/image-build.sh b/container/oci-image/image-build.sh index 45ba5a279..8ec532687 100644 --- a/container/oci-image/image-build.sh +++ b/container/oci-image/image-build.sh @@ -11,6 +11,8 @@ while (($#)); do case "$1" in --output) output=$(realpath -m "${2:?output path required}"); shift 2 ;; --staged) staged=true; shift ;; + --load) load_image=${CROWDB_CONTAINER_IMAGE:-crowdb-node:dev}; shift ;; + --load-image) load_image=${2:?image reference required}; shift 2 ;; --platform) [[ ${2:?platform required} == linux/amd64 ]] || { echo 'Only linux/amd64 is supported' >&2; exit 1; } args+=("--platform" "$2"); shift 2 ;; @@ -21,5 +23,8 @@ while (($#)); do done pixi run --manifest-path container/oci-image/pixi.toml python "$PWD/container/oci-image/preflight.py" "${preflight_args[@]}" if [[ "${staged:-false}" != true ]]; then bash container/oci-image/stage.sh; fi -exec pixi run --manifest-path container/oci-image/pixi.toml build \ +pixi run --manifest-path container/oci-image/pixi.toml build \ --context "$PWD/target/container-runtime" --output "$output" "${args[@]}" +if [[ -n ${load_image:-} ]]; then + pixi run --manifest-path container/oci-image/pixi.toml import "$output" --image "$load_image" +fi diff --git a/container/oci-image/node-profile.toml b/container/oci-image/node-profile.toml new file mode 100644 index 000000000..76ac257b4 --- /dev/null +++ b/container/oci-image/node-profile.toml @@ -0,0 +1,149 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +# Programs and supervision defaults only. Accepted node policy and committed +# service intents supply membership, media, configuration and actual listeners. + +version = 1 +name = "node-container" +display_name = "CROWDB Node" +placement_mode = "node-policy" +s3_tenant = "default" +iceberg_catalog = "default" + +[paths] +install_root = "/opt/crowdb" +bin_root = "/opt/crowdb/bin" +template_root = "/opt/crowdb/etc/templates" +data_root = "/opt/crowdb/data" +run_root = "/opt/crowdb/run" +log_root = "/opt/crowdb/data/log" + +[logs] +max_file_bytes = 31457280 +max_files = 5 +mirror_warnings_to_stderr = true + +[[public_endpoints]] +id = "s3" +bind = "0.0.0.0" +port = 9091 + +[[public_endpoints]] +id = "iceberg" +bind = "0.0.0.0" +port = 9092 + +[[public_endpoints]] +id = "web" +bind = "0.0.0.0" +port = 9090 + +[[public_endpoints]] +id = "dataset" +bind = "0.0.0.0" +port = 9093 + +[[services]] +id = "kv" +program = "/opt/crowdb/bin/crowdb-kv-server" +args = [] +dependencies = [] +[services.probe] +kind = "http" +target = "http://127.0.0.1:10000/health" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "diskdb" +program = "/opt/crowdb/bin/crowdb-diskdb" +args = [] +dependencies = ["kv"] +[services.probe] +kind = "http" +target = "http://127.0.0.1:7101/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "diskio" +program = "/opt/crowdb/bin/crowdb-diskio" +args = [] +dependencies = ["kv", "diskdb"] +[services.probe] +kind = "rpc-ping" +target = "127.0.0.1:7200" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "chunkdb" +program = "/opt/crowdb/bin/crowdb-chunkdb" +args = [] +dependencies = ["kv", "diskdb", "diskio"] +[services.probe] +kind = "http" +target = "http://127.0.0.1:7300/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "chunk-kv" +program = "/opt/crowdb/bin/crowdb-chunk-kv-server" +args = [] +dependencies = ["kv", "chunkdb", "diskio"] +[services.probe] +kind = "http" +target = "http://127.0.0.1:7400/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "access" +program = "/opt/crowdb/bin/crowdb-access-server" +args = [] +dependencies = ["kv", "chunk-kv", "chunkdb", "diskio"] +[services.probe] +kind = "http" +target = "http://127.0.0.1:9094/_crowdb/health/ready" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 + +[[services]] +id = "web" +program = "/opt/crowdb/bin/crowdb-web" +args = [] +dependencies = [] +[services.probe] +kind = "http" +target = "http://127.0.0.1:9090/healthz" +timeout_ms = 1000 +failure_threshold = 5 +[services.restart] +max_attempts = 5 +backoff_base_ms = 250 +backoff_max_ms = 5000 diff --git a/container/oci-image/run-node.py b/container/oci-image/run-node.py index 05400482d..0c408143d 100755 --- a/container/oci-image/run-node.py +++ b/container/oci-image/run-node.py @@ -6,10 +6,12 @@ import argparse import os import platform +import re +import socket import stat from pathlib import Path -from runtime import Runtime +from runtime import Runtime, node_policy_file from storage import inputs @@ -18,9 +20,28 @@ def digest_reference(reference): return digest.startswith("sha256:") and len(digest) == 71 and all(c in "0123456789abcdef" for c in digest[7:]) -def validate(args, parser): +# Fixed in-image listeners; host networking admits only one node per host. +HOST_PORTS = (2222, 7100, 7101, 7200, 7300, 7301, 7400, 7401, + 9090, 9091, 9092, 9093, 9094, 9095, 10000, 10100) + + +def check_host_ports(ports=HOST_PORTS): + for port in ports: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + # Match the node listener: closed connections in TIME_WAIT are not + # a running port owner; an active listener still rejects this bind. + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + listener.bind(("0.0.0.0", port)) + except OSError as error: + raise ValueError(f"host node TCP port {port} is already occupied") from error + + +def validate(args, parser, check_ports=True): if (platform.system(), platform.machine()) != ("Linux", "x86_64"): parser.error("Linux amd64 host required; macOS VM backend is reserved") + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,47}", args.cluster): + parser.error("cluster must be 1–48 ASCII letters, digits, dots, underscores or hyphens") if not digest_reference(args.image): parser.error("image must use an immutable SHA256 digest") if not Path("/sys/fs/cgroup/cgroup.controllers").is_file(): @@ -50,6 +71,7 @@ def validate(args, parser): elif not (args.data_root / "ssh/authorized_keys").is_file(): parser.error("production requires a password file or preinstalled authorized_keys") try: + node_policy_file(args) inputs(args) except (OSError, ValueError) as error: parser.error(str(error)) @@ -58,6 +80,12 @@ def validate(args, parser): if args.network == "host": if args.ui_port is not None: parser.error("host networking uses the node's port 9090 directly") + try: + socket.if_nametoindex(args.interface) + if check_ports: + check_host_ports() + except (OSError, ValueError) as error: + parser.error(str(error)) elif args.runtime != "docker": parser.error("initial containerd production support requires host networking") elif not args.ui_port or not 1 <= args.ui_port <= 65535: @@ -68,7 +96,6 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--runtime", choices=["docker", "containerd"], default="docker") parser.add_argument("--address", default="/run/containerd/containerd.sock") - parser.add_argument("--namespace", default="crowdb") parser.add_argument("--snapshotter", choices=["overlayfs", "native"], default="overlayfs") parser.add_argument("--image", required=True) parser.add_argument("--name", required=True) @@ -83,14 +110,17 @@ def main(): parser.add_argument("--memory-mib", type=int, required=True) parser.add_argument("--ui-port", type=int) parser.add_argument("--password-file", type=Path) + parser.add_argument("--node-policy-file", type=Path, + help="Bounded TOML node inputs, mounted read-only and sealed by the monitor") parser.add_argument("--device", type=Path, action="append", default=[]) parser.add_argument("--disk-file", type=Path, action="append", default=[], help="Existing disk file, mapped beneath /opt/crowdb/data/disks by filename") parser.add_argument("--seed", action="append", default=[]) - parser.add_argument("--startup-mode", choices=["manual", "single"], default="manual") + parser.add_argument("--cluster", required=True) + parser.add_argument("--initial-voters", type=int, choices=(1, 3), default=3) args = parser.parse_args() validate(args, parser) - runtime = Runtime(args.runtime, args.address, args.namespace, args.snapshotter) + runtime = Runtime(args.runtime, args.address, "crowdb-" + args.cluster, args.snapshotter) image = runtime.image(args.image) if (image["Os"], image["Architecture"]) != ("linux", "amd64"): parser.error("this packaging supports Linux amd64") diff --git a/container/oci-image/runtime.py b/container/oci-image/runtime.py index 1d98f2c3a..fe6b8b241 100644 --- a/container/oci-image/runtime.py +++ b/container/oci-image/runtime.py @@ -6,7 +6,10 @@ import math import os import subprocess +import stat +import tomllib import uuid +from pathlib import Path from builder import tool from storage import inputs, record_inputs, runtime_arguments, validate_retained_inputs @@ -14,6 +17,41 @@ OWNER = "org.crowdb.launcher" +def node_policy_file(args): + """Check the bounded startup file; the monitor validates and seals its semantics.""" + path = getattr(args, "node_policy_file", None) + if path is None: + return None + path = Path(path) + if (not path.is_absolute() or any(part.is_symlink() for part in (path, *path.parents)) + or any(character in str(path) for character in (",", "\n", "\r"))): + raise ValueError("node policy must be an absolute non-symlink path without mount delimiters") + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC | os.O_NONBLOCK) + try: + metadata = os.fstat(descriptor) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > 32768: + raise ValueError("node policy must be a regular TOML file of at most 32768 bytes") + if not (metadata.st_mode & 0o004 + or metadata.st_uid == 10001 and metadata.st_mode & 0o400 + or metadata.st_gid == 10001 and metadata.st_mode & 0o040): + raise ValueError("node policy must be readable by container uid 10001") + content = b"" + while len(content) < 32769: + part = os.read(descriptor, 32769 - len(content)) + if not part: + break + content += part + finally: + os.close(descriptor) + if len(content) > 32768: + raise ValueError("node policy exceeds 32768 bytes") + try: + tomllib.loads(content.decode("utf-8")) + except (UnicodeError, tomllib.TOMLDecodeError): + raise ValueError("node policy is not valid UTF-8 TOML") from None + return path + + class Runtime: def __init__(self, name, address="/run/containerd/containerd.sock", namespace="crowdb", snapshotter="overlayfs"): self.name = name @@ -28,7 +66,7 @@ def invoke(self, *args): result = subprocess.run([*self.command, *map(str, args)], check=False, text=True, capture_output=True, timeout=60) if result.returncode: raise RuntimeError(f"{self.name} {args[0]} failed: {result.stderr}") - return result.stdout + return result.stdout + result.stderr if args and args[0] == 'logs' else result.stdout def inspect(self, name, image=False): return json.loads(self.invoke(*(["image", "inspect"] if image else ["inspect"]), name))[0] @@ -66,18 +104,19 @@ def verify(self, name, cpus, memory_mib, host_network, cpu_shares=None): raise RuntimeError("container does not share this host network namespace") def create(self, args): + policy = node_policy_file(args) disks = inputs(args) - if getattr(args, "startup_mode", "manual") == "single" and disks: - raise ValueError("single startup uses fixed preview disks; supplied media require manual mode") validate_retained_inputs(args.data_root, disks) - token = str(uuid.uuid4()) + token = getattr(args, "owner_token", None) or str(uuid.uuid4()) shares = getattr(args, "cpu_shares", None) cpu_args = ["--cpu-shares", str(shares)] if shares is not None else ["--cpus", str(args.cpus)] command = ["run", "-d", "--name", args.name, "--label", f"{OWNER}={token}", + "--label", f"org.crowdb.cluster={args.cluster}", "--network", args.network, *cpu_args, "--memory", f"{args.memory_mib}m", "--restart", "unless-stopped", "--cgroupns", "private", "--mount", f"type=bind,source={args.data_root},target=/opt/crowdb/data", - "-e", f"CROWDB_STARTUP_MODE={args.startup_mode}", + "-e", f"CROWDB_CLUSTER={args.cluster}", + "-e", f"CROWDB_INITIAL_VOTERS={args.initial_voters}", "-e", "CROWDB_DEPLOYMENT_MODE=production", "-e", f"CROWDB_PHYSICAL_HOST_ID={args.physical_host_id}", "-e", f"CROWDB_MANAGEMENT_INTERFACE={args.interface}"] @@ -87,6 +126,9 @@ def create(self, args): if args.password_file: command += ["--mount", f"type=bind,source={args.password_file},target=/run/crowdb-ssh-password,readonly", "-e", "CROWDB_SSH_PASSWORD_FILE=/run/crowdb-ssh-password"] + if policy is not None: + command += ["--mount", f"type=bind,source={policy},target=/run/crowdb-node-policy.toml,readonly", + "-e", "CROWDB_NODE_POLICY_FILE=/run/crowdb-node-policy.toml"] command += runtime_arguments(disks) if args.seed: command += ["-e", "CROWDB_DISCOVERY_SEEDS=" + ",".join(args.seed)] @@ -99,13 +141,11 @@ def create(self, args): kind = "-f" if disk["backend"] == "file" else "-b" self.invoke("exec", "--user", "10001:10001", container, "sh", "-c", 'test "$1" "$2" && test -r "$2" && test -w "$2"', "sh", kind, disk["path"]) - # Single bootstrap requires a pristine root before its manifest is - # accepted. An empty external-mapping receipt must not race it. - if args.startup_mode != "single": - record_inputs(args.data_root, disks) + record_inputs(args.data_root, disks) return container except BaseException: - # A failed create can leave a container; only this operation owns it. + # A failed verification must retain owned runtime and storage state + # for diagnosis/resume. An independent explicit cleanup owns removal. try: current = self.inspect(args.name) if current.get("Config", {}).get("Labels", {}).get(OWNER) == token: @@ -114,7 +154,7 @@ def create(self, args): print(self.invoke("logs", args.name), flush=True) except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as log_error: print(f"Startup failure logs unavailable: {log_error}", flush=True) - self.invoke("rm", "-f", args.name) + print(f"Owned failed container retained: {args.name}", flush=True) except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as cleanup_error: print(f"Inspect/cleanup after launch failure: {cleanup_error}", flush=True) raise diff --git a/container/oci-image/stage.sh b/container/oci-image/stage.sh index f4a9d1965..56c338ca9 100644 --- a/container/oci-image/stage.sh +++ b/container/oci-image/stage.sh @@ -32,9 +32,9 @@ staging=$(mktemp -d "$PWD/target/container-runtime.XXXXXX") trap 'rm -rf "$staging"' EXIT bash container/oci-image/collect-libs.sh "$staging" cp -a app/crowdb-web/ui/dist "$staging/ui" -cp -a container/single-node-container/templates "$staging/templates" cp container/oci-image/Dockerfile "$staging/" -cp container/single-node-container/{profile.toml,entrypoint.sh} "$staging/" +cp container/oci-image/node-profile.toml "$staging/profile.toml" +cp container/single-node-container/entrypoint.sh "$staging/" git rev-parse HEAD > "$staging/SOURCE_REVISION" printf '%s\n' "$source_tree" > "$staging/SOURCE_TREE" cp VERSION "$staging/VERSION" diff --git a/container/oci-image/tests/docker_local_test.py b/container/oci-image/tests/docker_local_test.py new file mode 100644 index 000000000..330a9cde9 --- /dev/null +++ b/container/oci-image/tests/docker_local_test.py @@ -0,0 +1,181 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Scoped cleanup preserves data and rejects changed resource ownership.""" + +import json +import os +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from docker_local_cleanup import clean, image_garbage, selected +from docker_local_launch import ensure, verify_container +from docker_local_state import CLUSTER, OWNER, Docker, load, seal + + +class DockerLocalTest(unittest.TestCase): + def setUp(self): + self.record = {"version": 1, "cluster": "first", "owner": str(uuid.uuid4()), + "runtime": {"daemon": "owned", "context": "default", "endpoint": "unix:///owned"}, + "image_id": "sha256:retained", "image": "crowdb-node:dev", "nodes": 1, + "port_base": 9090, "cpus": 1, "memory_mib": 1536} + self.labels = {CLUSTER: "first", OWNER: self.record["owner"]} + self.docker = MagicMock() + self.docker.identity.return_value = self.record["runtime"] + self.docker.owned.return_value = [] + + def network(self, attached=False): + return {"Id": "bridge-id", "Name": "crowdb-first", "Labels": self.labels, + "Containers": {"foreign": {}} if attached else {}} + + def test_clean_removes_owned_empty_bridge_and_preserves_data(self): + network = self.network() + self.docker.owned.side_effect = lambda record, kind: [network] if kind == "network" else [] + self.docker.call.return_value = json.dumps([network]) + clean(self.docker, self.record) + self.docker.call.assert_any_call("network", "rm", "bridge-id") + self.assertFalse(any(call.args[:2] == ("volume", "rm") for call in self.docker.call.call_args_list)) + + def test_attached_or_relabelled_bridge_is_not_removed(self): + for attached in (True, False): + with self.subTest(attached=attached): + self.docker.reset_mock() + old = self.network() + observed = self.network(attached) + if not attached: + observed["Labels"] = {CLUSTER: "other", OWNER: "foreign"} + self.docker.owned.side_effect = lambda record, kind: [old] if kind == "network" else [] + self.docker.call.return_value = json.dumps([observed]) + with self.assertRaises(ValueError): + clean(self.docker, self.record) + self.assertFalse(any(call.args[:2] == ("network", "rm") for call in self.docker.call.call_args_list)) + + def test_cleanup_inventory_does_not_inspect_unrelated_disappearing_containers(self): + docker = Docker() + owned = {"Id": "owned-id", "Name": "/crowdb-first-1", "Config": {"Labels": self.labels}} + + def call(*args): + if args[:2] == ("ps", "-aq"): + scoped = (f"label={CLUSTER}=first" in args + and f"label={OWNER}={self.record['owner']}" in args) + return "owned-id" if scoped else "owned-id\nretired-unrelated-id" + if args[:2] == ("container", "inspect"): + if "retired-unrelated-id" in args: + raise RuntimeError("No such container: retired-unrelated-id") + return json.dumps([owned]) + raise AssertionError(args) + + with patch.object(docker, "call", side_effect=call): + self.assertEqual(docker.owned(self.record, "container"), [owned]) + + def test_unknown_resource_names_are_not_adopted_from_copied_labels(self): + docker = Docker() + owned = self.network() + foreign = dict(owned, Name="other-network", Id="foreign") + with patch.object(docker, "objects", return_value=[owned, foreign]): + self.assertEqual(docker.owned(self.record, "network"), [owned]) + + def test_changed_daemon_rejects_before_cleanup_effects(self): + self.docker.identity.return_value = {"daemon": "different"} + with self.assertRaisesRegex(ValueError, "endpoint/context"): + clean(self.docker, self.record) + self.docker.call.assert_not_called() + + def test_delete_data_rejects_a_volume_referenced_by_another_container(self): + volume = {"Name": "crowdb-first-1-data", "Labels": self.labels} + self.docker.owned.side_effect = lambda record, kind: [volume] if kind == "volume" else [] + self.docker.call.side_effect = lambda *args: json.dumps([volume]) if args[0] == "volume" else "foreign-id" + with self.assertRaisesRegex(ValueError, "another container"): + clean(self.docker, self.record, True) + self.assertFalse(any(call.args[:2] == ("volume", "rm") for call in self.docker.call.call_args_list)) + + def test_create_cannot_adopt_a_foreign_named_network(self): + network = self.network() + network["Labels"] = {} + self.docker.objects.return_value = [network] + with self.assertRaisesRegex(ValueError, "foreign"): + ensure(self.docker, self.record, "network", network["Name"]) + self.docker.call.assert_not_called() + + def test_changed_container_inputs_are_rejected_before_resume(self): + record = dict(self.record, nodes=1, port_base=9090, cpus=1, memory_mib=1536) + environment = ["CROWDB_CLUSTER=first", "CROWDB_INITIAL_VOTERS=1"] + for offset, name in enumerate(("S3", "ICEBERG", "DATASET"), start=1): + environment.append(f"CROWDB_{name}_PUBLIC_URI=http://127.0.0.1:{9090+offset}") + item = {"Name": "/crowdb-first-1", "Image": record["image_id"], "Config": {"Labels": self.labels, "Env": environment}, + "HostConfig": {"NanoCpus": 1_000_000_000, "Memory": 1536 * 1024 * 1024, + "NetworkMode": "crowdb-first", "PortBindings": { + f"{port}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(port)}] + for port in range(9090, 9094)}}, + "Mounts": [{"Destination": "/opt/crowdb/data", "Name": "owned-volume", "Type": "volume", "RW": True}]} + verify_container(item, record, 1, "owned-volume", "crowdb-first") + for field, value in (("Image", "foreign"), ("Mounts", []), ("HostConfig", {})): + with self.subTest(field=field), self.assertRaises((ValueError, KeyError)): + verify_container(dict(item, **{field: value}), record, 1, "owned-volume", "crowdb-first") + + def test_receipts_are_immutable_scoped_and_reject_symlinks(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + seal(self.record, directory) + self.assertEqual(load("first", directory), self.record) + self.assertEqual(selected("first", False, directory), [self.record]) + other = dict(self.record, cluster="second", owner=str(uuid.uuid4())) + seal(other, directory) + self.assertEqual(len(selected(None, True, directory)), 2) + with self.assertRaises(ValueError): + seal(dict(self.record, owner=str(uuid.uuid4())), directory) + (directory / "foreign.json").symlink_to(directory / "first.json") + with self.assertRaisesRegex(ValueError, "symlink"): + load("foreign", directory) + + def test_delete_data_retires_only_matching_optional_application_binding(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) / "oci-clusters" + path = Path(temporary) / "clusters/first/target.json" + path.parent.mkdir(parents=True) + target = {"version": 1, "name": "first", "runtime": {"owner": self.record["owner"], + "selection": {"runtime": "docker", "context": "default", "endpoint": "unix:///owned"}}} + path.write_text(json.dumps(target)) + os.chmod(path, 0o600) + seal(self.record, directory) + clean(self.docker, self.record, False, directory) + self.assertTrue(path.exists()) + clean(self.docker, self.record, True, directory) + self.assertFalse(path.exists()) + self.assertIsNone(load("first", directory)) + + def test_foreign_application_binding_prevents_data_deletion_before_effects(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) / "oci-clusters" + path = Path(temporary) / "clusters/first/target.json" + path.parent.mkdir(parents=True) + path.write_text(json.dumps({"version": 1, "name": "first", "runtime": None})) + os.chmod(path, 0o600) + with self.assertRaisesRegex(ValueError, "differs"): + clean(self.docker, self.record, True, directory) + self.docker.call.assert_not_called() + + def test_garbage_keeps_retained_and_stopped_container_images(self): + def call(*args): + if args[:2] == ("image", "ls"): + return "retained\nforeign-used\ngarbage" + if args[:2] == ("image", "inspect"): + return json.dumps([{"Id": "sha256:" + args[2], "RepoTags": [], "RepoDigests": [], "Config": {"Labels": {"org.opencontainers.image.source": "https://github.com/buzzcrow/crowdb"}}}]) + if args[:2] == ("ps", "-aq"): + return "stopped-foreign" if args[-1] == "ancestor=sha256:foreign-used" else "" + return "" + self.docker.call.side_effect = call + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + seal(self.record, directory) + image_garbage(self.docker, directory) + removals = [call.args for call in self.docker.call.call_args_list if call.args[:2] == ("image", "rm")] + self.assertEqual(removals, [("image", "rm", "sha256:garbage")]) + + +if __name__ == "__main__": + unittest.main() diff --git a/container/oci-image/tests/host_cleanup_test.py b/container/oci-image/tests/host_cleanup_test.py new file mode 100644 index 000000000..8290bfda2 --- /dev/null +++ b/container/oci-image/tests/host_cleanup_test.py @@ -0,0 +1,51 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Host cleanup stays independent of the CLI and cannot remove foreign state.""" + +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import host_cli_test as fixture +from host_cleanup import clean, selected + + +class HostCleanupTest(unittest.TestCase): + setUp = fixture.HostCliTest.setUp + def test_exact_owner_is_stopped_then_removed_without_data_effects(self): + self.runtime.current = fixture.container(self.record) + with patch('host_cleanup.HostRuntime', return_value=self.runtime): + clean(self.record) + self.assertEqual(self.runtime.effects, [('stop', 'owned-container'), ('rm', 'owned-container')]) + + def test_foreign_owner_or_daemon_never_receives_effects(self): + for changed_daemon in (False, True): + self.runtime.current = fixture.container(self.record) + if changed_daemon: self.runtime.daemon['daemon'] = 'foreign' + else: self.runtime.current['Config']['Labels']['org.crowdb.launcher'] = 'foreign' + with patch('host_cleanup.HostRuntime', return_value=self.runtime), self.assertRaises(ValueError): + clean(self.record) + self.assertEqual(self.runtime.effects, []) + + def test_private_receipt_binding_and_symlink_fences(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + path = root/'persistent/oci-hosts/test.json' + path.parent.mkdir(parents=True) + path.write_text(json.dumps(self.record)) + path.chmod(0o600) + self.assertEqual(selected('test', root), self.record) + binding = root/'persistent/clusters/test/target.json' + binding.parent.mkdir(parents=True) + binding.write_text(json.dumps(dict(version=1, name='test', management_endpoints=self.record['origins'], + runtime=dict(selection=self.record['runtime']['selection'], owner='foreign')))) + binding.chmod(0o600) + with self.assertRaisesRegex(ValueError, 'target conflicts'): selected('test', root) + binding.unlink() + path.chmod(0o644) + with self.assertRaisesRegex(ValueError, 'private regular'): selected('test', root) + path.unlink() + path.symlink_to('/missing') + with self.assertRaisesRegex(ValueError, 'symlinks'): selected('test', root) diff --git a/container/oci-image/tests/host_cli_test.py b/container/oci-image/tests/host_cli_test.py new file mode 100644 index 000000000..a897e8955 --- /dev/null +++ b/container/oci-image/tests/host_cli_test.py @@ -0,0 +1,189 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Interrupted host launches resume only their sealed runtime and node inputs.""" + +import copy +import importlib.util +import json +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +OCI = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(OCI)) +spec = importlib.util.spec_from_file_location('run_node', OCI / 'run-node.py') +run_node = importlib.util.module_from_spec(spec) +sys.modules['run_node'] = run_node +spec.loader.exec_module(run_node) +import host_lifecycle as lifecycle +from runtime import OWNER +from host_runtime import require_owned + +DIGEST = 'sha256:' + 'a' * 64 +OPTIONS = dict(runtime='containerd', network='host', snapshotter='native', runtime_endpoint=None, + image=DIGEST, nodes=3, cpus=2, memory_mib=2048, data_root='/data', + node_policy_file='/policy', password_file=None, device=[], disk_file=[], + seed=[], physical_host_id='host-1', interface='eth0', management_address='10.0.0.1') +IDENTITY = dict(selection=dict(runtime='containerd', endpoint='/run/containerd/containerd.sock', + context='crowdb-test'), daemon='daemon-1', snapshotter='native', binary='/nerdctl') + + +def container(record): + values = dict(CROWDB_CLUSTER='test', CROWDB_INITIAL_VOTERS='3', CROWDB_DEPLOYMENT_MODE='production', + CROWDB_PHYSICAL_HOST_ID='host-1', CROWDB_MANAGEMENT_INTERFACE='eth0', + CROWDB_NODE_POLICY_FILE='/run/crowdb-node-policy.toml', CROWDB_DISK_INPUTS='[]') + return dict(Id='owned-container', Name='/crowdb-test-host', Image=DIGEST, + Config=dict(Labels={OWNER: record['owner'], 'org.crowdb.cluster': 'test'}, + Env=[key + '=' + value for key, value in values.items()]), + State=dict(Running=True), HostConfig=dict(NetworkMode='host', CpuQuota=200000, + CpuPeriod=100000, Memory=2048*1024*1024), Mounts=[ + dict(Destination='/opt/crowdb/data', Source='/data', RW=True), + dict(Destination='/run/crowdb-node-policy.toml', Source='/policy', RW=False)]) + + +class TestRuntime: + name = 'containerd' + def __init__(self): + self.current = None + self.daemon = copy.deepcopy(IDENTITY) + self.effects = [] + self.lose_reply = False + def identity(self): return copy.deepcopy(self.daemon) + def named(self, name): return self.current + def inspect(self, name, image=False): + if image: + return dict(Id=DIGEST, Os='linux', Architecture='amd64', Config=dict(Labels={ + 'org.opencontainers.image.source': 'https://github.com/buzzcrow/crowdb'})) + return self.current + def check(self, record): + if self.daemon != record['runtime']: raise ValueError('runtime identity changed') + def verify(self, *args): pass + def invoke(self, action, identity): + self.effects.append((action, identity)) + self.current['State']['Running'] = action == 'start' + def create(self, args): + self.effects.append(('create', args.name)) + self.current = container(dict(owner=args.owner_token)) + if self.lose_reply: raise RuntimeError('lost runtime reply') + + +class HostCliTest(unittest.TestCase): + def setUp(self): + self.runtime = TestRuntime() + self.validated = ([], dict(node_policy='policy-hash', password=None), ['http://10.0.0.1:9090']) + self.patches = [patch('host_lifecycle.HostRuntime', return_value=self.runtime), + patch('host_lifecycle.options', side_effect=lambda request, previous: copy.deepcopy(OPTIONS)), + patch('host_lifecycle.validate', side_effect=lambda record, running: copy.deepcopy(self.validated))] + for item in self.patches: item.start() + self.addCleanup(lambda: [item.stop() for item in reversed(self.patches)]) + self.request = dict(cluster='test', options={}) + self.record = lifecycle.prepare(self.request) + self.request['record'] = self.record + + def test_unsealed_run_has_no_effects(self): + with self.assertRaisesRegex(ValueError, 'seal accepted'): + lifecycle.run(dict(cluster='test', options={})) + self.assertEqual(self.runtime.effects, []) + + def test_lost_create_reply_resumes_same_owned_container(self): + self.runtime.lose_reply = True + with self.assertRaisesRegex(RuntimeError, 'lost runtime reply'): + lifecycle.run(self.request) + self.assertEqual(lifecycle.run(self.request)['container_id'], 'owned-container') + self.assertEqual(self.runtime.effects, [('create', 'crowdb-test-host')]) + + def test_daemon_change_blocks_start_and_stop(self): + self.runtime.current = container(self.record) + self.runtime.daemon['daemon'] = 'foreign' + for action in (lifecycle.run, lifecycle.stop): + with self.subTest(action=action.__name__), self.assertRaisesRegex(ValueError, 'runtime identity'): + action(self.request) + self.assertEqual(self.runtime.effects, []) + + def test_changed_policy_credentials_or_media_block_effects(self): + for index, value in [(0, [dict(path='changed')]), (1, dict(node_policy='changed', password=None)), + (1, dict(node_policy='policy-hash', password='changed'))]: + with self.subTest(value=value): + current = list(self.validated) + current[index] = value + with patch('host_lifecycle.validate', return_value=tuple(current)): + with self.assertRaisesRegex(ValueError, 'differ from accepted'): + lifecycle.run(self.request) + self.assertEqual(self.runtime.effects, []) + + def test_foreign_or_changed_stopped_container_is_never_started(self): + for kind in ('owner', 'image', 'mount', 'memory', 'environment'): + with self.subTest(kind=kind): + current = container(self.record) + current['State']['Running'] = False + if kind == 'owner': current['Config']['Labels'][OWNER] = 'foreign' + if kind == 'image': current['Image'] = 'foreign' + if kind == 'mount': current['Mounts'][0]['Source'] = '/foreign' + if kind == 'memory': current['HostConfig']['Memory'] = 1 + if kind == 'environment': current['Config']['Env'][0] = 'CROWDB_CLUSTER=foreign' + self.runtime.current = current + for action in (lifecycle.run, lifecycle.stop): + with self.assertRaises(ValueError): action(self.request) + self.assertEqual(self.runtime.effects, []) + + def test_only_verified_running_owner_skips_port_check(self): + for running in (True, False): + self.runtime.current = container(self.record) + self.runtime.current['State']['Running'] = running + with patch('host_lifecycle.validate', return_value=self.validated) as validate: + lifecycle.prepare(self.request) + self.assertEqual(validate.call_args.args[1], running) + self.runtime.current['Config']['Labels'][OWNER] = 'foreign' + with patch('host_lifecycle.validate') as validate, self.assertRaises(ValueError): + lifecycle.prepare(self.request) + validate.assert_not_called() + + def test_stop_retains_container_and_inputs(self): + self.runtime.current = container(self.record) + saved = copy.deepcopy(self.record) + self.assertEqual(lifecycle.stop(self.request), dict(data_preserved=True)) + self.assertEqual(self.runtime.effects, [('stop', 'owned-container')]) + self.assertEqual(self.record, saved) + + def test_diskless_nerdctl_null_device_list_preserves_owned_mount_checks(self): + self.runtime.current = container(self.record) + self.runtime.current['HostConfig']['Devices'] = None + self.assertEqual(lifecycle.run(self.request)['container_id'], 'owned-container') + self.runtime.current['Mounts'][0]['Source'] = '/foreign' + with self.assertRaisesRegex(ValueError, 'mount differs'): + lifecycle.run(self.request) + self.assertEqual(self.runtime.effects, []) + + def test_docker_nanocpus_and_containerd_quota_limits_must_match_even_when_stopped(self): + for running in (True, False): + current = container(self.record) + current['State']['Running'] = running + current['HostConfig'].update(NanoCpus=2_000_000_000,CpuPeriod=0,CpuQuota=0) + require_owned(current,self.record) + current['HostConfig']['NanoCpus'] = 1_000_000_000 + with self.assertRaisesRegex(ValueError,'resource limits'): + require_owned(current,self.record) + current['HostConfig'].update(NanoCpus=2_000_000_000,CpuPeriod=100000,CpuQuota=100000) + with self.assertRaisesRegex(ValueError,'resource limits'): + require_owned(current,self.record) + current['HostConfig'].update(NanoCpus=0,CpuQuota=200000) + require_owned(current,self.record) + + def test_raw_device_requires_exact_source_target_and_permissions(self): + record = copy.deepcopy(self.record) + record['disks'] = [dict(backend='block-device', source='/dev/vdf', path='/opt/crowdb/data/disks/vdf', + backing_identity='device:1', resource_identity='device:1')] + current = container(record) + declaration = [{key: value for key, value in record['disks'][0].items() if key != 'source'}] + current['Config']['Env'][-1] = 'CROWDB_DISK_INPUTS=' + json.dumps(declaration) + device = dict(PathOnHost='/dev/vdf', PathInContainer='/opt/crowdb/data/disks/vdf', CgroupPermissions='rw') + current['HostConfig']['Devices'] = [device] + require_owned(current, record) + for key, value in [('PathOnHost', '/dev/vdg'), ('CgroupPermissions', 'rwm')]: + changed = copy.deepcopy(current) + changed['HostConfig']['Devices'][0][key] = value + with self.assertRaises(ValueError): require_owned(changed, record) + + +if __name__ == '__main__': unittest.main() diff --git a/container/oci-image/tests/host_inputs_test.py b/container/oci-image/tests/host_inputs_test.py new file mode 100644 index 000000000..05400046f --- /dev/null +++ b/container/oci-image/tests/host_inputs_test.py @@ -0,0 +1,48 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Host listeners are validated before creating a node container.""" + +import importlib.util +from pathlib import Path +import socket +import unittest +from types import SimpleNamespace +from unittest.mock import patch +from runtime import Runtime + +spec = importlib.util.spec_from_file_location("run_node", Path(__file__).resolve().parents[1] / "run-node.py") +run_node = importlib.util.module_from_spec(spec) +spec.loader.exec_module(run_node) + + +class HostInputsTest(unittest.TestCase): + def test_runtime_logs_preserve_stderr_without_polluting_machine_readable_inspection(self): + with patch('runtime.subprocess.run', return_value=SimpleNamespace( + returncode=0,stdout='{"status":"running"}',stderr='startup failed on stderr')): + runtime = Runtime('docker') + self.assertEqual(runtime.invoke('inspect','owned'),'{"status":"running"}') + self.assertEqual(runtime.invoke('logs','owned'),'{"status":"running"}startup failed on stderr') + + def test_closed_server_time_wait_does_not_prevent_immediate_recreation(self): + with socket.socket() as listener: + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(('127.0.0.1', 0)) + port = listener.getsockname()[1] + listener.listen() + with socket.create_connection(('127.0.0.1', port)) as client: + accepted, _ = listener.accept() + accepted.close() + self.assertEqual(client.recv(1), b'') + with socket.socket() as without_reuse: + with self.assertRaises(OSError): + without_reuse.bind(('0.0.0.0', port)) + run_node.check_host_ports([port]) + + def test_occupied_host_listener_rejects_before_runtime_launch(self): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: + listener.bind(("127.0.0.1", 0)) + port = listener.getsockname()[1] + listener.listen() + with self.assertRaisesRegex(ValueError, f"port {port} is already occupied"): + run_node.check_host_ports([port]) + self.assertEqual(listener.getsockname()[1], port) diff --git a/container/oci-image/tests/node_policy_file_test.py b/container/oci-image/tests/node_policy_file_test.py new file mode 100644 index 000000000..d05d8044d --- /dev/null +++ b/container/oci-image/tests/node_policy_file_test.py @@ -0,0 +1,67 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from runtime import Runtime, node_policy_file + + +class NodePolicyFileTest(unittest.TestCase): + def test_readonly_policy_is_forwarded_with_the_same_contract_on_both_runtimes(self): + for engine in ("docker", "containerd"): + with self.subTest(engine=engine), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + policy = root / "policy.toml" + content = 'version = 1\nmode = "production"\ninitial_voters = 3\n' + policy.write_text(content) + data = root / "data" + data.mkdir() + args = SimpleNamespace(node_policy_file=policy, data_root=data, + disk_file=[], device=[], name="node", cpus=1, memory_mib=512, + network="host", cluster="test", initial_voters=3, + physical_host_id="host", interface="eth0", password_file=None, + seed=[], ui_port=None, image="sha256:test") + with patch("runtime.tool", return_value="nerdctl"): + runtime = Runtime(engine) + with patch.object(runtime, "invoke", return_value="node") as invoke, \ + patch.object(runtime, "verify"): + runtime.create(args) + command = invoke.call_args_list[0].args + self.assertIn(f"type=bind,source={policy},target=/run/crowdb-node-policy.toml,readonly", command) + self.assertIn("CROWDB_NODE_POLICY_FILE=/run/crowdb-node-policy.toml", command) + self.assertEqual(policy.read_text(), content) + + def test_invalid_file_rejects_before_any_runtime_access(self): + with tempfile.TemporaryDirectory() as directory: + policy = Path(directory) / "policy.toml" + policy.write_bytes(b'password = "must-not-appear-in-the-error\xff') + args = SimpleNamespace(node_policy_file=policy) + runtime = Runtime("docker") + with patch.object(runtime, "invoke") as invoke: + with self.assertRaisesRegex(ValueError, "not valid UTF-8 TOML") as caught: + runtime.create(args) + self.assertNotIn("must-not-appear", str(caught.exception)) + invoke.assert_not_called() + + def test_unreadable_oversized_symlink_and_nonregular_files_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + policy = root / "policy.toml" + policy.write_text('version = 1\n') + policy.chmod(0o000) + with self.assertRaises((PermissionError, ValueError)): + node_policy_file(SimpleNamespace(node_policy_file=policy)) + policy.chmod(0o644) + policy.write_bytes(b'x' * 32769) + with self.assertRaisesRegex(ValueError, "32768"): + node_policy_file(SimpleNamespace(node_policy_file=policy)) + alias = root / "alias" + alias.symlink_to(policy) + with self.assertRaisesRegex(ValueError, "non-symlink"): + node_policy_file(SimpleNamespace(node_policy_file=alias)) + with self.assertRaisesRegex(ValueError, "regular TOML"): + node_policy_file(SimpleNamespace(node_policy_file=root)) diff --git a/container/oci-image/tests/storage_test.py b/container/oci-image/tests/storage_test.py index 30cbd1f59..bd083ff12 100644 --- a/container/oci-image/tests/storage_test.py +++ b/container/oci-image/tests/storage_test.py @@ -18,34 +18,34 @@ class StorageTest(unittest.TestCase): - def test_single_startup_keeps_the_initial_root_clean_and_rejects_external_media(self): - for supplied in (False, True): - with self.subTest(supplied=supplied), tempfile.TemporaryDirectory() as directory: + def test_one_and_three_voters_use_normal_policy_and_preserve_supplied_media(self): + for voters in (1, 3): + with self.subTest(voters=voters), tempfile.TemporaryDirectory() as directory: root = Path(directory) / "node" root.mkdir() source = Path(directory) / "external.img" source.write_bytes(b"retained") - args = SimpleNamespace(data_root=root, disk_file=[source] if supplied else [], device=[], + args = SimpleNamespace(data_root=root, disk_file=[source], device=[], name="node", cpus=1, memory_mib=512, network="host", - startup_mode="single", physical_host_id="host", interface="eth0", + cluster="test-cluster", initial_voters=voters, + physical_host_id="host", interface="eth0", password_file=None, seed=[], ui_port=None, image="image") runtime = Runtime("docker") with patch.object(runtime, "invoke", return_value="container") as invoke, patch.object(runtime, "verify"): - if supplied: - with self.assertRaisesRegex(ValueError, "single.*supplied"): - runtime.create(args) - invoke.assert_not_called() - else: - self.assertEqual(runtime.create(args), "container") - self.assertEqual(list(root.iterdir()), [], "single bootstrap owns its initial data directory") + self.assertEqual(runtime.create(args), "container") + launch = invoke.call_args_list[0].args + self.assertIn("CROWDB_CLUSTER=test-cluster", launch) + self.assertIn(f"CROWDB_INITIAL_VOTERS={voters}", launch) + self.assertFalse(any("STARTUP_MODE" in arg for arg in launch)) + self.assertTrue((root / "container-storage.json").is_file()) self.assertEqual(source.read_bytes(), b"retained") - def test_failed_start_preserves_logs_before_owned_cleanup(self): + def test_failed_start_preserves_logs_and_runtime_until_explicit_cleanup(self): for owned in (True, False): with self.subTest(owned=owned), tempfile.TemporaryDirectory() as directory: args = SimpleNamespace(data_root=Path(directory), disk_file=[], device=[], name="node", cpus=1, memory_mib=512, network="host", - startup_mode="single", physical_host_id="host", interface="eth0", + cluster="test-cluster", initial_voters=1, physical_host_id="host", interface="eth0", password_file=None, seed=[], ui_port=None, image="image") calls = [] token = None @@ -65,8 +65,8 @@ def invoke(*command): patch("builtins.print") as output: with self.assertRaisesRegex(RuntimeError, "stopped"): runtime.create(args) - self.assertEqual(calls[1:], [("logs", "node"), ("rm", "-f", "node")] if owned else []) - self.assertEqual(output.call_count, 2 * int(owned)) + self.assertEqual(calls[1:], [("logs", "node")] if owned else []) + self.assertEqual(output.call_count, 3 * int(owned)) def test_replacement_preserves_original_mappings_before_runtime_launch(self): with tempfile.TemporaryDirectory() as directory: @@ -140,7 +140,7 @@ def test_both_runtimes_receive_common_paths_and_retained_declaration(self): source.write_bytes(b"preserved") args = SimpleNamespace(disk_file=[source], device=[], data_root=root, name="node", cpus=1, memory_mib=512, - network="host", startup_mode="manual", + network="host", cluster="test-cluster", initial_voters=3, physical_host_id="host", interface="eth0", password_file=None, seed=[], ui_port=None, image="image@sha256:test") diff --git a/container/single-node-container/README.md b/container/single-node-container/README.md index c8214142f..36719546b 100644 --- a/container/single-node-container/README.md +++ b/container/single-node-container/README.md @@ -1,255 +1,69 @@ -# Node container deployment and single-node development - -This page describes building and running CROWDB from source on a Linux amd64 -development or CI host. - -The public single-container Web template explicitly sets `readonly = true`. -Docker Web otherwise defaults to writable logical and Access operations; physical -management remains unavailable. To use writable Web internally, mount a normal -Web template with `readonly = false` at -`/opt/crowdb/etc/templates/crowdb-web.toml`. The same release image renders that -configuration at startup; no rebuild is required. `/api/mode` reports readonly -and the browser disables write controls when it is enabled. - -## Service selection and inspectable data - -The same release image supports the complete managed profile, a mounted profile -for an explicitly selected service set, or one standalone server. Service -selection does not rebuild the image. Without a command, the usual startup -mode and complete bundled profile apply. To supply a validated profile, mount -it read-only and set `CROWDB_PROFILE_PATH` to its container path; the monitor -validates the profile and its dependency graph before startup. The automatic -`single-node-container` profile supports selecting `kv`, `diskdb`, `diskio`, -`chunkdb`, `chunk-kv`, `access` and `web` through its `services` list. Keep each -selected service's declared dependencies selected. KV is required; KV-only skips -disk creation, while profiles without Access skip S3 and Iceberg initialization. -Unknown services and missing dependencies fail before node identity/discovery or -SSH startup. Restore with the same profile and templates: durable bootstrap -identity rejects changed selections rather than adopting existing data. - -Standalone services use the normal entrypoint command `server SERVICE [arguments]`. -Supported services are `kv`, `diskdb`, `diskio`, `chunkdb`, `chunk-kv`, `access` -and `web`; arguments are the corresponding released server's CLI arguments. -This starts only that server, without monitor automatic recovery or SSH. Supply -its required external dependencies explicitly. Unknown services fail; an existing -managed-node data root cannot be switched to standalone mode. +# Local node storage and client usage + +One node is a topology case of the normal editable cluster lifecycle. For image +builds, named Docker shortcuts, independent Docker commands and Compose, use the +[shared OCI guide](../oci-image/README.md). The old manual/single startup mode +selector is removed; use the accepted deployment mode and initial voter policy. + +## Node-first local storage policy + +The node-first local policy accepts one or three initial voters. Each member +prepares one sparse 16-GiB file with a 16-GiB zone, 1-MiB allocation units and +a 1-GiB free-space reserve. These are test media; sparse capacity does not +reserve physical host space. Production does not fall back to this policy. + +- One-node writes use one mirror copy; three-node writes use two copies. +- Tree, stream and Access payload allocation uses 1-MiB mirror strips. Tree + chunk capacity remains independent of strip capacity. +- Mirror-to-EC conversion is disabled, including Dataset payload writes. +- Local placement declares zero physical-node failure tolerance. Colocated + containers retain the same physical-host identity. The explicit local + placement allowance can put two copies on separate extents of the same + disk; copy count does not promise disk, node or rack protection. Inspect + each strip's placement assessment for its actual protection. +- Accepted media and operation identities survive restart. Failed preparation + is retained for diagnosis; restarting is not permission to replace its data. + +The node-first local client origins default to `http://127.0.0.1` on ports +9091 (S3), 9092 (Iceberg) and 9093 (Dataset). When publishing different host +ports or addresses, supply `CROWDB_S3_PUBLIC_URI`, `CROWDB_ICEBERG_PUBLIC_URI` +and `CROWDB_DATASET_PUBLIC_URI` to each node. These HTTP origins are sealed in +the node policy before startup; omitted restart values retain the accepted +origins. Internal discovery, service listeners and Console requests continue +to use node addresses. The node does not inspect the Docker socket for NAT. + +Read client credentials explicitly with: ```sh -pixi run docker run --name kv-node --network my-owned-bridge \ - --mount type=bind,source=/absolute/node-data,target=/opt/crowdb/data \ - --ulimit core=-1 crowdb-node:dev server kv \ - --root /opt/crowdb/data/kv --management-port 7000 --ports 7001..7011 \ - --stores 0 --groups 1 --replica 1 --log-dir /opt/crowdb/data/log/kv --log +pixi run docker exec NODE crowdb-monitor credentials show --format env ``` -Use different host data directories per node/cluster. Data and service logs are -locally readable under that mount; relative kernel core patterns use -`data/crash` as the standalone process working directory. Host core collectors -still follow host policy; see the crash debugging section below. Preserve the -matching binary/debug information when inspecting a dump. +This exports S3 credentials, the Iceberg writer token and the Dataset bearer +with their public origins. The private client file is +`/opt/crowdb/data/console/secrets/client.env`; it has mode `0600`. Server master +keys stay in the protected server files. Do not put client credential output +into logs or cluster status. -Set `CROWDB_KEEP_DEBUG_INFO=1` while running `pixi run stage-single-node-container` -to retain debug information in the release-optimized Rust and C++ binaries. -The normal OCI build records `org.crowdb.debug-info` and the receipt's -`debug_info` mode. Build once and reuse its immutable image for every selected -service set; default packaging continues to strip debug information. +## Client listeners and readiness -This profile explicitly selects `test_single_node`. KV groups have one voter; -chunk writes use one 1 MiB mirror copy with no EC or conversion. The profile -provides no data protection, and a failed copy returns an I/O error. Production -requires at least three voting nodes and protected placement. +The same Access service exposes S3 at 9091, Iceberg at 9092 and Dataset at 9093. +The Web UI listens at 9090. Publish these ports when using a bridge. A listener +or an old completed setup record is not proof of current storage availability. -The profile keeps chunk capacity separate from strip size. Tree chunks use -`storage.tree_chunk_capacity_bytes` in `chunk-kv.toml`; a 16 MiB tree chunk -contains multiple 1 MiB mirror strips. Stream chunks use -`storage.stream_chunk_capacity_bytes`. S3 and Iceberg each accept their own -`s3.small_write.chunk_capacity_bytes` or -`iceberg.small_write.chunk_capacity_bytes` and large `max_chunk_size` values -in `access.toml`. The shared `small_write` section remains a fallback. The -profile also sets RPC worker and client connection counts explicitly so local -resource use can be tuned without changing production defaults. +Use the client origins returned by the credential command. Iceberg metadata and +Parquet URLs and Dataset payload references must be reachable from the client; +changing Docker port mappings without changing the accepted public origins +before first startup produces unreachable references. Internal service addresses +remain independent of public origins. -```sh -pixi run build-single-node-container -pixi run test-single-node-container -``` - -For an interactive local container workflow, use: - -```sh -pixi run start-container -pixi run inject-container -pixi run clean-container -``` - -`start-container` builds the image, starts `crowdb-single-node`, exposes -ports 9090–9094 on the container network, waits for readiness, and prints client -credentials. It uses the named volume `crowdb-single-node-data` by default; -set `CROWDB_CONTAINER_DATA_VOLUME` to choose another. Starting an existing -stopped container preserves its original mounts, including older anonymous -volumes. `inject-container` loads TPC-H SF1 into native Iceberg and copies its -Parquet files to the container's S3 endpoint as an independent benchmark copy. -The S3 report's bucket and keys are not Iceberg locations or backing storage; -Iceberg manifests use the native listener's returned table-scoped URLs. -The loader retains its data and -reports under `.crowdb-runtime/artifacts/tpc-loader/`. `clean-container` -removes only the selected container and retains its data volumes. It does not -prune images, other containers or unrelated volumes. Recreating the container -with the same named volume retains its data. Set -`CROWDB_CONTAINER_IMAGE` or `CROWDB_CONTAINER_NAME` to select another image or -container name. - -`inject-container` discovers running CROWDB containers by their image metadata -or CROWDB image name. If exactly one is running, it uses that container. With -multiple candidates it lists their IDs, names, images and IP addresses and exits -without loading data. Select one explicitly (also supported through -`CROWDB_CONTAINER_NAME`): - -```sh -pixi run inject-container crowdb-single-node -pixi run inject-container 671a2e13b8ba -pixi run inject-container 172.17.0.2 -``` - -- Compilation runs on the host with the locked repository dependencies. Cargo - and CMake reuse existing build outputs; npm uses its local download cache. -- The build stages release programs, their required shared libraries, UI and - deployment files under `target/container-runtime`. Existing runtime data and - credentials are never part of the Docker context. -- Standalone Pixi-managed BuildKit packages these files into the pinned Ubuntu OCI image. It - does not install Pixi or compilers, compile source, or use a custom base image. -- Packaging checks dynamic linkage inside Ubuntu and verifies the staged - revision/version against image metadata. A different host ABI must pass these - checks and the container tests before its artifacts can be used. -- The default local image is `crowdb-node:dev`. Set - `CROWDB_CONTAINER_IMAGE` to build and test a separate candidate tag. - -`pixi run stage-single-node-container` produces the runtime directory without -building a Docker image. Work on a `release/` branch whose `VERSION` -matches the branch name, such as `release/0.2.2`. After pushing each candidate -commit, select that branch in the GitHub Actions manual run form, or dispatch -it from a clean checkout that matches the remote branch: - -```sh -pixi run -- python tools/release.py --dry-run -pixi run -- python tools/release.py --execute -``` - -After the image passes container verification, publication updates eight tags -to the same image digest: - -- `crowdb/crowdb-node:` and `crowdb/crowdb-node:latest` (canonical). -- `crowdb/crowdb-iceberg:` and `crowdb/crowdb-iceberg:latest`. -- `crowdb/crowdb-s3:` and `crowdb/crowdb-s3:latest`. -- `crowdb/crowdb-dataset:` and `crowdb/crowdb-dataset:latest`. - -The node repository is the default entry point for manual cluster setup. -Iceberg, S3 and Dataset names are the single-node usage entry points. All four -contain the same runtime, including one Access Server listening on Iceberg -port 9092 and S3 port 9091. Publish the ports needed by the client; using both -interfaces requires only one container. The container verification runs boto3, -AWS CLI and PyIceberg clients against that runtime, including reads after recovery -and persisted-volume restart. - -Rerunning publication from an older release branch also updates all four `latest` -tags, so use the newest release branch for moving tags. - -The script only dispatches the workflow; it does not change files or push. -The dry run does not contact GitHub. - -The workflow builds and tests the container, then waits for DockerHub environment approval. -Before publishing, it checks that the remote branch still points to the same -commit. A newer branch commit requires a new run. Each successful run replaces -all four repositories' version and `latest` tags and signs the digest in each -repository. It does not -create a Git tag or GitHub Release. Fix a failed candidate on the release -branch and run the workflow again. - -The workflow constructs one OCI artifact without Docker, verifies that artifact -on Docker and containerd, then copies its exact graph to the registry. Publishing -preserves the verified digest and attaches signatures, SBOM and provenance. - -## Public ports - -The default host mappings match the container listener ports: - -- Console: `9090:9090`, optional. -- S3: `9091:9091`. -- Iceberg catalog and native FileIO: `9092:9092`. - -The `crowdb/crowdb-dataset` name reserves the single-node Dataset usage entry -point. Port `9093` is reserved for future Dataset access; the current image does -not listen on it. Publishing this alias does not enable a Dataset service. - -Publish only the interfaces needed by the client. The generated default client -addresses use `localhost` with these ports. An Iceberg deployment with a different -host name or mapped port must also configure its advertised public URI; changing -the catalog connection URI alone does not change delegated FileIO endpoints. -The Access Server runs as the existing unprivileged user without a low-port -binding capability. - -For a local host workflow, map the three client-facing ports. The profile's -loopback addresses then match the host mappings, including delegated Iceberg -FileIO locations: - -```sh -pixi run docker run -d --name crowdb-iceberg -e CROWDB_STARTUP_MODE=single \ - --mount type=volume,source=crowdb-iceberg-data,target=/opt/crowdb/data \ - -p 127.0.0.1:9090:9090 \ - -p 127.0.0.1:9091:9091 \ - -p 127.0.0.1:9092:9092 \ - crowdb/crowdb-iceberg:latest -``` - -For a different reachable address or a client on the same Docker network, set -`CROWDB_S3_PUBLIC_URI` and `CROWDB_ICEBERG_PUBLIC_URI` at startup. The monitor -passes those values to the access service instead of the profile defaults. - -## S3 container usage - -Start the S3 single-node entry point with a named data volume and map host port 9091 -to the container's S3 port 9091: - -```sh -pixi run docker run -d --name crowdb-s3 -e CROWDB_STARTUP_MODE=single \ - --mount type=volume,source=crowdb-s3-data,target=/opt/crowdb/data \ - -p 127.0.0.1:9091:9091 \ - crowdb/crowdb-s3:latest -pixi run docker exec crowdb-s3 crowdb-monitor readiness -``` - -Wait for the readiness probe to succeed, then obtain the generated client credentials: - -```sh -pixi run docker exec crowdb-s3 crowdb-monitor credentials show --format env -``` - -Configure boto3 with endpoint `http://127.0.0.1:9091`, the displayed AWS -credentials and region, and path-style addressing: - -```python -import boto3 -from botocore.config import Config - -client = boto3.client( - "s3", - endpoint_url="http://127.0.0.1:9091", - config=Config(s3={"addressing_style": "path"}), -) -client.create_bucket(Bucket="example") -client.put_object(Bucket="example", Key="hello.txt", Body=b"hello") -print(client.get_object(Bucket="example", Key="hello.txt")["Body"].read()) -``` - -Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_DEFAULT_REGION` -in the client's environment using the displayed values before running this -example. To use Iceberg from the same container, also map container port 9092. -The single-node examples explicitly select the development profile described above. -Image names do not change startup mode: use `CROWDB_STARTUP_MODE=single` for -automatic initialization; the default is manual cluster setup. +All members of one local cluster share the accepted client credential set. +Retain each member's own data volume across stop, clean and image replacement. +Default `oci-clean` removes the containers and their unused owned bridge; +`--delete-data` additionally deletes the selected helper's owned volumes. +Ordinary `docker compose down` also preserves its named volumes. ## Tested S3 client recipes @@ -308,151 +122,15 @@ available. ListBuckets CreationDate is a stable Unix-epoch placeholder. ListObjectsV2 supports `encoding-type=url` for keys requiring XML-safe encoding. -## Tested Iceberg ecosystem recipes - -The [verified client matrix](../../e2e-test/clients/ecosystem/README.md) covers PyIceberg 0.11.1, -Arrow 25.0.0, pandas 3.0.6, Polars 1.35.2, DuckDB 1.4.3, Spark 3.5.6 and Flink -1.20.2 with Iceberg 1.11.0, and Trino 483 pinned by image digest. Each listed -profile passed against the actual container, including selected rows after a -persisted-volume restart. Spark/Flink mutation and Python snapshot handoff are -separate from merely discovering a catalog. - -```sh -pixi run build-single-node-container -pixi run -e iceberg-ecosystem test-iceberg-container-ecosystem -``` - -The **Iceberg container ecosystem** GitHub Actions workflow runs only by manual -`workflow_dispatch`, like the Rust Iceberg and optional S3 language SDK workflows. -It defaults to the complete `all` matrix and offers a Python/DuckDB diagnostic -profile. The environment and task are outside default tests and CI dependencies. -Reports contain pinned versions, image/source identity and redacted diagnostics; -the fixture removes owned containers/networks and retains mounted data/logs on success and failure. - -The [fixture recipes](../../e2e-test/clients/ecosystem/README.md#connection-recipes) define exact -REST, delegated FileIO, endpoint and classpath setup. Trino explicitly uses the -Iceberg/FileIO origin with path-style S3 access and vended table credentials. -The matrix selects Parquet/v2. It does not establish ORC, arbitrary engine -operations, Kafka Connect ingestion, an untested BI tool, production storage or -upgrade compatibility. Performance follow-ups and known diagnostic observations -remain in the [Access acceptance guide](../../app/crowdb-access-server/tests/README.md). - -## Crash collection boundary - -For host configuration, restoring its collector, and GDB commands for both -container and bare-metal cores, see the -[crash debugging guide](../../doc/dev/crash_debugging.md). - -The image does not configure the host's Linux core collector. Inspect -`/proc/sys/kernel/core_pattern` on the Docker host before expecting a dump in -the mounted data volume. A leading `|` sends a crash to a host-side collector; -relative `core` or `core.*` patterns write in the crashing process's working -directory. The container runs the monitor and managed children from the private -`/opt/crowdb/data/crash` directory. On monitor startup and after a child is -reaped, it removes older regular `core` files and keeps the newest one. -The directory has mode `0700`; symlinks named `core.*` are not followed or -deleted. This is one-core retention, not a promise that the host creates a -volume file. Other relative filename patterns are outside this retention rule. +## Iceberg and Dataset clients -For a host with a relative `core` pattern, add a size bound to `docker run`: - -```sh ---ulimit core=1073741824:1073741824 -``` - -The example bounds each core to 1 GiB. A small bound may truncate a dump and -make some stack frames unavailable. Core collection can also be suppressed by -the host's dumpability policy, including for executables with file capabilities. -The container never changes `core_pattern` or the host's dumpability policy. - -- On a systemd-coredump host, use `coredumpctl list crowdb-kv-server` to find - the host report, then `coredumpctl --output=/private/core dump - crowdb-kv-server` as an authorized host user to export it. Check that the - result is readable and nonempty before symbolization. -- On an Ubuntu Apport host, find the matching report in `/var/crash` on the - Docker host. Create a private directory, then run `sudo apport-unpack - /var/crash/REPORT.crash /private/crowdb-core/unpacked`. The extracted - `CoreDump` is the file to pass to the symbolizer. Apport reports may be - readable only by the host administrator; preserve the private permissions - when granting the debugging user access. A pipe pattern does not create a - volume file. Apport may fail to resolve a CROWDB executable because its - `/opt/crowdb/bin` path exists only inside the container; it may also ignore - executables outside host distribution packages. Check the host's Apport log - when no report appears. - If the report or `CoreDump` is absent, collection is unavailable for that - crash; do not substitute a log or an unrelated dump. -- On Docker Desktop, inspect the Linux VM's collector. The desktop host's - native crash directory is not the container's core directory. - -Core dumps can contain credentials and user data. Keep exports in a private -directory and do not attach them to ordinary logs or issues. The release image -contains stripped binaries, so a core may give only a limited stack trace. -No host collector change is required by the image build. - -Collector behavior follows the [Linux core pattern documentation](https://docs.kernel.org/admin-guide/sysctl/kernel.html), -[systemd-coredump manual](https://www.freedesktop.org/software/systemd/man/250/systemd-coredump.socket.html), -and [Ubuntu Apport documentation](https://ubuntu.com/project/docs/contributors/debugging/apport/). - -## Manual multi-node Docker deployment - -The same Linux amd64 image supports `CROWDB_STARTUP_MODE=manual`. Each node starts -SSH on 2222, monitor discovery/control and Web on 9090. KV and application servers -start through cluster preparation and shared management. Default startup is manual for one or more nodes. Explicit `single` selects -automatic initialization with virtual disks and disabled editing. - -For an isolated bridge test using the system Docker installation: - -```bash -pixi run docker network create crowdb-nodes -pixi run docker run -d --name node-one --network crowdb-nodes --cpus 2 --memory 1g \ - --mount type=volume,source=node-one-data,target=/opt/crowdb/data \ - -e CROWDB_STARTUP_MODE=manual -e CROWDB_PHYSICAL_HOST_ID=test-host \ - -p 127.0.0.1:19090:9090 crowdb-node:dev -``` - -Repeat with distinct names, volumes and published UI ports. No peer list is needed -on a multicast-capable bridge. Open a UI, create a rack, move candidates with SSH -access and initialize selected nodes. The test login is `crowdb` / `crowdb`; -production uses an explicit password file or preinstalled authorized keys. Every -node keeps its own Ed25519 private key. Public keys, authorized keys and SSH host -keys persist under the mounted root; initialization passwords are never persisted -in console or Group 0 records. Ports advertised to peers are container addresses, -not the browser's published host ports. - -Production Linux host networking uses an immutable image digest, explicit resource -limits, management interface, physical host identity and persistent directory -owned by uid 10001. Run the launcher through Pixi: - -```bash -pixi run python container/single-node-container/run-node.py \ - --image sha256: --name crowdb-node --network host \ - --data-root /var/lib/crowdb --physical-host-id host-one --interface eno1 \ - --cpus 2 --memory-mib 1024 --password-file /private/ssh-password -``` - -The password file must be private and absolute. Existing key access can omit it. -Permitted block devices use explicit `--device` arguments and must allow read/write -access for container uid/gid 10001; configure that access on the host before launch. -`--seed` supplies monitor addresses when multicast is unavailable. The launcher -validates Docker resources, -image architecture/digest, mounts and credentials; no runtime socket is mounted -inside nodes. Host networking uses UI port 9090 directly. Container recreation and -image upgrades preserve the same persistent root. System Docker and Pixi-managed containerd/nerdctl use the same OCI image. -See the [shared image guide](../oci-image/README.md) for Docker-free build, -rootless host prerequisites, lightweight runtime startup and daemon supervision. -Existing build/start helpers require system Docker for local import/start and -explicitly select automatic single-node startup. - -Acceptance harnesses use the same pinned image: - -```bash -pixi run test-container-e2e --profile bridge-admission --concurrency 1 --node-cpus 0.5 --cpus 4 --memory-mib 6144 -pixi run python container/single-node-container/tests/host-node.py -``` +The [Iceberg ecosystem guide](../../e2e-test/clients/ecosystem/README.md) describes +locked PyIceberg and external ecosystem clients. PyIceberg tests write real +metadata/Parquet objects and read them through each node's configured public +origin. Dataset has its own bearer token and REST namespace; use its returned +payload references rather than constructing storage URLs. -Set `CROWDB_CONTAINER_IMAGE` to its immutable local image ID. Bridge acceptance -covers independent clusters, cross-UI bootstrap recovery, SSH cancellation, quorum -loss, stale commands, persistent endpoint/rack updates, voting catch-up and partial -cleanup. The host harness checks explicit credentials/resources and persisted -identity/key recovery. The existing container suite covers automatic single-node -clients, browser behavior, crash/hang recovery and invalid persistent inputs. +The OCI image carries the same server binaries for all protocols. It neither +mounts the Docker socket nor delegates bootstrap or storage preparation to the +host CLI. Monitor-owned accepted operations continue after the launching shell +or browser exits. Changing an image does not reset node identity or storage. diff --git a/container/single-node-container/dev-container.sh b/container/single-node-container/dev-container.sh index b0e764650..85e82f4d5 100644 --- a/container/single-node-container/dev-container.sh +++ b/container/single-node-container/dev-container.sh @@ -97,9 +97,9 @@ select_inject_container() { echo 'Candidates (ID NAME IMAGE IP):' >&2 printf '%s\n' "${candidates[@]}" >&2 else - echo "No running CROWDB containers found; use 'pixi run start-container'." >&2 + echo "No running CROWDB containers found; use 'pixi run oci-run-1c'." >&2 fi - echo 'Usage: pixi run inject-container ' >&2 + echo 'Usage: pixi run bash container/single-node-container/dev-container.sh inject ' >&2 return 1 fi name=${matches[0]} diff --git a/container/single-node-container/entrypoint.sh b/container/single-node-container/entrypoint.sh index 665c2574d..efc153a1e 100644 --- a/container/single-node-container/entrypoint.sh +++ b/container/single-node-container/entrypoint.sh @@ -53,40 +53,60 @@ fi profile=${CROWDB_PROFILE_PATH:-/opt/crowdb/etc/profile.toml} /opt/crowdb/bin/crowdb-monitor validate "$profile" -mode=${CROWDB_STARTUP_MODE:-manual} -case "$mode" in single|manual) ;; *) echo 'CROWDB_STARTUP_MODE must be single or manual' >&2; exit 1 ;; esac -previous= -if test -f /opt/crowdb/data/accepted-node.json; then - previous=manual -elif test -f /opt/crowdb/data/bootstrap/manifest.json; then - previous=single -fi -if test -n "$previous" && test "$previous" != "$mode"; then - echo 'Startup mode differs from persistent node policy; restore the original mode.' >&2 +if test -n "${CROWDB_STARTUP_MODE:-}"; then + echo 'CROWDB_STARTUP_MODE was removed; use CROWDB_DEPLOYMENT_MODE and CROWDB_INITIAL_VOTERS.' >&2 exit 1 fi -if test "$mode" = manual && test -z "${CROWDB_PHYSICAL_HOST_ID:-}"; then - echo 'Manual mode requires CROWDB_PHYSICAL_HOST_ID' >&2 +accepted_mode=$(setpriv --reuid=crowdb --regid=crowdb --init-groups \ + /opt/crowdb/bin/crowdb-monitor accept-policy --profile "$profile") +CROWDB_DEPLOYMENT_MODE=$accepted_mode +export CROWDB_DEPLOYMENT_MODE +disks=/opt/crowdb/data/disks +if test -L "$disks" || { test -e "$disks" && ! test -d "$disks"; }; then + echo 'Disk directory must be a real directory, not a symlink or file.' >&2 exit 1 fi -if test "$mode" = manual; then - disks=/opt/crowdb/data/disks - if test -L "$disks" || { test -e "$disks" && ! test -d "$disks"; }; then - echo 'Disk directory must be a real directory, not a symlink or file.' >&2 - exit 1 - fi - mkdir -p "$disks" - chown -h crowdb:crowdb "$disks" +mkdir -p "$disks" +chown -h crowdb:crowdb "$disks" +if test -L /opt/crowdb/data/ssh || { test -e /opt/crowdb/data/ssh && ! test -d /opt/crowdb/data/ssh; }; then + echo 'SSH identity directory must be a real directory.' >&2 + exit 1 fi +mkdir -p /opt/crowdb/data/ssh +chmod 700 /opt/crowdb/data/ssh ssh_password_auth=yes if test -n "${CROWDB_SSH_PASSWORD_FILE:-}"; then test -f "$CROWDB_SSH_PASSWORD_FILE" || { echo 'SSH password file missing' >&2; exit 1; } CROWDB_SSH_PASSWORD=$(cat "$CROWDB_SSH_PASSWORD_FILE") fi +if test "$accepted_mode" = local; then + local_password=/opt/crowdb/data/ssh/local-admission-password + if test -L "$local_password" || { test -e "$local_password" && ! test -f "$local_password"; }; then + echo 'Local admission credential must be a regular private file.' >&2 + exit 1 + fi + if test -f "$local_password"; then + test -s "$local_password" && test "$(stat -c %s "$local_password")" -le 4096 && + test "$(stat -c %a "$local_password")" = 600 || { + echo 'Retained local admission credential is empty, oversized or not private.' >&2 + exit 1 + } + fi + if test -z "${CROWDB_SSH_PASSWORD:-}" && test -f "$local_password"; then + CROWDB_SSH_PASSWORD=$(cat "$local_password") + test -n "$CROWDB_SSH_PASSWORD" || { echo 'Retained local admission credential is invalid.' >&2; exit 1; } + fi + CROWDB_SSH_PASSWORD=${CROWDB_SSH_PASSWORD:-crowdb} + password_tmp=$(mktemp /opt/crowdb/data/ssh/.local-admission.XXXXXX) + printf '%s' "$CROWDB_SSH_PASSWORD" > "$password_tmp" + chmod 600 "$password_tmp" + chown crowdb:crowdb "$password_tmp" + mv "$password_tmp" "$local_password" +fi if test -n "${CROWDB_SSH_PASSWORD:-}"; then printf 'crowdb:%s\n' "$CROWDB_SSH_PASSWORD" | chpasswd unset CROWDB_SSH_PASSWORD -elif test "${CROWDB_DEPLOYMENT_MODE:-test}" = production; then +elif test "$accepted_mode" = production; then if ! test -s /opt/crowdb/data/ssh/authorized_keys; then echo 'Production requires explicit SSH credentials or preinstalled authorized_keys' >&2 exit 1 @@ -120,9 +140,5 @@ PidFile /opt/crowdb/run/sshd.pid Subsystem sftp internal-sftp EOF /usr/sbin/sshd -f /opt/crowdb/run/sshd_config -if test "$mode" = manual; then - test -n "${CROWDB_PHYSICAL_HOST_ID:-}" || { echo 'Manual mode requires CROWDB_PHYSICAL_HOST_ID' >&2; exit 1; } - exec setpriv --reuid=crowdb --regid=crowdb --init-groups /opt/crowdb/bin/crowdb-monitor run \ - --profile "$profile" --mode manual --interface "${CROWDB_MANAGEMENT_INTERFACE:-eth0}" --physical-host-id "$CROWDB_PHYSICAL_HOST_ID" -fi -exec setpriv --reuid=crowdb --regid=crowdb --init-groups /opt/crowdb/bin/crowdb-monitor run --profile "$profile" --mode single +exec setpriv --reuid=crowdb --regid=crowdb --init-groups /opt/crowdb/bin/crowdb-monitor run \ + --profile "$profile" --interface "${CROWDB_MANAGEMENT_INTERFACE:-eth0}" diff --git a/container/single-node-container/templates/access.toml b/container/single-node-container/templates/access.toml index 08995eec6..e75b63408 100644 --- a/container/single-node-container/templates/access.toml +++ b/container/single-node-container/templates/access.toml @@ -2,7 +2,7 @@ # Secrets stay in /opt/crowdb/data/secrets/server.env. [deployment] -mode = "test_single_node" +mode = "local" max_node_failures = 0 [common] diff --git a/container/single-node-container/templates/chunkdb.toml b/container/single-node-container/templates/chunkdb.toml index 05ec3f2ba..96ca1a04a 100644 --- a/container/single-node-container/templates/chunkdb.toml +++ b/container/single-node-container/templates/chunkdb.toml @@ -3,7 +3,7 @@ service_instances = [1] storage_groups = [{ store_id = 0, group_id = 1 }] [deployment] -mode = "test_single_node" +mode = "local" max_node_failures = 0 [server] diff --git a/container/single-node-container/tests/container-e2e.sh b/container/single-node-container/tests/container-e2e.sh index 2a865ae9d..95a47fdf6 100644 --- a/container/single-node-container/tests/container-e2e.sh +++ b/container/single-node-container/tests/container-e2e.sh @@ -4,4 +4,6 @@ set -euo pipefail exec pixi run test-container-e2e --layer deployment \ - --image "${CROWDB_CONTAINER_IMAGE:-crowdb-node:dev}" "$@" + --image "${CROWDB_CONTAINER_IMAGE:-crowdb-node:dev}" \ + --cpus "${CROWDB_E2E_CPUS:-4}" \ + --memory-mib "${CROWDB_E2E_MEMORY_MIB:-8192}" "$@" diff --git a/container/single-node-container/tests/entrypoint-policy.py b/container/single-node-container/tests/entrypoint-policy.py new file mode 100644 index 000000000..6b092c395 --- /dev/null +++ b/container/single-node-container/tests/entrypoint-policy.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""Exercise entrypoint ordering/credentials in Docker, with a monitor boundary stub.""" + +import argparse +import os +from pathlib import Path +import secrets +import subprocess +import tempfile +import uuid + + +MONITOR = """#!/bin/sh +set -eu +case "$1" in + validate) exit 0 ;; + accept-policy) + test "$CROWDB_TEST_REJECT" != 1 || exit 7 + umask 077 + printf '{"version":1}' > /opt/crowdb/data/node-policy.json + printf '%s\\n' "$CROWDB_TEST_MODE" + ;; + run) + test "$CROWDB_DEPLOYMENT_MODE" = "$CROWDB_TEST_MODE" + test -s /opt/crowdb/data/ssh/id_ed25519 + test -s /opt/crowdb/data/ssh/ssh_host_ed25519_key + touch /opt/crowdb/data/entrypoint-ran + ;; + *) exit 2 ;; +esac +""" + +CHECK = """#!/bin/sh +set -eu +if test "$CROWDB_TEST_CORRUPT" = 1; then + mkdir -p /opt/crowdb/data/ssh + touch /opt/crowdb/data/ssh/local-admission-password + chmod 600 /opt/crowdb/data/ssh/local-admission-password +fi +set +e +/bin/sh /entrypoint-under-test +result=$? +set -e +if test "$CROWDB_TEST_OBSOLETE" != 0; then + test "$result" != 0 + test ! -e /opt/crowdb/data/ssh + test ! -e /opt/crowdb/data/node-policy.json +elif test "$CROWDB_TEST_REJECT" = 1; then + test "$result" != 0 + test ! -e /opt/crowdb/data/ssh + test ! -e /opt/crowdb/data/disks + test ! -e /opt/crowdb/data/node-policy.json +elif test "$CROWDB_TEST_CORRUPT" = 1; then + test "$result" != 0 + test ! -s /opt/crowdb/data/ssh/local-admission-password + test ! -e /opt/crowdb/data/ssh/id_ed25519 +elif test "$CROWDB_TEST_MODE" = production && test "$CROWDB_TEST_PASSWORD" = 0; then + test "$result" != 0 + test ! -e /opt/crowdb/data/ssh/id_ed25519 + test ! -e /opt/crowdb/data/ssh/local-admission-password +else + test "$result" = 0 + test -e /opt/crowdb/data/entrypoint-ran + test "$(stat -c %a /opt/crowdb/data/node-policy.json)" = 600 + if test "$CROWDB_TEST_MODE" = local; then + test "$(stat -c %a /opt/crowdb/data/ssh/local-admission-password)" = 600 + test "$(stat -c %u /opt/crowdb/data/ssh/local-admission-password)" = 10001 + cmp /run/test-password /opt/crowdb/data/ssh/local-admission-password + else + test ! -e /opt/crowdb/data/ssh/local-admission-password + fi +fi +""" + + +def invoke(*arguments): + return subprocess.run(["docker", *arguments], text=True, capture_output=True, timeout=30) + + +def run_case(image, entrypoint, mode, supplied, reject, corrupt=False, obsolete=""): + name = "crowdb-entrypoint-policy-" + uuid.uuid4().hex[:12] + secret = secrets.token_urlsafe(24) if supplied else "crowdb" + with tempfile.TemporaryDirectory(prefix="crowdb-entrypoint-policy-") as directory: + directory = Path(directory) + for filename, body in (("monitor", MONITOR), ("check", CHECK)): + path = directory / filename + path.write_text(body) + path.chmod(0o755) + password = directory / "password" + password.write_text(secret) + password.chmod(0o600) + arguments = ["run", "--rm", "--name", name, "--network", "none", + "--label", "org.crowdb.test=entrypoint-policy", + "--tmpfs", "/opt/crowdb/data:uid=10001,gid=10001,mode=0755", + "--entrypoint", "/bin/sh"] + for source, target in ((entrypoint, "/entrypoint-under-test"), + (directory / "monitor", "/opt/crowdb/bin/crowdb-monitor"), + (directory / "check", "/entrypoint-check"), + (password, "/run/test-password")): + arguments += ["--mount", f"type=bind,source={source},target={target},readonly"] + for key, value in (("CROWDB_TEST_MODE", mode), ("CROWDB_TEST_REJECT", int(reject)), + ("CROWDB_TEST_PASSWORD", int(supplied)), + ("CROWDB_TEST_CORRUPT", int(corrupt)), + ("CROWDB_STARTUP_MODE", obsolete), + ("CROWDB_TEST_OBSOLETE", obsolete or "0")): + arguments += ["--env", f"{key}={value}"] + if supplied: + arguments += ["--env", "CROWDB_SSH_PASSWORD_FILE=/run/test-password"] + try: + result = invoke(*arguments, image, "/entrypoint-check") + output = (result.stdout + result.stderr).replace(secret, "") + if result.returncode: + raise RuntimeError(f"case mode={mode} supplied={supplied} reject={reject}:\n{output}") + if secret != "crowdb" and secret in result.stdout + result.stderr: + raise RuntimeError("entrypoint exposed a private SSH credential") + print(f"PASS entrypoint mode={mode} supplied={supplied} reject={reject} corrupt={corrupt}", flush=True) + finally: + inspected = invoke("container", "inspect", name) + if inspected.returncode == 0: + removed = invoke("rm", "--force", name) + if removed.returncode: + raise RuntimeError(removed.stdout + removed.stderr) + elif "No such" not in inspected.stderr: + raise RuntimeError(inspected.stdout + inspected.stderr) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--image", default=os.environ.get("CROWDB_CONTAINER_IMAGE", "crowdb-node:dev")) + args = parser.parse_args() + entrypoint = Path(__file__).resolve().parents[1] / "entrypoint.sh" + for mode, supplied, reject in (("local", False, True), ("local", False, False), + ("local", True, False), ("production", False, False), + ("production", True, False)): + run_case(args.image, entrypoint, mode, supplied, reject) + run_case(args.image, entrypoint, "local", False, False, corrupt=True) + for obsolete in ("single", "manual"): + run_case(args.image, entrypoint, "local", False, False, obsolete=obsolete) + print("Entrypoint boundary checks passed; this does not certify application startup or image content.", flush=True) + + +if __name__ == "__main__": + main() diff --git a/doc/backlog/R196-access-upload-benchmark-regression.md b/doc/backlog/R196-access-upload-benchmark-regression.md deleted file mode 100644 index e904c36ed..000000000 --- a/doc/backlog/R196-access-upload-benchmark-regression.md +++ /dev/null @@ -1,154 +0,0 @@ - - - -### R196: access server — S3 and Iceberg upload benchmark regression - -#### Problem - -The current `crowdb-cli bench s3` measures an invocation-owned memory-backed -cluster, not an HTTP request through a durable access server. The S3 E2E Python -benchmark captures request samples inside a full-stack test but has no reusable -regression command or failure sentinel. Iceberg has focused upload timings, but -no equivalent repeatable benchmark. The chunk IO regression script exercises -the writer below HTTP decoding, request integrity, and protocol publication. -Consequently, a change can improve chunk IO while slowing S3 or Iceberg PUT, -multipart part upload, or metadata publication without a comparable measurement. -See the [access server design](../design/access-server/design-crowdb-access-server.md), -[Iceberg upload-flow analysis](../design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md), -and [implemented upload-flow analysis](../design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md). - -#### Solution - -Extend `crowdb-cli bench` with one real HTTP upload workload driver that uses -the same measurement, payload generation, concurrency, and result schema for -S3 and Iceberg. Keep the existing memory-backed `bench s3` contract unchanged. -Protocol adapters perform authentication, target setup, direct PUT or multipart -requests, response validation, and cleanup. A timed sample starts immediately -before request submission and ends only when the server acknowledges the -durable write and the protocol's corresponding object, file, or part record is -published. Multipart completion is a separate timed phase; an Iceberg table -snapshot commit is not included in FileIO upload latency and is reported -separately if exercised. - -The workload reuses a prepared, deterministic 1-MiB payload block for large -objects and computes expected integrity values before timing. Streaming does -not allocate or copy an object-sized client buffer, hash the full object in the -timed loop, or read the object back after every successful upload. A bounded -sample outside the timed interval verifies stored bytes and range boundaries. -The CLI reports client preparation and transfer timings separately so client -work is not mistaken for server time. - -The regression scripts own a reproducible local single-node storage and -access-server deployment by default, using the project's existing deployment -and configuration facilities. They retain service logs, metrics snapshots, -machine-readable per-operation samples, and a summary under one run directory. -An explicit remote-endpoint mode uses the same workload against a supplied -server without claiming local process or storage metrics. The scripts compare -only like-for-like topology and configuration profiles. - -The following invariants define the benchmark: - -- **I1 — Real protocol path.** Every timed S3 and Iceberg sample crosses the - HTTP access server, request integrity checks, chunk writer, and the relevant - protocol publication. The report distinguishes direct PUT, multipart part, - multipart completion, and optional table commit. -- **I2 — Controlled input.** Size, concurrency, operation count or duration, - warmup, payload seed, request integrity mode, and deployment profile are - recorded. Client payload memory is bounded independently of object size. - Preparation, namespace/table/session setup, final verification, and cleanup - are outside the timed upload samples. -- **I3 — Honest completion.** The driver records admitted, completed, failed, - and incomplete operations, drains admitted work before exiting, validates - success responses and ETags/digests, and exits nonzero for any failed, - missing, or unverified operation. A timeout or metrics-collection failure - remains visible in the retained result. -- **I4 — Comparable measurements.** Both protocols emit the same JSON and TSV - fields for object size, concurrency, request count, logical bytes, elapsed - time, throughput, average, p50, p95, p99 when sample counts support them, - and error classes. Percentiles with too few samples are absent, not inferred. - The report includes client CPU/RSS and, for local runs, access-server - CPU/RSS and before/after server metric deltas. The shared write-flow stage - counters from R195 are included when available, with concurrent stage - durations reported separately from wall-clock latency. -- **I5 — Regression decision.** The default sentinel gates correctness, - completion, timeout, artifact presence, and required metric availability. - Throughput and latency comparisons use an explicitly selected baseline for - the same profile and a documented tolerance; they do not use one fixed - hardware-dependent number. The result states whether a case was measured, - comparable, regressed, or invalid, and names the failing condition. - -Work items: - -1. Add a real HTTP upload verb and shared result model under - `app/crowdb-cli/src/commands/bench/` and its workload implementation under - `lib/crowdb-console-shared/src/ops/`. The CLI accepts protocol, endpoint, - direct or multipart mode, workload bounds, output path, and credentials via - existing environment/config conventions. Reuse the same input producer and - percentile/accounting logic for both protocol adapters. Do not route this - workload through the memory-backed `bench s3` engine. -2. Implement S3 and Iceberg upload adapters. For S3, cover direct PUT and - UploadPart plus CompleteMultipart. For Iceberg, cover native FileIO direct - PUT and multipart part plus completion against an authorized table/file - scope. Preserve each protocol's signing, integrity, ETag, and publication - rules; never count a successful part as a published final object. Report - client-side signing or hashing work separately if the selected integrity - mode requires it. -3. Add `tools/benchmark/` regression scripts modeled on - `bench-chunkio-write-regression.sh`. Use a named local deployment profile, - build required binaries through `pixi run`, start and clean up the stack, - execute isolated S3 and Iceberg cases, retain full output and service - metrics, and print a compact per-case summary. Provide bounded timeouts - and a case filter for focused runs. Support explicit remote endpoints - without trying to destroy a remote deployment. -4. Include a small-path case below the configured threshold, a boundary - case, direct 100-MiB PUT, and multipart 100-MiB upload for each protocol. - Run at least one single-client and one concurrent-client case. Record the - exact threshold, part size, mirror/EC policy, durability setting, and - software revision with each run so changed profiles are not silently - compared. Keep setup and verification outside timed samples. -5. Save a baseline artifact and comparison contract for each supported local - profile. Accept an optional reference result and tolerance in the runner; - reject comparisons across incompatible profiles. Preserve raw samples and - metrics for diagnosis even when a case fails. - -#### Dependencies - -- The existing local deployment facilities, access-server S3 and Iceberg - HTTP routes, protocol credentials, and chunk IO benchmark artifact - conventions are the baseline. The shared upload flow is implemented and its current performance is accepted; - this requirement owns reproducible repeated measurements and regression gates. -- The implemented shared upload flow supplies finer write-flow stage metrics. - The local sentinel requires those fields for both protocols and reports any - missing field explicitly rather than fabricating stage measurements. -- Official client compatibility remains covered by the existing E2E suites; - this benchmark's common producer isolates server upload performance from - PyArrow or boto3 buffering. A separate client-inclusive profile may reuse - the same result schema without replacing the controlled regression profile. - -#### Acceptance - -- Given a local single-node deployment and the same prepared payload, run S3 - and Iceberg direct PUT cases; assert each traverses the HTTP server, - validates integrity, publishes the correct object or file record, and emits - comparable latency and byte fields (I1, I2, I4). E2E test. -- Given S3 and Iceberg multipart workloads, upload parts and complete them; - assert part and completion latencies and outcomes are distinct, and a part - alone does not count as a published final object (I1, I3). E2E test. -- Given small, threshold-boundary, and 100-MiB cases at one and multiple - clients, run the CLI with a fixed seed; assert the recorded case profile, - bounded client payload memory, complete drain, verified bytes, and retained - JSON/TSV samples match the submitted operations (I2–I4). E2E test. -- Given a rejected digest, failed publication, timeout, or interrupted client, - run a case; assert the command exits nonzero, records the correct failure or - incomplete count, and retains logs and metrics for diagnosis (I3, I5). - Integration test. -- Given a matching baseline and a mismatched profile, compare both with the - same result; assert the matching comparison applies its configured tolerance - and reports a regression when exceeded, while the mismatched comparison is - rejected without a performance verdict (I4, I5). Unit test. -- Given R195 metrics on S3 and Iceberg servers, run the local scripts; assert - before/after deltas include the shared stage counts and waits, have no - object-derived labels, and keep overlapping durations separate from total - request latency (I4, I5). E2E test. - -Run `pixi run rs-fmt-check`, `pixi run rs-lint`, `pixi run cargo test -p crowdb-console-shared`, `pixi run cargo test -p crowdb-cli`, and the focused S3 and Iceberg benchmark regression scripts through `pixi run` for the implemented scope. diff --git a/doc/backlog/R196-refine-benchmark-tests.md b/doc/backlog/R196-refine-benchmark-tests.md new file mode 100644 index 000000000..a5de1d89d --- /dev/null +++ b/doc/backlog/R196-refine-benchmark-tests.md @@ -0,0 +1,282 @@ + + + +### R196: benchmark — Refine benchmark tests + +#### Problem + +Service benchmarks currently mix native regression fixtures, standalone Rust +bench binaries, and `crowdb-cli bench` commands. Result parsing, phase checks, +CPU accounting, and reports are repeated in shell/Python drivers. The existing +S3 CLI already sends authenticated HTTP requests to a selected cluster, but +does not observe server resource usage. Iceberg lacks an equivalent reusable +upload regression. These differences make storage and access measurements hard +to reproduce or relate to one another. + +The current Stream measurements demonstrate why deployment and CPU conditions +belong with every result: a client and services sharing CPUs 0–3 behave very +differently from the same binaries allowed on CPUs 0–31. Chunk-KV uses Stream +for its WAL, so Stream waits must also be observable in the KV workload. Object +performance needs its own measurements above ChunkIO. Neither an isolated +Stream microbenchmark nor an HTTP upload result describes all these layers. + +This requirement unifies the service benchmark regressions under +`tools/benchmark/`, including RPC, KV, DiskDB, ChunkDB, ChunkIO/object, +Chunk-Stream, Chunk-KV, S3, and Iceberg. Relevant contracts are the +[OCI cluster lifecycle](../../container/oci-image/README.md), +[Stream design](../design/chunkds/design-crowdb-chunk-stream.md), +[Chunk-KV design](../design/chunkds/design-crowdb-chunk-kv.md), +[object writer design](../design/chunkio/design-crowdb-chunkio-small-object-writer.md), +and [upload flow](../design/access-server/iceberge/design-crowdb-iceberg-upload-flow.md). + +#### Solution + +The default regression deployment is a **three-node cluster with one node per +container**. A host-side `crowdb-cli bench` process is the external benchmark +tool. Publish the test servers' required ports and advertise endpoints reachable +by that client. All service regression drivers adopt this deployment and CLI +path; the current native Stream fixture remains only until this work lands. + +Responsibilities are explicit: + +- Existing OCI/cluster lifecycle code builds and starts the three containers, + prepares storage, checks full application readiness, exposes endpoints, and + stops only the invocation's owned resources. +- `crowdb-cli bench` owns workload execution, sampling, resource/metrics + collection, accounting, validation, and the common structured report. Reuse + common-library system counters and existing management/status APIs. +- Each driving script selects cases and deployment/CPU profiles, invokes the + external CLI, retains its exit status and artifacts, and records measured + results in that same script's comments. Do not maintain separate tracked + result documents or baseline JSON files, or duplicate collectors in scripts. + +The following invariants define the benchmark: + +- **I1 — Actual service path.** The external CLI reaches the selected cluster + over published ports. Storage workloads use the deployed services; S3 and + Iceberg use authenticated HTTP. The CLI does not create an in-process cluster + as a substitute. A successful write means the layer's configured durability + and publication contract is satisfied. +- **I2 — Controlled input and timing.** Record payload seed, size, concurrency, + count/duration, warmup, read-cache policy, and media/durability profile. Bound + client payload memory independently of large object size. Separate cluster + setup, dataset preparation, warmup, measurement, drain, verification, and + cleanup. A duration-limited case cannot silently finish early due to a small + operation limit. Timed 4-KiB single-/32-producer write cases run at least + 60 seconds by default; short pipeline smoke tests are labelled separately. +- **I3 — Complete accounting.** Track admitted, completed, failed, and incomplete + operations, drain admitted work, and verify deterministic bytes outside the + timed interval. A timeout, missing result/required metric, failed cleanup, + or unverified write fails the case. Missing cases cannot disappear from a + passing summary. Bounded memory and queue limits remain regression gates. +- **I4 — Attributable measurements.** Identify client, all three nodes, service + instances, and measurement windows. Counts and sums are interval deltas; + interval percentiles come from matching samples/histogram buckets, never + subtraction of cumulative percentiles. Do not average node percentiles into + a cluster percentile. Concurrent phases overlap and cannot be summed as if + they partitioned request latency. Counter resets or replaced processes are + explicit. Optional unavailable counters are `unsupported`, never zero. +- **I5 — Comparable conditions.** A result includes exact commands and software, + image, topology, CPU, runtime, storage, and durability identities. Compare + only matching profiles with documented tolerances. Default placement and a + constrained cpuset are separate profiles; native MemDisk and container disk + results are also distinct. The accepted implementation is the baseline; + the runner does not build or run an older binary to produce a comparison. +- **I6 — Owned three-node deployment.** Readiness requires all three members + and usable storage. Port conflicts, missing peers, or unreachable advertised + endpoints fail before measurement, without singleton/native fallback. + Failure/interruption retains diagnostics and cleans only resources owned by + this invocation. Image builds, compilation, tests, and other workloads run + outside performance measurement windows. +- **I7 — Results live with the driver.** Record the dated measured summary and + its conditions directly in each driving script. The CLI can emit JSON/TSV + and raw artifacts into ignored runtime directories and generate a comment + block for the driver. Retain useful previous measurements there with their + profile/revision so comparisons have context. Do not create an additional + tracked result file; optional raw traces do not replace the script summary. + +Work items: + +1. Reuse `container/oci-image/` and the existing CLI cluster lifecycle to run + the owned three-container cluster. Publish HTTP and the management/RPC/data + endpoints needed by each selected workload; verify client reachability of + every address returned by discovery, routing, and redirects. Supply the + accepted public origins when creating the deployment. The CLI runs outside + all node containers, using explicit target bindings and private credentials. + Record exact container/image identities and port bindings. Do not implement + another bootstrap or silently attach to an unrelated running cluster. +2. Refine the commands under `app/crowdb-cli/src/commands/bench/`, extending its + existing `metrics` and `result` responsibilities and reusing workload code + in `lib/crowdb-console-shared/src/ops/` and owning storage libraries. Add + Stream and Chunk-KV entry points where absent; move the reusable Stream + workload behind the CLI. Preserve S3's existing real HTTP path and extend + it for bounded large uploads and multipart. Add the Iceberg FileIO adapter. + All workload families share accounting, collector lifetimes, output schema, + and failure handling instead of parsing human-readable metrics in scripts. +3. Implement one CLI collection lifecycle around preparation, measurement, + drain, and teardown. Take service snapshots before/after the measured window + and flush final client metrics. Use bounded histograms and existing periodic + system metrics; make per-operation traces optional. Report collection cost + and window boundaries, with no request-path file/process polling. Retrieve + server process/cgroup evidence through the owned deployment or management + APIs, mapping container identities explicitly; never interpret a container + PID as a host PID or report CLI-only CPU as cluster CPU. +4. Capture the report contents and workload matrix below. Reuse shared system + counters for CPU and C6; do not add another hardware counter implementation + to the scripts or CLI. Permission failures leave optional fields explicitly + `unsupported`; default benchmarks need no extra perf privileges or power + policy changes. A missing required workload/phase field still fails. +5. Convert every service regression driver under `tools/benchmark/` to the + three-node container plus external CLI path, retaining case filters, bounded + timeouts, and complete failure output. The scripts carry named case/gate + configuration and measured result comments; collection, latency math, and + report validation belong in the CLI. Preserve correctness gates during the + conversion. Recapture results for the new container profile rather than + treating native measurements as directly comparable. Support an explicitly + selected remote cluster through the same CLI, with unavailable local-host + measurements marked `unsupported` and no remote deployment cleanup. + +Required report contents: + +- **Conditions:** UTC date; CLI command; source revision and dirty status; + client/service binary hashes and image digest; host CPU model, kernel, + physical/logical cores, SMT/NUMA topology; three node identities, placement, + ports, mirror/EC policy, media type/device, KV/WAL backend, fsync/durability, + dataset/cache policy, and runtime/RPC worker counts. Retained MemDisk, null + data sources, sparse files, and real disks must be named accurately. +- **CPU placement and usage:** requested and effective cpusets for client, + containers, and each service's worker threads; cgroup CPU quota/period and + throttling deltas; client and per-node/service user/system CPU seconds, + average CPU %, RSS/peak/growth, context switches and migrations where + available. State that 100% means one logical CPU. Separate timed-window + usage from whole-command usage that includes dataset preparation. +- **System evidence:** disk and network bytes/operations and resource pressure + when available; per-core C6 and per-package C6 residency with units, elapsed + interval and availability. Counter support/permission/reset states stay + distinct. Optional detailed perf profiles have explicit collection settings + and their own comparable profile; no per-operation privileged probes. +- **Workload outcome:** logical/physical bytes, operations/s and MiB/s, actual + duration, admission/drain counts, errors/retries/timeouts, error classes, + count/mean/p50/p95/p99/max latency and a mergeable histogram. State sample + counts and resolution; omit unsupported percentiles with a reason. Separate + client payload generation/signing/hash work from transfer/server stages. +- **Latency attribution:** queue depth/bytes and wait; RPC round-trip and + response scheduling; append batch/mirror waits; asynchronous capacity and + candidate preparation versus foreground wait; seal/rollover, StreamChunkRecord + publication/build/get/scan/CAS, checkpoint and replay; cache/physical reads + and reclaimed bytes. Chunk-KV adds WAL append/commit, tree read/write, + flush/snapshot/compaction, and read-cache evidence. Object adds admission, + batching, reservation/preparation, payload write, parity, and metadata + publication. Use the actual available stage counters and identify required + fields by workload; capture count, sum, distribution, and max separately. +- **Protocol stages:** S3/Iceberg direct PUT, UploadPart, multipart completion, + and optional table commit are separate operations. A successful part does + not count as a published object. Upload completion includes protocol metadata + publication; FileIO latency does not include an unrelated snapshot commit. + +Minimum workload matrix: + +- **Existing RPC/KV/DiskDB/ChunkDB cases:** retain their meaningful workload and + correctness coverage while moving startup and measurement to the common path. +- **Stream:** preserve the current seven cases: 4-KiB append at 1/32 producers, + 1-MiB append at 8, 4-MiB rollover, random 4-KiB read at 32, 320-MiB replay, + and 576-MiB GC. Ordinary append must not checkpoint the cursor; metadata + publication scales with chunk transitions, not historical record count. +- **Chunk-KV:** 4-KiB values at 1/32 clients for put/get, plus batch and ordered + scan; record key/value sizes, batch size, read/write mix and cache state. + Include a run crossing WAL chunk rollover and a bounded restart/replay case. + A sustained case must expose background flush/snapshot/compaction work and + cannot end before queued durable work drains. Correlate end-to-end KV latency + with its Stream WAL metrics in the same report. +- **Object/ChunkIO:** small and large write/read plus mixed reads through the + existing object writers/readers; include 4-KiB at 1/32 clients, 1-MiB at 8, + and the configured small/large boundary. Record physical write amplification, + batching, placement and verification rather than inferring performance from + Stream or KV alone. +- **S3/Iceberg:** small-path and threshold-boundary uploads, direct 100-MiB PUT, + and multipart 100-MiB upload, at single and concurrent clients. Reuse bounded + deterministic payload blocks and precompute expected integrity data outside + timing; record the integrity mode and part size. Preserve S3 read/range/list + and mix coverage through the same collector. Final bounded byte/range checks + occur outside upload timing. + +#### Dependencies + +- Existing three-node OCI cluster lifecycle, readiness, target bindings, and + port/public-origin configuration. Add missing external RPC/data exposure in + that lifecycle; an unreachable route blocks the affected benchmark rather + than falling back to native services. +- Existing storage benchmark workloads, Stream/Chunk-KV APIs, object writers + and readers, and S3/Iceberg authentication/publication paths. Reuse these + contracts and existing native results as recorded evidence; this requirement + owns their CLI/container regression conversion and new profile measurements. +- Shared metrics/system-status counters and management snapshots provide + collection. Hardware-dependent counters remain optional if unavailable; + required stage coverage is implemented or its case fails explicitly. +- Official-client compatibility remains in the existing E2E suites. A + client-inclusive profile can use the common result schema without replacing + the controlled CLI workload or mixing their results. + +#### Acceptance + +- Given an owned three-container cluster, run every converted driver with the + host CLI; assert three ready members, published reachable endpoints and real + service requests, with no native/in-process replacement (I1, I6). E2E test. +- Given a port collision, absent third member, or an unreachable advertised + RPC/HTTP address, start a case; assert failure before timing, useful + diagnostics, and no singleton fallback (I1, I6). E2E test. +- Given an existing unrelated cluster and an interrupted/failed benchmark, + stop the run; assert only owned containers/data are cleaned under the declared + retention policy and all failure logs remain available (I3, I6). E2E test. +- Given each minimum workload and fixed deterministic inputs, execute through + the CLI; assert complete operation/byte accounting, bounded memory/queues, + real stored-byte verification, and a common report (I1–I4). E2E test. +- Given S3 and Iceberg multipart uploads, complete both; assert distinct part, + completion and optional table-commit measurements, correct integrity checks, + and no part counted as a final object (I1, I3, I4). E2E test. +- Given a 60-second 4-KiB case and a small operation cap, execute the duration + profile; assert it cannot receive a passing full-duration verdict early and + that setup/warmup/drain have distinct windows (I2, I3). Integration test. +- Given permission-denied hardware counters and remote endpoints, collect + metrics; assert per-core and package C6/local-resource availability is + `unsupported` where appropriate, with no fabricated zeros or default sudo + requirement (I4). Integration test. +- Given independent client/container cpusets and a throttled node, run a case; + assert effective per-service/thread placement, quota, throttling, CPU time, + RSS and timing windows are attributed correctly (I4, I5). E2E test. +- Given overlapping stage histograms, low sample counts, process replacement + and counter reset, build the report; assert interval aggregation is valid, + unsupported percentiles are identified, and neither percentile subtraction + nor exclusive summation of overlapping phases occurs (I4). Unit test. +- Given Chunk-KV WAL rollover/replay and object small/large-boundary workloads, + run the CLI cases; assert their own verified outcomes, latency distributions, + Stream WAL/object stages, and drained durable work (I1–I4). E2E test. +- Given a failed request, missing phase/result/case, malformed report, timeout + or cleanup failure, summarize the run; assert nonzero status and retained + diagnostic evidence instead of a partial passing result (I3, I4). Integration test. +- Given matching and mismatched measurement profiles, assess recorded results; + assert only matching profiles receive a tolerance verdict and that the current + baseline does not require running an older binary (I5). Unit test. +- Given a completed run, generate its driver comment block; assert it includes + results, date, exact command, image/revision, cpusets, CPU usage and artifact + references, and can update the driving script without another tracked result + file. Assert scripts delegate collection/validation to the CLI (I7). Integration test. + +Run `pixi run rs-fmt-check`, `pixi run rs-lint`, +`pixi run cargo test -p crowdb-console-shared`, +`pixi run cargo test -p crowdb-cli`, and +`pixi run test-container-e2e-fixture`. +Run each converted service regression serially through +`pixi run bash tools/benchmark/bench-chunk-stream-regression.sh`, +`pixi run bash tools/benchmark/bench-chunk-kv-regression.sh`, +`pixi run bash tools/benchmark/bench-chunkio-write-regression.sh`, +`pixi run bash tools/benchmark/bench-chunkio-small-write-regression.sh`, +`pixi run bash tools/benchmark/bench-chunkio-read-regression.sh`, +`pixi run bash tools/benchmark/bench-rpc-regression.sh`, +`pixi run bash tools/benchmark/bench-kv-write-regression.sh`, +`pixi run bash tools/benchmark/bench-kv-read-regression.sh`, +`pixi run bash tools/benchmark/bench-kv-scan-regression.sh`, +`pixi run bash tools/benchmark/bench-diskdb-regression.sh`, and +`pixi run bash tools/benchmark/bench-chunkdb-regression.sh`. +Verify the S3/Iceberg adapters with `pixi run test-e2e-s3` and +`pixi run test-e2e-iceberg`, including their new CLI regression cases. diff --git a/doc/backlog/R207-chunkdb-repo-metadata-chunk-kv.md b/doc/backlog/R207-chunkdb-repo-metadata-chunk-kv.md index fab99bc7b..0703d6da3 100644 --- a/doc/backlog/R207-chunkdb-repo-metadata-chunk-kv.md +++ b/doc/backlog/R207-chunkdb-repo-metadata-chunk-kv.md @@ -47,13 +47,13 @@ or runtime correctness is claimed by this audit. direct-KV clients. Allocation, updates, listing, reservations, task scanning and owner queries have no complete repo backend path. A shared backend switch would not establish the required separate system/repo operation logic. -- **Recursive dependency:** chunk-kv's journal uses chunk-stream, whose - `MirrorChunkWriter` currently allocates `Stream`. Routing all Stream chunks - to chunk-kv before distinguishing system `Wal` would make journal metadata - depend on the storage it must recover. Owner-key validation and reopen also - currently expect Stream. The binding's `owner_kind="chunk-kv-partition"` - does not itself change the allocated type. See - [mirror writer](../../lib/crowdb-chunk-client/src/chunk/mirror_chunk_writer.rs), +- **Recursive dependency:** chunk-kv's journal uses chunk-stream. Allocation, + owner validation and reopen now preserve an explicit system `Wal` or business + `Stream` purpose. Backend dispatch must preserve that distinction: routing + system WAL metadata to chunk-kv would make journal metadata depend on the + storage it must recover. The binding's `owner_kind="chunk-kv-partition"` + does not itself select the allocated type. See + [Stream allocation](../../lib/crowdb-chunk-stream/src/production_chunk/allocation.rs), [owner validation](../../lib/crowdb-protocol/src/chunk_stream.rs) and [storage assembly](../../app/crowdb-chunk-kv-server/src/storage.rs). - **Atomic publication:** chunk creation writes a Chunk, finalize task and diff --git a/doc/backlog/R83-chunkdb-complete-recovery-flow.md b/doc/backlog/R83-chunkdb-complete-recovery-flow.md index 9c04ec59f..cfff04f5e 100644 --- a/doc/backlog/R83-chunkdb-complete-recovery-flow.md +++ b/doc/backlog/R83-chunkdb-complete-recovery-flow.md @@ -48,7 +48,7 @@ `chunkdb.fbs` (`Chunk`, `ChunkStrip`, `MirrorStrip`, `EcStrip`, `StripType`, `ECState`, `ChunkState`, `ChunkdbService` — `AllocateChunk` / - `AppendChunk` / `SealChunk` / `DeleteChunk` / `UpdateChunkStrip` / + `AppendChunk` / `SealChunk` / `DeleteChunk` / `ReplaceChunkStripRange` / `ListChunks`), `diskio.fbs` (`DiskWrite` / `DiskRead`). CROWDB's chunkdb owns disk-failure recovery: it rebuilds lost blocks from mirror/EC at the chunk layer, driving rebuild I/O and pacing it. @@ -60,7 +60,7 @@ disks, reads the surviving data via `diskio` `DiskRead`, allocates a new segment on a healthy disk via diskdb `AllocateBlocks`, writes the rebuilt replica via `diskio` `DiskWrite`, and updates the - chunk's strip (`UpdateChunkStrip`) to reference the new `Segment`. + chunk's strip (`ReplaceChunkStripRange`) to reference the new `Segment`. The old segment on the `Bad` disk is freed (`FreeBlocks`) once the disk is replaced/removed. The chunk is whole again. - **EC data-block reconstruction** — an EC strip (`data_num` + `code_num`) @@ -105,7 +105,7 @@ 1. **chunkdb server component (prerequisite)** — `app/crowdb-chunkdb` (new crate, not yet existing) implementing `ChunkdbService` (`AllocateChunk` / `AppendChunk` / `SealChunk` / - `DeleteChunk` / `UpdateChunkStrip` / `ListChunks`). This is a + `DeleteChunk` / `ReplaceChunkStripRange` / `ListChunks`). This is a prerequisite for R83, not part of it — it should be filed as its own backlog item. R83 assumes chunkdb exists and owns the chunk→strip→segment mapping persisted to CROWDB KV. @@ -124,7 +124,7 @@ affinity against the `Bad` disk). Prerequisite: the `diskio` server component (also unbuilt; should be its own backlog item). 4. **Strip update + old-segment free** — after a rebuilt block is - durably written to its new segment, `UpdateChunkStrip` the chunk's + durably written to its new segment, `ReplaceChunkStripRange` the chunk's strip to reference the new `Segment`, then `FreeBlocks` the old segment on the `Bad` disk (after the disk is replaced/removed, or once the strip no longer references it). @@ -167,7 +167,7 @@ diskdb AllocateBlocks (healthy disk, exclude Bad) ──► diskio DiskWrite (new segment) │ ▼ - UpdateChunkStrip (new Segment) ──► FreeBlocks (old segment on Bad disk) + ReplaceChunkStripRange (new Segment) ──► FreeBlocks (old segment on Bad disk) │ ▼ persist per-chunk/strip progress (item 6) ──► next strip / next chunk @@ -223,7 +223,7 @@ replicas (one on the `Bad` disk, one on a healthy disk) → chunkdb reads the surviving replica via `diskio`, allocates a new segment on a healthy disk (excluding the `Bad` disk), writes the rebuilt - replica, `UpdateChunkStrip`s to the new `Segment`, and frees the old + replica, `ReplaceChunkStripRange` updates to the new `Segment`, and frees the old segment → the chunk reads back correctly from the new replica. E2E test (pending chunkdb + diskio). - **EC data-block reconstruction**: an EC strip (`data_num=4`, diff --git a/doc/backlog/backlog.md b/doc/backlog/backlog.md index fd14c6ed2..fe4879e96 100644 --- a/doc/backlog/backlog.md +++ b/doc/backlog/backlog.md @@ -9,7 +9,7 @@ requirements and their temporary execution plans are removed during the final live in the linked document; implementation follows [`implement-requirement`](../../.agents/skills/implement-requirement/SKILL.md). -**Next R number: R238** — R221 and R234 are reserved by other workstreams. +**Next R number: R245** ## Dataset and access @@ -38,9 +38,10 @@ live in the linked document; implementation follows - **[R186](R186-access-iceberg-orc-validation.md)** — selected ORC validation. **Area:** access server / Iceberg. **Complexity:** Medium. **Status:** Deferred as an independent follow-up by user decision. -- **[R196](R196-access-upload-benchmark-regression.md)** — S3 and Iceberg HTTP - upload benchmark regression. **Area:** access server / CLI / benchmarks. - **Complexity:** Medium. **Dependencies:** completed upload correctness paths. +- **[R196](R196-refine-benchmark-tests.md)** — refine benchmark tests: external + CLI workloads and shared collection against a three-node container cluster; + record results in each driver. **Area:** CLI / storage / access / benchmarks. + **Complexity:** Medium. **Dependencies:** existing OCI and service contracts. ## Chunk, Chunk-KV, and recovery diff --git a/doc/design/access-server/design-crowdb-access-server.md b/doc/design/access-server/design-crowdb-access-server.md index 14988f811..b57b3b0fd 100644 --- a/doc/design/access-server/design-crowdb-access-server.md +++ b/doc/design/access-server/design-crowdb-access-server.md @@ -207,6 +207,31 @@ path. Readiness observes the same storage dependencies and cleanup backlog as the data dispatcher. Retained launch records identify the dedicated health address; a missing address requires deployment reconciliation before restart. +One Access background sampler runs every second and publishes an immutable +health snapshot through ArcSwap. It independently checks the routed Chunk-KV +bucket namespace, ChunkDB and online disks through the process's storage clients. +Metadata and chunk checks have separate deadlines inside an 800 ms total bound; +only one sampling round can run at a time. Physical checks read existing disk +bytes without creating objects, allocating chunks or repairing storage. +Credential-cache freshness, listener state, native-pool pressure and cleanup +backlog are included in the same snapshot. Application requests only record +their own errors and metrics; their outcomes cannot overwrite health evidence. + +Health requests read the snapshot without dependency I/O or a new sampling task. +Responses retain the check timestamp, generation, age, exact flags and diagnostic. +New observations replace old failures or successes. Evidence older than three +seconds, or an absent first sample, reports Unknown and HTTP 503; readers never +extend its validity or clear it to Ready. Process stop is an immediate lifecycle +fence even when the cached sample was healthy. The sampler is cancelled with the +listener. Recovery needs neither application traffic nor an Access restart. + +Web independently samples local monitor evidence, signed protocol reads and +service diagnostics in its backend. Its node-observation endpoint only returns +that bounded-age snapshot, including original operation identities and producer +timestamps. Status readers aggregate cached member evidence with current +committed intent, so stop/delete and ownership fences override older samples. +Frontend query failures do not mutate these backend health observations. + ## 8. Correctness invariants - **AS-I1 — First-class models:** S3, Iceberg, and Dataset each own their diff --git a/doc/design/chunkdb/design-crowdb-chunkdb-rpc.md b/doc/design/chunkdb/design-crowdb-chunkdb-rpc.md index 740ff028c..b8896359f 100644 --- a/doc/design/chunkdb/design-crowdb-chunkdb-rpc.md +++ b/doc/design/chunkdb/design-crowdb-chunkdb-rpc.md @@ -3,277 +3,109 @@ # CROWDB - Design: ChunkdbService RPC -Depends on: [`design-crowdb-rpc.md`](../rpc/design-crowdb-rpc.md) §4.4, §5, §6; [`design-crowdb-chunkdb.md`](design-crowdb-chunkdb.md) §4 -Satisfies: [`design-crowdb-chunkdb.md`](design-crowdb-chunkdb.md) §4 (RPC layer) +Depends on: [RPC transport](../rpc/design-crowdb-rpc.md), +[ChunkDB lifecycle](design-crowdb-chunkdb.md). +Satisfies: ChunkDB control-plane transport, routing and mutation fences. -The chunkdb service (Allocate/Append/AdvanceWrite/Query/Seal/Delete/DeleteRange/ -UpdateStrip/ListChunks plus replacement allocate/swap/discard) uses the -**crowdb-rpc flatbuffer transport** — the -same engine as the KV consensus hot path, on a dedicated port with a -dedicated schema (`chunkdb.fbs`). The transport is selected -programmatically (`ChunkdbClient::with_rpc_transport`). Architecture -decisions and rationale live in the root RPC design; this doc covers the -chunkdb-specific schema, handler, and transport detail. +ChunkDB uses the crowdb-rpc FlatBuffer transport for lifecycle, placement, +reservation, replacement, conversion and repair operations. Clients and services +use one current schema. Unknown messages and malformed requests fail explicitly. ## Table of Contents -- [1. Port Allocation](#1-port-allocation) -- [2. Flatbuffer Schema (chunkdb.fbs)](#2-flatbuffer-schema-chunkdbfbs) -- [3. Zero-Copy Wrapper Classes](#3-zero-copy-wrapper-classes) -- [4. Server-Side Handler (ChunkdbRpcService)](#4-server-side-handler-chunkdbrpcservice) -- [5. Client-Side Transport (ChunkdbRpcTransport)](#5-client-side-transport-chunkdbrpctransport) -- [6. ChunkdbClient Transport Selection](#6-chunkdbclient-transport-selection) -- [7. Server Wiring](#7-server-wiring) -- [8. Error Model](#8-error-model) - ---- - -## 1. Port Allocation - -The chunkdb RPC port is `CHUNKDB_RPC_BASE = 9961` (stride 1, paired -with HTTP 9972). - -```rust -pub const CHUNKDB_RPC_BASE: u16 = 9961; -``` - -Fills the gap between diskdb RPC (9931-9940) and chunkdb HTTP -(9972). Stride 1 (one port per instance). The `ChunkdbRpc` variant -in `ServicePort` provides `base() => CHUNKDB_RPC_BASE`, `stride() => 1`. - -## 2. Flatbuffer Schema (chunkdb.fbs) - -File: `lib/crowdb-protocol/src/fbs/chunkdb.fbs` - -Includes `common_type.fbs` (provides `FBInt128`) and `diskdb.fbs` -(provides `FBSegment` as an inline struct in the `crowdb.diskdb` -namespace). - -**Enums:** - -- `FBChunkdbRetCode` — Success, InvalidArgument, NotFound, AlreadyExists, - FailedPrecondition, Aborted, Internal, Unavailable, NotMyRange, - StripIndexOutOfRange -- `FBEcState` — NoParity, Parity -- `FBChunkState` — Init, Active, Sealed, Deleted -- `FBStripType` — Mirror, Ec -- `FBChunkType` — Wal, BtreePage, PageIndex, Stream, S3, IcebergTable. - Wire value zero is Reserved and rejected; allocation requires explicit purpose. - Chunk ID prefix, stored purpose, and stream owner attribution must agree. - -**Nested types:** - -- `FBMirrorStrip` — `segments:[FBSegment]` -- `FBEcStrip` — `data_num`, `code_num`, `ec_state`, `segments:[FBSegment]` -- `FBStripBody` union — None, Mirror, Ec (the strip body union) -- `FBChunkStrip` — `chunk_offset`, `strip_sequence`, `unit_kb`, - `capacity`, `create_ts_ms`, `sealed_ts_ms`, `sealed_length`, - `strip_type`, `strip_body:FBStripBody`, `usage_bitmap:[ubyte]`, and - unavailable segment identities -- `FBChunk` — `id:FBInt128`, `state`, `create_ts_ms`, `sealed_ts_ms`, - `capacity`, `sealed_length`, `strips:[FBChunkStrip]`, `chunk_type`, - monotonic next sequence, cleanup intents, and last replacement operation - -**Request/response tables:** - -All 12 request types follow the same shape: `id` + `rpc_create_nano` -first, then the schema fields. All 12 response types carry `id` + -`rpc_create_nano` + `ret_code` + `error_msg` + `range_start` + -`range_end` (diagnostic for `NotMyRange`) + the response data. - -The `range_start`/`range_end` fields are on every response (default 0) — -they are only populated when `ret_code = NotMyRange`. This avoids a -separate `NotMyRangeHint` message. - -`UpdateChunkStripRequest` embeds a `FBChunkStrip` (the `strip` field). -`ListChunksResponse` carries `chunks:[FBChunk]` + -`next_token:FBInt128`. `QueryChunkResponse` also advertises the reader layout -validity duration. Replacement requests carry tentative segment geometry, -failed-disk exclusions, exact old/new strip ranges, expected revision, and a -stable operation identity. - -**Message type IDs** (3300-3323 range): - -- `EAllocateChunkRequest = 3300` … `EListChunksResponse = 3315` -- `EAdvanceChunkWriteRequest = 3316` … `EAdvanceChunkWriteResponse = 3317` -- `EAllocateReplacementSegmentRequest = 3318` … - `EDiscardReplacementSegmentResponse = 3323` - -**Build + re-exports:** - -`build.rs` adds `chunkdb.fbs` to `fbs_files` + a `flatc --rust --gen-all` -invocation (inlines `common_type.fbs` + `diskdb.fbs` so `FBInt128` + -`FBSegment` resolve). `lib.rs` adds the `chunkdb_generated` private -module + `chunkdb_fb` public re-export module. - -## 3. Zero-Copy Wrapper Classes - -File: `lib/crowdb-protocol/src/fb_wrappers/chunkdb.rs` - -Frequently accessed responses use a zero-copy `Ref` wrapper. Each follows the -zero-copy pattern: parse the root pointer once, read fields through it -without copying. - -```rust -pub struct FBAllocateChunkResponseRef<'a> { - root: Option>, -} -``` - -Each `Ref` provides: `new`, `valid`, `ret_code`, `error_msg`, -`request_id`, `ok`, `range_start`, `range_end`, plus per-response data -accessor (e.g. `chunk()` for chunk-carrying responses, `chunks()` + -`next_token()` for `ListChunksResponse`). - -Invalid buffer → `valid()` returns false, `ret_code()` returns -`Internal`, all data accessors return `None`/0. - -## 4. Server-Side Handler (ChunkdbRpcService) - -File: `app/crowdb-chunkdb/src/service/chunkdb_rpc_service.rs` - -The `ChunkdbRpcService` struct holds `Arc` + a -`tokio::runtime::Handle`. The handler delegates to `LifecycleHandler` -for all 12 RPCs — same lifecycle logic, different -wire format. No transparent forwarding (chunkdb has no leader — range -routing is client-side via `RangeBindingClient`). - -`register_handlers` wires one handler per request `msg_type` into the -`RpcServer`, using the `make_handler` closure pattern. Each handler: - -a. Extract `request_id`, `rpc_create_nano`, `msg_type` from - `ServerRequest`. -b. Parse the request flatbuffer via `flatbuffers::root::Request>`. - On parse failure, submit an error response with - `ret_code = InvalidArgument`. -c. Extract fields from the request (chunk_id, strip_type, etc.). -d. Spawn a tokio task (via `Handle::spawn`) that calls the - `LifecycleHandler` method. -e. On success, build the flatbuffer response with `ret_code = Success` - + the chunk data. -f. On `LifecycleError::NotMyRange`, build the response with - `ret_code = NotMyRange` + `range_start`/`range_end`. -g. On other errors, map `LifecycleError` to `FBChunkdbRetCode` + - `error_msg`. -h. Submit via `server.submit_response`. - -**Response builders:** one `build__response` function per -response type. The `Chunk` → `FBChunk` conversion builds nested -`FBChunkStrip` + `FBMirrorStrip`/`FBEcStrip` + `FBSegment` vectors — -the most complex part since the `Chunk` flatbuffer has nested `Vec` -with a `oneof strip` field. - -## 5. Client-Side Transport (ChunkdbRpcTransport) - -File: `lib/crowdb-chunkdb-client/src/rpc_transport.rs` - -```rust -pub struct ChunkdbRpcTransport { - server: Arc, - rpc: Arc, - connections: ConnectionPoolIndex, - next_req_id: AtomicU64, -} -``` - -The `RpcServer` is the client-side transport — it does not listen but -establishes connections to remote endpoints. `conn_for(endpoint)` normalizes -the endpoint and selects from an immutable generated pool. Cold connections -are established outside the index and installed with compare-and-swap; -retryable failures invalidate only the selected generation. - -12 `send_*` methods (one per RPC): build request flatbuffer → -`rpc.call(&server, &conn, req_id, control, None, msg_type)` → -`fut.await` → parse response via `Ref` wrapper → map to the response -type → return. - -The response types are still returned — the caller's retry + -`RangeBindingClient` logic is unchanged; only the wire send changes. - -## 6. ChunkdbClient Transport Selection - -File: `lib/crowdb-chunkdb-client/src/client.rs` - -`ChunkdbClient` holds an `Option>` field, set -via `with_rpc_transport`. Each public method (`allocate_chunk`, -`append_chunk`, `query_chunk`, `seal_chunk`, `delete_chunk`, -`delete_chunk_range`, `update_chunk_strip`, `list_chunks`) checks -`self.rpc_transport` first: - -a. When set, delegate to the transport's `send_*` method via - `with_rpc_retry`: resolve endpoint via `range_binding.route(chunk_id)` - or `first_endpoint()`, call `transport.send_*(endpoint, req)`, on - `NotMyRange` refresh binding + re-route, on transient error retry - with backoff. - -The `with_rpc_retry` helper implements the retry loop — retry semantics -and `RangeBindingClient` integration, with the crowdb-rpc wire send. - -## 7. Server Wiring - -File: `app/crowdb-chunkdb/src/main.rs` - -The chunkdb server runs inside `#[tokio::main]` so -`tokio::runtime::Handle::current()` is available. The crowdb-rpc server -is the chunkdb RPC transport. - -Startup sequence: - -1. Load config from TOML. -2. Build KV client for group-0 topology access. -3. Create topology cache + refresh loop + notify handler. -4. Validate complete fixed service/storage slot maps and install caches. -5. Create independent system and user-data lifecycle/task runtimes. -6. Spawn service endpoint/binding refresh without changing ownership. -7. Register service-registry keep-alive. -8. Create diskdb client pool + chunk allocator. -9. Create per-chunk lock map + metrics. -10. Spawn sweep task for idle lock reaping. -11. Create lifecycle handler. -12. Create crowdb-rpc server: parse `rpc_listen_addr`, `RpcServer::new`, - `listen`, `start`, `ChunkdbRpcService::new(handler, Handle)`, - `register_handlers`. -13. Start HTTP health + metrics + cache invalidation server. -14. On shutdown: stop crowdb-rpc server via shutdown signal. - -The `handler` is shared between the crowdb-rpc service — same -`Arc`. - -The `rpc_listen_addr` config field (default `0.0.0.0:9961`) is -validated as a `SocketAddr` in `ChunkdbConfig::validate`. - -## 8. Error Model - -**`LifecycleError` → `FBChunkdbRetCode` mapping:** - -- `InvalidStateTransition` → `FailedPrecondition` -- `ChunkNotFound` → `NotFound` -- `ChunkAlreadyExists` → `AlreadyExists` -- `StateConflict` → `Aborted` -- `Allocation` / `Storage` → `Internal` -- `InvalidRequest` → `InvalidArgument` -- `NotMyRange { bucket }` → `NotMyRange` + `range_start`/`range_end` -- `LockBusy` / `LockTimeout` → `Unavailable` -- `StripIndexOutOfRange` → `StripIndexOutOfRange` - -**`FBChunkdbRetCode` → `ChunkdbClientError` mapping** (client side): - -- `Success` → Ok -- `InvalidArgument` / `StripIndexOutOfRange` → `InvalidArgument` -- `NotFound` → `NotFound` -- `AlreadyExists` → `AlreadyExists` -- `FailedPrecondition` → `FailedPrecondition` -- `Aborted` → `Aborted` -- `Internal` → `Internal` -- `Unavailable` → `Unavailable` (transient — retryable) -- `NotMyRange` → `NotMyRange` (transient — refresh binding + re-route) - -**`RpcError` → `ChunkdbClientError` mapping:** - -- All `RpcError` variants → `ChunkdbClientError::Rpc` (not retryable - by default; connection-level errors surface as `Unreachable` during - endpoint resolution) - -`NotMyRange` is a response, not an error — the client checks `ret_code` -and re-routes. The `is_transient` predicate covers `Unavailable`, -`DeadlineExceeded`, `Unreachable`, and `NotMyRange`. +- [1. Transport and routing](#1-transport-and-routing) +- [2. Wire records](#2-wire-records) +- [3. Mutation contracts](#3-mutation-contracts) +- [4. Dispatch and responses](#4-dispatch-and-responses) +- [5. Errors and retry](#5-errors-and-retry) + +## 1. Transport and routing + +- `ChunkdbClient` requires an `Arc` at construction. + `RangeBindingClient` resolves a chunk's service-slot owner from Group 0. + `ServiceRegistryClient` supplies live endpoints; endpoint cache refresh does + not change ownership. +- `ChunkdbRpcTransport` uses an immutable `ConnectionPoolIndex`. Cold connection + creation happens outside the index, followed by compare-and-swap installation. + Retry invalidates only the selected connection generation. +- `ChunkdbRpcService` dispatches to the independently scoped system/user-data + lifecycle and task runtimes. Each mutation checks current range authority. +- The RPC listener defaults to port 9961; HTTP health and metrics default to + 9972. The configured RPC address must parse as a socket address. + +## 2. Wire records + +- Requests start with `id` and `rpc_create_nano`, followed by operation fields. + Responses carry request identity, `ret_code`, optional `error_msg`, diagnostic + `range_start`/`range_end`, and typed result data. +- Message identities are defined by `FBMsgType`. Generated Rust/C++ definitions + derive from the same schema; native producers use the same request contract. +- `FBChunkType` requires explicit Wal, BtreePage, PageIndex, Stream, S3 or + IcebergTable purpose. Reserved zero is invalid. Chunk-ID purpose, stored type + and owner attribution must agree. Stream/Wal carry their logical Stream name; + other types have an empty owner key. +- `FBChunk` includes writer epoch, exact acknowledged byte cursor, lease deadline, + physical strips, coarse capacity/sealed geometry, cleanup intents and the last + replacement operation. `FBChunkStrip` identifies sequence, logical interval, + unit geometry, mirror/EC segments and unavailable segment identities. +- Segment identity includes disk, zone, unit range, allocation incarnation and + owning chunk. Mirror and EC bodies carry their explicit physical geometry. +- `QueryChunkResponse` also returns reader-layout validity duration. + `ListChunksResponse` carries a bounded chunk page and continuation identity. +- Response `Ref` wrappers parse one root and expose typed borrowed accessors. + Invalid buffers remain invalid; callers check validity and return codes before + accepting data. + +## 3. Mutation contracts + +- Allocation specifies purpose, writer epoch, lease and applicable owner key. + The active chunk and its finalization task are created atomically. Allocation + retries retain operation identity rather than create unrelated chunks. +- Append and strip reservation extend physical layout under their current fences. + Reservations remain hidden until confirmation. A consumed reservation prevents + conflicting seal transitions. +- `AdvanceChunkWrite` carries expected metadata revision, writer epoch, exact + byte cursor and optional closed-strip sequence. It cannot move progress backward + or beyond capacity. +- `SealChunkRequest.seal_bytes` is the exact effective byte boundary, including + zero for an empty seal. KiB geometry does not authorize extra readable bytes. + Native clients validate the returned identity, epoch, exact cursor and state. +- `ReplaceChunkStripRange` carries expected revision, exact old strips, replacement + strips and a stable operation ID. The lifecycle validates geometry and ownership, + commits new segments before publication, and retains retired segments through + cleanup grace. One-strip changes use this same fenced range contract. +- Replacement allocation and discard manage tentative segments. Conversion and + repair publish through their fenced lifecycle transitions and durable tasks. +- Delete publishes a tombstone with durable cleanup work before freeing blocks. + Whole-chunk retries are idempotent. `DeleteChunkRange` returns `Unimplemented` + until shared-object range reclamation is supported; overlap alone cannot free + a strip containing other live objects. + +## 4. Dispatch and responses + +- Each registered message has a request metric guard and typed handler. Dispatch + validates its FlatBuffer and required identities before spawning lifecycle work. +- Handlers execute on the service runtime and retain captured ownership checks + around external operations. Responses are submitted through the RPC server. +- Successful responses contain current typed state. Lifecycle errors map to a + typed return code; `NotMyRange` includes routing diagnostics for the client. +- Layout replacement, conversion and relocation share the `layout_mutation` + request metric family. Allocate, append, reservation, cursor, query, seal, + deletion, listing and ad-hoc recovery retain their own families. +- Startup validates topology and service/storage slot maps, constructs separate + domain runtimes, registers RPC handlers and starts discovery refresh plus HTTP + observability. Shutdown stops background work and transport through the runtime. + +## 5. Errors and retry + +- Invalid arguments and strip indices map to `InvalidArgument` or + `StripIndexOutOfRange`. Missing/existing chunks map to `NotFound`/`AlreadyExists`. +- Invalid lifecycle transitions map to `FailedPrecondition`; revision/operation + conflicts map to `Aborted`. Storage/allocation failures map to `Internal`. +- Lock admission failures map to `Unavailable`. Ownership failures return + `NotMyRange`, causing binding refresh and routing to the current owner. +- Client retries are bounded and apply only to transient failures. Typed fence + failures and malformed responses do not become successes. Mutations that may + have completed retain their operation identity and reconcile durable state. +- A transport response alone does not establish Stream writer ownership or a + synchronized DiskIO write; those contracts belong to their respective layers. diff --git a/doc/design/chunkdb/design-crowdb-chunkdb.md b/doc/design/chunkdb/design-crowdb-chunkdb.md index 47da4a181..f6e1a931a 100644 --- a/doc/design/chunkdb/design-crowdb-chunkdb.md +++ b/doc/design/chunkdb/design-crowdb-chunkdb.md @@ -708,9 +708,9 @@ Init ──> Active ──> Sealed ──> Deleted **State transitions**: - `Active → Sealed`: Via `SealChunk` RPC. Validates state and records the exact - `seal_bytes` boundary in `acknowledged_cursor`. KiB `sealed_length` fields are - compatibility summaries; they do not enlarge the valid byte range. Legacy - KiB-only requests retain their original contract. Finalization preserves the + `seal_bytes` boundary in `acknowledged_cursor`, including zero for an empty + chunk. The RPC has one exact-byte form. KiB `sealed_length` fields summarize + geometry and do not enlarge the valid byte range. Finalization preserves the exact acknowledged cursor. Data alignment and physical block allocation never add logical bytes to the seal boundary. - `Active → Deleted`: Persist cleanup intent, free segments, persist tombstone. @@ -739,9 +739,31 @@ entry and task record in one task-partition transaction; no second expiry is stored in chunk metadata. The owner self-fences before that authority can expire. The task scanner reads due task-index entries only, never scans the chunk table. After claim and the shared request-age/skew safety window, it -parses frames from offset zero and seals at the last complete verified frame, -or deletes an empty chunk. A restarted owner always allocates a new Active -chunk. +requires ordered synchronization proof from every segment in the captured +layout, including ordinary empty chunks and unavailable mirror identities. +Failure retains the layout and retries without scanning or freeing blocks. +It then observes content using at-most-1-MiB windows and the shared bounded frame +parser. Stream/Wal scans include allocated capacity beyond the persisted +checkpoint and commit only complete append groups. Ordinary object publication +already persists its physical cursor; its finalization validates that prefix. +An exact-capacity end needs no extra header request. Drain and scan share one +30-second IO budget; drain has at most its first half. Observation and comparison +reserve time within the remainder rather than restarting a timeout per copy. + +Available mirror candidates run concurrently within one IO deadline. Select +the shortest healthy complete prefix and verify agreement on overlapping +contents. Failed, partial or corrupt observations do not become shorter +candidates; valid conflicting contents fail explicitly. No readable candidate +means retry while retaining the Active layout, including a Stream/Wal with a +zero checkpoint. A scan must cover the persisted `acknowledged_cursor` before +sealing or deleting. The task writer generation must match the chunk epoch. +Sealing compares the exact captured chunk metadata while holding its lifecycle +guard; a concurrent revision/layout change forces another observation. Empty +chunks first seal through that same fence, then resume tombstone publication +after restart without accessing physical segments that were already retired. +Finalization uses this durable task path exclusively; an expired metadata +checkpoint cannot independently authorize sealing. A restarted owner always +allocates a new Active chunk. ## 10. Per-Chunk-ID Lifecycle Lock + Chunk Cache @@ -930,8 +952,6 @@ mutating RPC acquires the per-chunk lock before its RMW cycle: returns `Unimplemented` without changing storage until shared-object range reclamation is supported. An intersecting strip is not permission to free its blocks while neighbouring objects remain live. -- `update_chunk_strip`: compatibility wrapper over the one-strip form of - `replace_chunk_strip_range`. - `replace_chunk_strip_range`: `check_range` → `acquire` → validate Active or Sealed state, expected revision, exact old range, contiguous capacity, monotonic sequences, segment ownership, and mirror/EC geometry → commit only @@ -972,10 +992,14 @@ pages, so an old prefix cannot starve later records. It deterministically admits a completed special group to the normal conversion task path. An expired incomplete group is cancelled and reclaimed when its consumed strips carry persisted planned cursors and the writer lease deadline plus -`lifecycle.layout_validity_ms` has elapsed. This reuse grace exceeds the data -RPC retry window, so a delayed write finishes or times out before DiskDB can -reallocate the extent. Consumed records without planned cursors remain allocated -fail-safe. DiskIO does not validate allocation ownership. Terminal reservation +`lifecycle.layout_validity_ms` has elapsed. This grace bounds late request +admission; ordered completion proof and producer exclusion are also required +before physical reuse because an expired RPC can still have backend IO in flight. +Current reservation records require explicit planned +cursor/sequence vectors; a consumed strip with a zero planned cursor is malformed +and fails before reconciliation changes storage. Placement records require +explicit policy and repair-state fields. DiskIO does not validate allocation +ownership. Terminal reservation records release their quota only after the durable record has been removed; ambiguous persistence retains the permit unless a linearizable read proves that no record exists. @@ -1102,7 +1126,6 @@ app/crowdb-chunkdb/ # chunkdb server binary │ │ └── state.rs # lifecycle state transitions │ ├── service.rs # pure service index │ ├── service/ -│ │ ├── chunkdb_service.rs │ │ ├── chunkdb_rpc_service.rs │ │ └── chunkdb_rpc_service/ │ │ ├── service.rs # state and handler registration @@ -1190,11 +1213,13 @@ the shared [`service configuration design`](../config/design-crowdb-config.md). ## 15. Full-Stack Deployment and Benchmark -`crowdb-cli cluster local-deploy -t combined` creates the canonical local -ChunkDB profile: six KV/storage nodes split evenly across three racks, one -four-disk group and one DiskDB instance per node, one three-replica data -group, three ChunkDB instances, and complete bucket ownership. Services share -one configured log root and are registered before readiness is reported. +The shared OCI node lifecycle prepares Group 0, the ordinary metadata group, +DiskDB, DiskIO and ChunkDB through accepted node policy and committed service +intent. The common cluster lifecycle creates runtime resources and observes +readiness; benchmarks attach to the selected cluster without creating groups, +resetting topology or starting and stopping servers. Production placement uses +actual failure domains; a Local cluster on one physical host has zero physical +host failure tolerance. See the [deployment design](../deploy/design-crowdb-deploy.md). `crowdb-cli bench chunkdb allocate` measures mirror or EC allocation with configurable shape, duration, seed, and concurrency. `bench chunkdb mix` uses @@ -1205,11 +1230,14 @@ total to equal the DiskDB busy-space delta after compaction. Capacity exhaustion is a successful stop reason; any correctness error invalidates the sample. -`tools/benchmark/bench-chunkdb-regression.sh` builds all four release binaries and uses -a fresh timestamped combined cluster for mirror, EC 4+2, EC 8+4, lifecycle -mix, concurrency, and capacity-exhaustion cases. It retains each case's logs, -destroys each cluster, runs all later cases after a failure, and returns a -nonzero aggregate status. +The allocation regression uses an explicit test-only measurement fixture with +three co-located logical nodes, one rack, three data groups and four logical +4-TiB disks per DiskDB. It measures one EC 8+4 strip per allocation with varied +concurrency and retains exact space-accounting checks. The fixture owns every +server by PID and process-start identity, records accepted bootstrap inputs and +provides normal clients with explicit targets. Each invocation retains metrics, +logs and a checked shutdown receipt. This fixture does not qualify physical host +redundancy or replace real-storage OCI acceptance. ## 16. Implementation Scope diff --git a/doc/design/chunkds/design-crowdb-chunk-kv-server.md b/doc/design/chunkds/design-crowdb-chunk-kv-server.md index a9908d9b7..a182772fe 100644 --- a/doc/design/chunkds/design-crowdb-chunk-kv-server.md +++ b/doc/design/chunkds/design-crowdb-chunk-kv-server.md @@ -42,6 +42,15 @@ immutable snapshot when a new head or page is absent, corrupt, incomplete, or regressing. Unchanged pages may be referenced by later heads through their original page generation. +An owner's connection address may change without transferring its partitions. +Under the Group-0 leader fence, a fresh instance registration advertising the +same hosted partition and owner epoch may publish an address-only successor. +The partition, range, owner instance ID, epoch, tree, stream and transition +remain unchanged, and old pages remain immutable. A recovering owner may update +its route, but receives no serving authority until normal recovery and grant +checks succeed. Fixed transition comparisons use owner instance identity rather +than requiring its historical connection address. + ## 2. Domain Monitors `EnsureDomainMonitor` carries a persisted descriptor with domain and registry @@ -57,6 +66,12 @@ before planning; it is never represented as an empty cluster. A supervisor restarts failed tasks from durable state and a new leader resumes outstanding transitions. +An unfinished split reserves both partition identities and participating owners +against automatic dead-owner transfer, as it does against ordinary balancing. +The published parent alone cannot prove that a competing transfer may claim its +WAL: durable split handoff can precede catalog publication. Owner restart resumes +that split; heartbeat expiry never authorizes discarding its recovery state. + Normal service discovery filters expired registrations. Monitor observation uses the raw scan, preserving expired entries and heartbeat timestamps so absence cannot be confused with owner death. The default heartbeat is two @@ -64,11 +79,24 @@ seconds, `Suspect` begins at six seconds, and `Dead` recovery begins at ten. ## 3. Serving Authority -The monitor issues one aggregate grant per instance. It contains a monotonic +Each boot claims one fixed Group-0 WAL process lease. It retains its predecessor +identity while claiming; a replaced boot cannot claim again or renew. Independent +background renewal starts before partition recovery. Local expiry or a renewal +gap permanently fences that boot. Lease records retain process epochs and prior +process/grant expiry boundaries after expiration. + +The monitor issues one aggregate grant per instance and process epoch. It contains a monotonic lease sequence, catalog generation, wall-clock expiry, sorted exact `(partition_id, owner_epoch)` assignments, and their canonical digest. An instance heartbeat advertises health but cannot authorize a request. +Heartbeat publication compares the exact process identity and mutates the same +lease key atomically with the observation. Grant publication compares that key +and atomically preserves its sequence and greatest expiry alongside the grant. +Replacement, renewal and grant publication therefore serialize on one record; +replacement cannot miss a concurrent extension. An observation for another +process epoch cannot receive a grant, and a boot rejects grants for other epochs. + On receipt, an owner subtracts maximum clock skew and its self-fence margin from the wall-clock remaining duration, then projects that duration onto its local monotonic clock. Request admission reads an immutable grant snapshot and @@ -78,6 +106,22 @@ one-second self-fence margin fence the old owner by ten seconds. A replacement waits until the old wall-clock expiry plus skew unless an explicit source fence is durably proven. +Authority revocation retains the highest accepted grant sequence. The same old +grant cannot reinstall after a clear. Each assignment also carries local +continuity across live renewal: expiry, removal or clear permanently invalidates +an admitted context, even if a later grant restores the same partition and epoch. +Grant publication uses the current monotonic time, so delayed installation +cannot hide a local expiry gap. + +Normal point execution captures that context before asynchronous WAL/tree work +and validates it again before returning success. A revoked context returns +`LeaseExpired` without advertising a successful journal position; already durable +data remains available to replay and a retry with the same request identity. +The captured IO deadline stays at its original value across continuous renewal. +Catching-up targets retain their separately validated transfer-artifact authority. +Both paths also require the live process scope before execution and before +returning success. Grant installation cannot revive a locally fenced process. + Client catalog revisions and owner epochs are routing references, not serving authority. A server may resolve an older reference through a process-local lineage, but it always authorizes the resolved current assignment against one @@ -131,6 +175,15 @@ the partition worker; retrying the same identity retrieves the recorded result. Reads may carry a mutation's returned journal position for explicit read-after-write ordering. +Multi-get and batch mutation validate their complete key group before IO, then +capture authority for every resolved serving assignment. All captured contexts +are checked against one grant snapshot before and after each member. Revoking +any context stops later members from entering the WAL; a replacement grant cannot +resume the old group. Continuous live renewal preserves the group. Time spent +waiting for earlier members counts toward the original request deadline. +Batch mutation preserves each member's stable identity and individual outcome, +so retry can retrieve already durable results without replaying their mutations. + ## 5. Ordered Reads Seek provides ceiling, higher, floor, and lower operations within one logical @@ -264,6 +317,9 @@ group-0 monitor advance them in this order: writes between the fixed frontier and that dispatch change. 3. Readiness binds the exact common cutover, the retained-parent next epoch, and the child's complete tail overlay. + Split plans retain the original parent tree/WAL identity in every phase. + The strict readiness record contains one child overlay and rejects unknown + fields; the planned timestamp is required. It is persisted before catalog publication. 4. One catalog generation atomically replaces the old parent entry with a smaller entry having the same partition/tree/stream identity and the next @@ -379,10 +435,16 @@ phase, release proof, initial readiness proof, final catch-up proof, and failure. The live-source phases and actions are: 1. `Planned` → `SourcePreparing`: the source remains `Serving`, checkpoints a - pinned base, creates the target-owned empty WAL, and records the initial - source cursor in the target overlay. + pinned base, registers the target WAL's stable `StreamRecord`, and records + the initial source cursor in the target overlay. It acquires no writer lease + and allocates no chunks for the target WAL. + After pinning the base, an ordered mutation-worker command publishes and + captures the WAL sequence, byte offset and Stream generation together. + Independently sampled counters cannot define a replay cutover: concurrent + mutations may be newer than the previously published Stream prefix. 2. `TargetPreparing`: the remote target validates the range, page-root and - stream identities, opens the exact immutable tree-manifest generation named + stream identities, initializes or recovers the target WAL under its own + process lease, opens the exact immutable tree-manifest generation named by the source-base proof rather than the latest root, and starts one shared async initialization coroutine that replays the source suffix into a `Prepared` overlay. The source remains writable. The live target handle and @@ -602,7 +664,7 @@ explicit; UI refresh does not bypass stale cursor rejection. ## 9. Lifecycle and Observability Startup ensures the monitor, registers the instance, loads a complete catalog, -opens each assignment's latest tree root and stream manifest as `Prepared`, +opens each assignment's latest tree root and Stream chunk record as `Prepared`, reads the applied sequence and WAL replay offset from that tree root, replays through the stream's current durable tail, reports readiness, and serves only after installing a matching grant. Shutdown @@ -611,6 +673,19 @@ operations retain handles and finish before bounded checkpoint/drain work. Catalog refresh recovers every new or changed local assignment first, then replaces the catalog and hosted-partition snapshots; unchanged exact-epoch handles remain live and departed assignments are dropped. +An unknown journal or post-journal apply outcome moves the affected writer to +`Recovering`, closing its admission and excluding it from serving grants. +An unchanged catalog cannot repair that terminal handle. The process detects +this state, clears authority, stops RPC admission and exits so its supervisor +can relaunch it. The successor reloads the durable catalog, tree checkpoint, +complete WAL and persisted split/transfer proofs before obtaining a fresh grant. +No uncertainty flag is cleared in place and no recovery artifact is deleted. +Diagnostics identify the partition, owner epoch and durable/applied sequences; +ordinary `Prepared` activation and idle instances do not trigger a restart. +The recovery check includes prepared transfer targets and retained split +dispatchers with both ingress writers, even after a newer catalog replaces their +assignments. Current-range readiness cannot hide terminal uncertainty in a +compatibility route that remains reachable by older client references. Catalog-head watch notifications trigger reconciliation immediately and check the matching serving grant afterward. Instance-specific grant notifications install authority without reopening or rescanning catalog assignments. diff --git a/doc/design/chunkds/design-crowdb-chunk-kv.md b/doc/design/chunkds/design-crowdb-chunk-kv.md index 19dc025bf..622889de0 100644 --- a/doc/design/chunkds/design-crowdb-chunk-kv.md +++ b/doc/design/chunkds/design-crowdb-chunk-kv.md @@ -242,7 +242,7 @@ second destination tree/WAL for the parent. Balancing one independently recoverable child reuses the same base-plus-tail representation. A live source checkpoint supplies the pinned base manifest, -base sequence, stream manifest generation, and replay offset. The target owns +base sequence, Stream chunk record generation, and replay offset. The target owns a distinct WAL and opens the shared tree root as `Prepared`. One shared async initialization coroutine replays the source stream while the source continues serving. It first reaches a persisted preparation cursor `P`, then follows the diff --git a/doc/design/chunkds/design-crowdb-chunk-stream.md b/doc/design/chunkds/design-crowdb-chunk-stream.md index 5d02a263d..2d47b4822 100644 --- a/doc/design/chunkds/design-crowdb-chunk-stream.md +++ b/doc/design/chunkds/design-crowdb-chunk-stream.md @@ -4,11 +4,14 @@ # CROWDB - Design: Chunk Stream The chunk stream presents finite mirrored chunks as one durable, ordered -logical byte space. It supplies the storage boundary used by the partition WAL -without owning WAL framing, request identity, or replay policy. +logical byte space. Append completion depends on mirrored data persistence; +Stream metadata records lifecycle transitions and bounded chunk record checkpoints. +The stream owns physical framing and addressing; the partition WAL owns +business-record framing, request identity, and replay policy. Depends on: [chunk IO](../chunkio/design-crowdb-chunkio.md), [chunk reader](../chunkio/design-crowdb-chunkio-reader.md), +[DiskIO](../diskio/design-crowdb-diskio.md), [chunkdb](../chunkdb/design-crowdb-chunkdb.md), and [group-0 sysdata](../kv/design-crowdb-kv-group0.md). @@ -23,7 +26,7 @@ Depends on: [chunk IO](../chunkio/design-crowdb-chunkio.md), 7. [Trim and reclamation](#7-trim-and-reclamation) 8. [Bounds and observation](#8-bounds-and-observation) 9. [Errors and invariants](#9-errors-and-invariants) -10. [Open Issues](#open-issues) +10. [Open issues](#10-open-issues) ## 1. Ownership and API @@ -32,286 +35,563 @@ opaque, time-ordered 128-bit identity bound to a logical partition, never to a process or node. It renders as 32 hexadecimal digits. `ChunkStream` exposes `create`, `open`, `open_read_only`, `tail`, vectored `append`, exact `read_at`, seekable bounded sequential reading, `trim_prefix`, and `close`. -`open_read_only` opens one exact durable epoch without adopting writer -authority, allocating a successor chunk, sealing the active extent, or -publishing metadata. It is the source-tail handle used by split and balance -overlays. - -Every nonempty `append` returns the selected chunk identity and exact logical -range. A zero-length ordinary append returns no chunk identity and performs no -IO. `append_chunk_bound` accepts caller-framed body+CRC bytes and makes the -stream worker add the selected chunk's canonical 128-bit ID after rollover -selection. The returned logical range includes that trailer. This keeps chunk -choice in the ordered worker while allowing a journal to bind each frame to its -physical source. - -Each writable handle owns one bounded Tokio MPSC queue and one worker task. The -task exclusively mutates the active descriptor, extent list, manifest -generation, tail, trim point, and rollover state. Producers and readers use -atomics plus immutable `ArcSwap` manifest snapshots; the hot path adds no lock. - -The stream does not interpret record headers, checksums, request identities, or -replay semantics. Ordinary append stores exact caller bytes. Chunk-bound append -adds only the fixed physical-provenance trailer requested by the caller; the -partition journal owns its surrounding frame and consumer watermarks. +An owner-side flush/checkpoint barrier captures and persists a synchronized +prefix on demand without changing ordinary append completion. + +Every nonempty append returns the selected chunk identity and exact logical +range. Logical lengths count caller payload bytes, excluding physical frame +headers and footers. A zero-length ordinary append returns no chunk identity +and performs no IO. `append_chunk_bound` and `append_chunk_bound_batch` keep +chunk selection in the ordered worker; the outer Stream frames carry the +selected chunk identity and CRC. They add no payload trailer. The stream does +not interpret the caller's business-record format or retry identity. + +One append payload is one indivisible validity unit. A successful append makes +the whole payload available; recovery never exposes a proper prefix of that +payload. Physical frame and IO splitting are internal. Batching preserves each +request's append boundary rather than making the entire batch one group. + +Each writable handle owns one bounded Tokio MPSC queue and one worker task. +The task exclusively mutates the active chunk's in-memory cursor and mappings, +logical tail, and lifecycle publication state. Producers and readers use +atomics and immutable `ArcSwap` views. Append and snapshot publication add no +hot-path lock. Batching preserves request order and exact returned ranges. + +`open_read_only` validates one exact writer epoch and captures a finite view. +It does not adopt writer authority, allocate a successor, seal a chunk, or +publish writer state itself. It may request a flush/checkpoint from the current +owner. If the owner is unavailable it uses the consistent prefix persisted in +ChunkDB and records that checkpoint as its finite bound; that view can lag +successful appends. A caller requiring a newer exact cutover must not silently +accept the older fallback. The handle supports source-tail replay during split +and balance with these explicit bounds. ## 2. Storage Contracts -Three injected async traits isolate the state machine from placement and -transport: - -- `StreamRegistry` loads and creates the small group-0 binding. It is used only - by create/open, never append or read. Binding keys are readable text. Stream - creation accepts a metadata-group selection and defaults to group 1. -- `StreamMetadataStore` loads the stable manifest head and immutable extent - pages. It writes fresh versioned pages before publishing the head through a - KV revision compare-and-set. `None` means the stream must not exist. -- `StreamChunkStore` allocates mirrored chunks, writes one contiguous range to - every mirror, advances and inspects the fenced durable cursor, seals chunks, - reads physical ranges, and idempotently releases trimmed chunks. - -The production chunk store uses a direct-buffer `MirrorChunkWriter` from -`crowdb-chunk-client`. It owns one chunk, appends mirror strips asynchronously, -and never constructs the EC pipeline. Every allocation uses the durable `StreamPurpose` in the binding and manifest: -system journals allocate `ChunkType::Wal`, business streams allocate -`ChunkType::Stream`. Both use a canonical `stream/` owner key followed by the 128-bit stream identity. -Legacy persisted and wire chunk records decode with an empty unattributed owner. -Stream rollover remains above the writer. Allocation, reopen, rollover and -repair preserve purpose and stream identity. Reopen rejects binding/manifest -purpose disagreement and chunk ID/metadata purpose disagreement. Purpose is -required in persisted bindings; legacy records are not inferred or retagged. -Chunk-kv journals require Wal streams at construction. +Three injected async traits separate Stream state from placement and transport: + +- `StreamRegistry` loads and creates the stable group-0 `StreamRecord`. + It is used by create/open, never by ordinary append or data reads. Typed + binary keys contain the Stream ID; metadata defaults to nonzero group 1. +- `StreamMetadataStore` selects the nonzero group containing ordered + `StreamChunkRecord` values. A shared record store implements conditional + lifecycle transitions, reverse seek, candidates, interval leases and GC. +- `StreamChunkStore` allocates and seals mirrored chunks, writes and synchronizes + the required replicas, exposes current chunk layout for bounded reads and + recovery, and idempotently releases trimmed chunks. It does not require a + separate ChunkDB cursor advance for each append. + +Chunk IO and ChunkDB own `chunkinfo`: strips, physical capacity, replica +placement, growth, replacement, and their durable internal transitions. Each +such transition updates chunk information independently. Stream metadata refers +to stable chunk identities and logical mappings; internal chunk changes are +transparent and never trigger Stream chunk record publication. +Recovery resolves current capacity and layout through the chunk layer instead +of trusting capacity cached in an older Stream checkpoint. + +DiskIO writes carry a boolean sync option. For a synchronized request the server +performs the ordered write, then the required flush, and returns one response +after both succeed. The client does not wait for a write reply and then send a +second fsync RPC. Buffered writes do not request the flush. When transport +limits require several requests, preserve strip-block write ordering and +complete required synchronization before acknowledging the append. + +Every allocation preserves the durable `StreamPurpose`: system journals use +`ChunkType::Wal`, business streams use `ChunkType::Stream`. Both use the +canonical `stream/` owner key followed by the 128-bit stream identity. Reopen +rejects purpose disagreement between StreamRecord, StreamChunkRecord, chunk identity, and +chunk information, including a missing or mismatched owner key. Chunk-KV +journals require Wal streams. `ProductionStreamRuntime` shares the process KV client and discovered chunk IO -routes across every stream handle. The chunk-KV server owns that runtime and -selects the metadata store while each Active binding selects its metadata -group. A control-plane-created binding is initialized through -`create_registered`; standalone creation may publish its own binding. - -The `test-util` feature provides one deterministic in-memory implementation of -all three traits, including write suspension, cursor outcomes, and metadata -publication failure injection. Production adapters retain the same ownership -and durability contract. +routes. The Chunk-KV server chooses the metadata store; the StreamRecord +chooses its metadata group. `create_registered` initializes a control-plane +StreamRecord. The `test-util` implementation supplies deterministic mirror-write, +sync, tail-content, ownership, and metadata-publication fault injection. ## 3. Metadata -Group 0 stores only `StreamBinding {purpose, stream_name, metadata_group_id, -binding_generation, state, owner_kind}`. One nonzero KV group stores one stable -head per stream plus immutable extent pages under versioned keys. - -`StreamManifest` contains the immutable purpose, writer epoch, generation, logical trim point, -sealed tail, optional active descriptor, page fences, and closed state. Every -head mutation uses conditional KV write; no competing writer may blind-write -that key. `StreamExtentPage` contains parallel arrays: - -- `chunk_ids[N]` -- `logical_offsets[N + 1]` -- `physical_offsets[N]` - -Entry `i` maps `[logical_offsets[i], logical_offsets[i + 1])` to the same-sized -physical range beginning at `physical_offsets[i]`. Validation rejects bad -array lengths, empty or unordered extents, gaps between retained pages, -identity or fence mismatches, invalid active cursors, and arithmetic overflow. -After trim compaction, coverage may begin at or before `trim_offset`; it must -continue exactly through `sealed_tail`. - -The open chunk stays outside the sealed extent pages. Its durable logical end -is `logical_start + acknowledged_cursor - physical_start`, using checked -arithmetic. +### 3.1 Identity and ordered keys + +Group 0 stores the stable `StreamRecord`: purpose, Stream ID, metadata group, +record version, registration state, consumer kind and ID, and creation time in +Unix milliseconds. Writer leases are separate Group-0 records. Append, +preparation, rollover, close and writer relocation preserve the identity value +and original creation time. Consumer identity describes who owns the Stream; +it does not identify the currently leased process. + +A nonzero metadata group stores `StreamChunkRecord` values under typed keys: + +```text +record namespace | stream_id[16] | kind[1] | order[8 big endian] +Attached: kind = 0, order = logical_start +Candidate: kind = 1, order = reservation_sequence +``` + +Each record has a chunk ID, purpose, state, reservation/attachment sequence, +writer epoch/session, publication generation, trim watermark, closed status and +compressed mappings. Attached records contain explicit half-open +`[logical_start, logical_end)` bounds. A Candidate has no assigned logical +range and accepts no user data. Attached Active and Sealed records sort by +logical start; candidates occupy a distinct suffix range. Reverse scanning the +attached prefix returns the current tail without a separate moving head. + +The tail record carries Stream trim/closed state and the publication counter. +A successor inherits that control state. An empty Stream has an empty Active +record from creation; close seals it and persists closed status. The tail +record is retained even when every byte is trimmed. The Active record's end +is a durable checkpoint, not a limit on recovery or the live in-memory tail. + +There is no persisted Stream-wide physical offset and no duplicated +`physical_end` in a StreamChunkRecord. Chunk-local physical positions derive +from append geometry; the chunk's exact seal length is authoritative at release. + +### 3.2 Compact complete-append mappings + +One mapping describes one complete append group, regardless of frame count. +Store append lengths, from which both logical and chunk-local physical offsets +are derived. With grouped-frame payload capacity 65,489 and overhead 47 bytes, +a group of length `L` advances physical position by +`L + 47 * ceil(L / 65489)`. A physical frame maximum of 64 KiB does not impose +a 64 KiB append maximum. + +Lengths are encoded in blocks of at most 256 entries. Equal lengths use a +constant run; other blocks choose canonical variable-length u32 values or raw +u32 values by encoded size. In-memory samples store each block's logical and +physical base and encoded position. A random seek decodes at most one block. +These blocks are an encoding detail inside one record, not persisted pages or +a Stream-owned index tree. + +The checksummed record validates key/start identity, purpose, exact decoded +logical coverage, group lengths and bounded physical geometry. A new checkpoint +preserves all previously published append boundaries. Rollover validates exact +continuity between the finalized predecessor and successor. Bounds detect +inconsistent coverage; CRC32C also detects same-length corruption. + +### 3.3 Physical framing + +All consumers use one versioned physical frame format and common Rust/C++ +codecs. Each frame is at most 64 KiB including header and footer. Magic identifies +purpose and version; a common header role identifies ordinary, head, or follower +frames. Only Stream uses grouped roles. User-data and tree consumers use ordinary +frames; user-data completeness remains a whole-object publication condition. +Native buffers retain bounded streaming and zero-copy ownership. + +Each nonempty Stream append is exactly one verifiable frame group in one chunk. +Its head declares total payload bytes and frame count, including the head. Even +a single-frame append is a head with count one. Followers identify their head +by its physical offset in the same chunk and carry their ordinal. Frame lengths, +roles, group identity, order, count, total payload length, CRC, and expected chunk +identity must all agree before the group is valid. There is no separate commit +frame or commit RPC. CRC32C covers the common header, group fields, and payload; +the footer chunk identity is validated separately against the expected chunk. +All codecs use the same CRC input and byte layout. +Declared lengths/counts are checked against append limits, remaining chunk +capacity, and checked arithmetic before scanning or reserving memory. A parser +does not allocate buffers proportional to an untrusted declared group size. + +Role-dependent header sizes determine payload capacities. Interior frames fill +their role's payload capacity and only the last may be short. Location arithmetic +derives group geometry from the format version, role capacities, and total +logical length, without one mapping per frame. + +Revision 2 retains the 14-byte fixed prefix: little-endian magic, payload offset, +payload size, and write timestamp. The low magic byte is 2; the high byte keeps +the public purpose. The role byte follows the prefix. Ordinary frames use role +0 and a 15-byte header, leaving 65,501 payload bytes. Stream heads use role 1 +and add total payload length (u64) and frame count (u32); followers use role 2 +and add head physical offset (u64) and ordinal (u32). Both grouped headers are +27 bytes and leave 65,489 payload bytes. Group fields are little-endian. +The 20-byte footer stores the little-endian CRC32C followed by the big-endian +chunk identity. Revision-2 CRC starts at zero and excludes the footer. + +Only the canonical revision-2 frame format is accepted. Unknown revisions, +roles outside their purpose and noncanonical header offsets are rejected. + +`StreamChunkRecord` uses a single current `CSCR` version-1 envelope with +exact body length and CRC32C covering header and body. Decode rejects unknown +versions, trailing bytes, invalid states, excessive counts and oversized bodies +before using mappings. No migration or older-service fallback is supported. + +### 3.4 Atomic publication + +A lifecycle publication conditionally updates the captured tail key revision. +Rollover atomically finalizes the predecessor, deletes its registered Candidate +reservation and inserts the new Active record under its actual logical start. +An empty predecessor and successor may share the same logical-start key; only +the successor remains attached there. Same-chunk checkpoint, trim and close +update only the tail record. Historical chunk records retain their bytes. + +The first attachment reserves Candidate sequence zero, then atomically deletes +that reservation and inserts the empty Active record, guarded by the Candidate's +concrete revision. It checks that no attached tail exists after reservation. +It never uses an absent attached key as its initialization fence: that key can +be deleted by later trimming, so an excluded initializer could otherwise insert +it again. A later initializer can replace an older owner's initial reservation; +its new revision excludes the delayed publication without a mutable Stream head. + +A revision conflict invalidates the captured proof. Re-read the tail and any +Candidate identity before retrying. For an unknown outcome, compare the complete +durable tail with the proposed transition; an exact match is committed. A +superseding writer/session rejects stale publication. Pin and Candidate changes +also fence through the tail revision, preventing stale retention observations +from racing publication or reclamation. ## 4. Append Protocol -Admission reserves a bounded request slot and queued bytes. A zero-length -append returns the current tail without allocation or IO. An append above the -configured maximum is rejected before enqueue. - -An idle worker submits the first request immediately. It drains only requests -already queued and stops at the request limit, byte limit, or active chunk -capacity. It does not use a batching delay. Vectored request bytes are assembled -in queue order into one retained staging buffer and sent through one -`write_mirrors` call. The worker then performs one fenced cursor advance. - -Completion occurs only after all mirror writes and the durable cursor update. -Each request receives its exact non-overlapping logical subrange. Mirror-write -failures use the retained strip image to replace a failed block; a confirmed -absent append can rotate to another chunk and retry. An ambiguous cursor -response is inspected without blind resubmission: the worker accepts it only -when the durable cursor equals the proposed end and the last-advance checksum -matches the staging checksum; an unchanged cursor proves absence. If durable -state cannot be read, the current append remains pending for resolution. A -fencing violation or corrupt durable state requires reopening and recovery; -it is not a transfer-quiescence state. - -For a chunk-bound request, the worker adds the selected `ChunkId` after the -caller's bytes before assembling the aggregate write. Admission, remaining -capacity, cursor advancement, and returned ranges include the trailer. Several -requests may still share one physical batch; each receives its own trailer. +Admission reserves bounded request slots and queued bytes. Oversized appends are +rejected before enqueue. An idle worker dispatches the first request immediately +and drains only requests already queued. It stops at the request limit, byte +target, or current chunk capacity, without adding a batching delay. One large +request may exceed the batch byte target while remaining within append bounds. + +The normal path for an already attached chunk is: + +1. Select non-overlapping logical ranges and encode one complete frame group + per append. Keep one location per payload rather than one extent per frame. +2. Submit mirror writes concurrently, requesting synchronization in the write + requests. The chunk layer owns strip ordering, repair images, replica + replacement, and durable updates of its own chunk information. +3. Wait for successful synchronized completion from every required replica in + the configured mirror set. Three-copy configuration waits for three replicas; + no fixed replica count is assumed by the Stream API. A smaller set of readable + replicas does not lower the write-completion requirement. The chunk layer + restores required protection after replacement before returning success. +4. Publish the completed in-memory cursor, logical tail, and extent snapshot, + then complete each request with its exact logical range. + +There is no per-batch `advance_chunk_write`, group-0 update, Stream chunk record +write, or metadata CAS. Encoding, buffer ownership, and staging remain bounded; +frame splitting does not justify repeated payload coalescing or per-frame KV +writes. Periodic lease maintenance is separate from the append data path. + +An ambiguous mirror response is not evidence of absence. Resolve it through the +chunk write/recovery contract; do not retry at a guessed offset or use an old +ChunkDB cursor to declare new bytes absent. A failure on one request cannot let +later requests skip a logical hole. Watchdogs observe the original future and +do not cancel, resubmit, or complete it. An interrupted append contributes no +partial logical range. Recovery may retain a complete append whose response was +lost, but never its proper prefix; consumer retry semantics handle that uncertain +completion. Differing valid replica tails follow section 5.2. ## 5. Rollover and Recovery -One append never straddles chunks, and one stream chunk contains at most 256 -MiB of logical data. If an append cannot fit, the worker seals the old chunk, -appends its logical mapping to the extent builder, writes a fresh immutable -version of the affected tail page, allocates a successor, and CAS-publishes the -successor head before writing the request. A crash before the head CAS leaves -the previous head complete and the new page unreachable. Objects not reachable -from authoritative metadata are orphans. - -Writer open resolves the authoritative tail metadata and last extent, then -queries the last chunk's durable cursor. It never appends to a chunk owned by a -previous process. The old chunk is sealed at its acknowledged cursor and a -fresh 256-MiB mirrored chunk is published before new bytes are admitted. Before -initial publication and every CAS retry, the writer compares the observed head -epoch with its local ownership epoch. A lower epoch returns `StaleWriter`; only -a higher epoch holding current ownership may adopt an older head. KV revision -CAS serializes publications, while the epoch comparison prevents authority -regression. Full extent pages remain immutable; short restart extents are -accepted and require no merge pass. - -A higher ownership epoch adopts the same manifest and byte history into a new -generation. Querying the active cursor establishes the new chunk-store fence; -publishing compares against the exact prior epoch and generation. A lower -epoch is rejected, and a previously open lower-epoch writer can no longer -write or advance the cursor. - -Idle maintenance compares the current manifest epoch, generation and active -chunk with the worker's head before renewal and before rollover. A superseded -worker stalls and performs no further idle renewals or publications. An -unavailable authority observation defers maintenance to the next interval. -Current owners may still rotate their own sealed chunk; renewal retries are -bounded per idle tick, and manifest publication remains fenced by CAS. - -Read-only open is deliberately different from writer open. It validates the -requested historical epoch against the durable manifest, captures its -generation, trim point, extents, active descriptor, and acknowledged cursor, -and never runs adoption. Consequently a target can replay a source suffix while -the source writer continues appending. Its captured durable end is finite; a -later read-only open is required to observe a newer preparation or release -cursor. +### 5.1 Lifecycle ordering + +One append never straddles chunks. A chunk contains at most the configured +physical framed-byte limit, 256 MiB by default. Chunk IO may allocate physical +strips incrementally; this remains internal to the chunk layer. + +Capacity preparation runs ahead of consumption. Measured write bandwidth +and preparation duration size the initialized spare capacity with a two-interval +margin. At least one initialized Candidate is allocated and registered while +the Active chunk accepts appends. Normal writes use ready capacity; exceptional +allocation/initialization delays apply backpressure instead of exposing unsafe +capacity. + +When an append cannot fit, drain preparation and writes for the current chunk, +physically seal its exact final length, take the prepared Candidate, and commit +the atomic record transition. The successor accepts data only after attachment. +Switching does not allocate or rewrite historical metadata. Seal and attachment +are distinct durability boundaries: a crash after seal but before publication +leaves the one last referenced chunk to reconcile. An unattached Candidate has +no acknowledged user data and can be reclaimed after owner exclusion. + +Different segment IO streams can drain concurrently, with at most eight drains +in flight. Every submitted drain must finish, including after a peer fails; +physical sealing requires all drains to succeed and current writer authority. + +### 5.2 Tail reconstruction + +The latest physical cursor is the end of the contiguous prefix of complete +append groups in the last referenced chunk. Recovery starts +at that chunk's beginning or a validated synchronized checkpoint and parses +forward within current authoritative capacity. It checks frame kind, lengths, +CRC, chunk identity, and address arithmetic. It tracks a tentative frame cursor +inside a group and advances the recovered physical cursor and logical tail only +after validating the entire group. It then adds one compact location. + +Before publishing a fresh attachment or admitting writes into newly allocated +capacity, zero and synchronize each new physical strip on every required mirror. +Initialization uses one submission per replica, without automatic retries. An +uncertain result leaves that capacity unavailable to the writer; the owner cannot +repeat initialization after later data writes. A replacement strip receives its +complete retained prefix and a zero suffix with the same single-attempt contract +before its metadata publication. + +Ordinary append writes exactly its complete group range. It never writes a marker +past that end: a delayed duplicate must not overlap the next acknowledged group. +Unwritten headers are supplied by the initialized suffix. Capacity preparation +reserves a complete 14-byte header below the hard chunk limit, growing before +writing when necessary. This keeps the previous complete prefix bounded inside +already initialized capacity if growth stops before its new strips are ready. +An exact hard-capacity end requires no extra header. Allocation, initialization +and repair traffic are accounted separately from ordinary append bytes. + +The bounded repair shadow retains only the unfinished strip prefix. Validation +failures before the first data IO do not consume that owner's fresh-write +admission. Block alignment must preserve all physical bytes outside the requested +range even when the partial-block cache is absent. + +An unwritten suffix, incomplete final frame, or incomplete group bounds the +candidate prefix at the previous complete append. No later group can skip that +hole. Individually valid frames in an incomplete group do not advance the tail. +Corruption in required retained data is not silently treated as EOF: use mirror +validation/repair or fail recovery. Bytes with a wrong chunk identity cannot +extend the tail. Syntactically valid bytes beyond the last checkpoint remain +candidates; a stale persisted cursor is not an authority to discard them. + +The shared parser keeps one at-most-64-KiB frame and constant group/hash state. +Input windows are at most 1 MiB; output batches contain at most 1,024 complete +locations. Observed bytes and parsed frames are distinct from committed group +progress. An exact-capacity end needs no extra header read. Required grouped +checkpoints are exact append boundaries. A SHA-256 digest covers only the encoded +complete prefix; incomplete groups do not enter it. A failed observation cannot +subsequently be finalized as a healthy shorter prefix. + +After owner exclusion, observe the current available mirrors concurrently +within a bounded IO budget. Use the shortest complete-append prefix among healthy +mirrors that finish observation. A mirror that fails or times out during the +read is unavailable, not a zero-length or partially observed prefix. Corrupt +replicas are not valid shorter candidates; use healthy copies or report an +error. All candidates must agree on retained overlapping contents. Conflicting +valid contents fail explicitly rather than waiting for an external arbiter. +No surviving readable candidate means recovery is unavailable. Repair restores +configured protection before the next successful append. Replica comparison and +checkpoints use complete-group boundaries, never an interior frame boundary. + +Only the last referenced chunk may require content parsing. Earlier chunks +have finalized bounds and chunk record mappings and are not scanned. A last +chunk whose seal and final mappings are already published needs no tail scan. +Recovery after an interrupted seal may still reconstruct that one last chunk. +Bound reads, buffered bytes, parsed frames, and total scan bytes by the active +chunk geometry, independently of retained Stream length. + +### 5.3 Writer ownership + +Writer open validates purpose and current ownership before exposing a writable +handle. A lower epoch is rejected. A higher epoch adopts retained history +without copying historical chunk records, resolves the old tail, seals/finalizes it, and +attaches a fresh chunk before new appends. Writers never resume appending to a +previous process's chunk. + +Group-0 owner leases remain authoritative between metadata checkpoints. The +writer uses a conservative local monotonic deadline derived from its grant and +checks authority before accepting work and before acknowledging it. Background +renewal is independent of append batches. Expiry or explicit fencing stops both +new admission and successful acknowledgment, including after a process pause. +Metadata CAS still fences lifecycle publications; DiskIO success alone does +not authorize an acknowledgment and DiskIO does not own Stream leases. + +Explicit transfer stops the old owner's admission, drains its in-flight work, +persists the final synchronized boundary, and records a fenced release proof +in Group 0. Once release and the new grant are authoritative, the new owner can +activate without waiting for the old lease's natural expiry. A verbal/local +promise or an unresolved release response is not the durable release proof. + +If the old owner cannot be contacted, activation waits for its lease's safe +expiry boundary, including the existing clock-skew allowance. Resolve in-flight +old writes using the bounded write-age/drain rules before fixing the old tail. +The new owner recovers that tail and publishes a newly allocated chunk; it +never resumes writing the old owner's chunk. Group-0 unavailability cannot +extend a local grant past its conservative deadline. + +### 5.4 Read-only capture + +Read-only open validates the requested epoch and captures the tail chunk record, +trim point, active identity, and one finite validated tail: + +- When the current owner is reachable, request an ordered flush/checkpoint + barrier. The owner captures its synchronized prefix, persists the ChunkDB + complete-group boundary, then publishes the corresponding chunk record checkpoint + before returning the matching epoch/chunk/physical cursor/logical end. A failed + record publication leaves the prior checkpoint authoritative even if ChunkDB + already holds a newer cursor. Later append does not widen that view. Already + synchronized appends do not require another data fsync to capture the prefix. +- When the owner is unreachable, use the captured persisted ChunkDB boundary + with matching Stream metadata. Reconstruct missing mappings only through that + boundary without writer adoption or publication. The result is an explicitly + older checkpoint view, possibly empty for the active chunk. + +Epoch, chunk identity, chunk record/checkpoint, and end must form one consistent view; +both physical and logical checkpoint bounds end complete groups. A raw newer +ChunkDB cursor is not combined with an unrelated chunk record or epoch. Do not combine +observations across ownership changes. Expose the captured bound +and whether it came from owner flush or checkpoint fallback. A requested exact +upper cutover beyond that bound is unavailable, not a successful shorter replay. +Checkpoint fallback is a read-visibility rule: writer failover still recovers +the content-derived tail and cannot discard acknowledged data beyond it. ## 6. Reads -Reads reject ranges below `trim_offset` or beyond the current durable tail. -For sealed data, the reader binary-searches manifest fences, loads only the -target extent pages, validates each loaded page against its fence, and resolves -the physical offset with checked arithmetic. Active reads use the immutable -acknowledged cursor snapshot. - -A sequential reader starts at a caller-supplied logical offset and accepts a -finite byte hint or `ToEnd`. It captures the corresponding durable end, returns -EOF there, and leaves EOF handling to its caller. Cached bytes are returned -while adjacent physical ranges are prefetched out of order within one bounded -window; delivery remains in logical order. The default retained window is 8 -MiB. Each physical request is at most 1 MiB and at most eight requests from the -window run concurrently. This keeps production DiskIO frames below their -deadline-sensitive large-message range without reducing sequential readahead. -Independent readers may run concurrently. Each physical read has an observation -watchdog that never cancels or retries the underlying future. -A stream handle keeps the current manifest generation's extent pages in an -`ArcSwap` snapshot. Reads hit the immutable snapshot without a lock; a miss -publishes a copy-on-write snapshot, and an older reader cannot replace a newer -generation. The cache therefore remains bounded by one manifest generation. -A provenance-aware reader also yields each logical segment's physical chunk -identity. A journal compares that identity with its frame trailer. The durable -acknowledged cursor remains the read and recovery upper bound; identity -validation never promotes residual bytes beyond it. - -An overlay reader supplies both a lower replay offset and an exact upper -cutover offset. The consumer owns record framing, range filtering, sequence -continuity, and retry-result semantics. The stream guarantees only that bytes -within the captured interval come from the validated source epoch and manifest -and that EOF occurs at the captured acknowledged cursor. +Reads reject ranges below `trim_offset` or beyond their captured logical tail. +For persisted data the reader reverse-seeks the greatest attached key whose +logical start is at or below the requested offset, validates coverage, and +locates its complete append through the record's compressed mapping samples. +Active unpublished data uses append-only in-memory mappings. Expand only the +intersecting physical frames and return the requested payload after validation. +Reads cross append and chunk boundaries while preserving logical order. + +A sequential reader accepts a start and finite byte hint or `ToEnd`, captures +its end once, and returns EOF there. The default retained window is 8 MiB, each +physical request is at most 1 MiB, and at most eight requests run concurrently. +Independent readers may run concurrently. Observation watchdogs do not cancel +or retry underlying IO. + +The lock-free record cache has 32 slots, each admitting at most 1 MiB of +encoded mappings plus seek samples. Oversized records remain readable but are +not cached. A record lookup loads only its target chunk, independently of +historical Stream length. Snapshot publication uses immutable ArcSwap views; +finite readers retain their captured view and interval lease through trim. + +A provenance-aware reader yields physical chunk identity along with payload. +The journal owns business-record validation, sequence continuity, filtering, +and retry-result semantics. Physical Stream frame validation applies regardless +of consumer. Readers and overlays preserve their captured bounds through later +append, rollover, and ownership changes. ## 7. Trim and Reclamation -`trim_prefix(g)` rejects regression and `g > tail`. It first publishes the new -logical trim point. Only afterward does it release complete sealed extents -whose logical end is at or below `g`. A boundary extent remains allocated. - -Cleanup is idempotent and bounded by `gc_bytes_per_pass`, with at least one -eligible extent processed so progress cannot stop behind a large chunk. A -second generation removes successfully released extents from the directory. -Calling trim again at the same watermark retries unfinished physical cleanup. -Reads below the published watermark remain inaccessible throughout every crash -point. - -Immutable metadata cleanup is separate from logical trim. A caller that owns -reader/checkpoint retention supplies the oldest still-readable manifest -generation to `reclaim_metadata_before`. Each call deletes only older extent -pages and is bounded by a caller-supplied page count; the current generation is -never eligible. - -Shared-tail consumers contribute retention pins outside the stream. A caller -may advance trim or metadata reclamation only after no catalog artifact, -checkpoint, recovery operation, retry floor, or forwarding grace interval -references the affected source offsets or manifest generation. Reopening a -read-only handle is not a substitute for a pin: once the authoritative caller -releases the last pin, later trim may make that historical interval -unreadable. +`trim_prefix(g)` rejects regression and `g > tail`. Persist the new trim +watermark on the tail before reclaiming data. Only complete Sealed chunks whose +logical end is at or below the watermark are eligible. Partial trim preserves +the chunk record's original bounds and mappings. The current tail is retained. + +A finite reader leases its logical interval and exact Active chunk identity. +The latter protects unpublished local mappings. Pin creation, renewal and +release conditionally fence the tail revision; collection captures that revision +before observing pins. A changed revision invalidates the entire proof. +Reader pins are limited to 512 identities of at most 1 KiB each. They renew +within a 60-second lifetime and enforce an earlier local monotonic deadline. +Expired pins protect IO for a further 31 seconds before bounded cleanup. +Expiry cannot be renewed into a new live view. WAL consumers publish their +safe replay/retry watermark before trim; split/transfer checkpoint suppression +and finite captures preserve their retained dependencies. + +Collection combines the trim watermark, caller-proven generation horizon, +whole-chunk state and all live interval protections. It scans a bounded prefix +with a logical-start continuation and processes at most one eligible physical +chunk per pass. Mark the record Deleting under the captured tail revision, +release the chunk using its exact derived complete physical length, and delete +the record only after release is confirmed. A failed release or delete leaves +a retryable Deleting record. Repeating trim resumes cleanup. At least one +eligible whole chunk can progress even if it exceeds the physical byte target. + +A generation alone never authorizes deleting retained data. There are no +Stream directory roots, page reachability walks or historical metadata copies. ## 8. Bounds and Observation -Defaults bound the queue at 1,024 requests and 64 MiB, a batch at 64 requests -and 1 MiB, one append at 64 MiB, one chunk at 256 MiB, an extent page at 1,024 -entries, a sequential read window at 8 MiB with eight concurrent physical -reads capped at 1 MiB each, and one GC pass at 64 MiB. All bounds are -configurable and validated as nonzero. - -Lock-free counters report submitted/completed/failed requests, logical and -three-mirror physical bytes, batch/request counts, rollovers, read bytes, -reclaimed bytes, queue high-water marks, metadata publications, extent-page -cache hits/misses, physical-read amplification, and watchdog observations. -Append and read watchdogs log the -stream identity, epoch, logical range or offset, operation age, queue age, -batch size, and stage at every interval while retaining the original future as -the sole completion owner. - -The production regression baseline uses a three-node loopback deployment with -three NullDisk instances, memory-backed KV/WAL, an Intel Core i9-7960X, and -Linux 6.11. A ten-second reference run produced these results and gates: - -| Workload | Reference throughput | Reference p99 | Regression floor | p99 ceiling | Peak RSS growth | -|--------------------------|---------------------:|--------------:|-----------------:|------------:|----------------:| -| 4 KiB append, 1 task | 470 ops/s | 3.3 ms | 250 ops/s | 10 ms | 0.3 MiB | -| 4 KiB append, 32 tasks | 7,076 ops/s | 5.5 ms | 3,000 ops/s | 20 ms | 1.5 MiB | -| 1 MiB append, 8 tasks | 284 MiB/s | 38 ms | 125 MiB/s | 150 ms | 33 MiB | -| 4 MiB rollover, 8 tasks | 444 MiB/s | 112 ms | 200 MiB/s | 500 ms | 91 MiB | -| 4 KiB random, 32 tasks | 71,606 ops/s | 0.7 ms | 30,000 ops/s | 5 ms | 10 MiB | -| 320 MiB replay, 1 task | 746 MiB/s | 478 ms | 300 MiB/s | 2,000 ms | 144 MiB | -| 576 MiB prefix trim | 28,888 MiB/s | 8.9 ms | 1,000 MiB/s | 100 ms | 0 MiB | - -The sentinel also requires zero errors and watchdog observations, queue high -water below configured admission bounds, and peak RSS growth below 256 MiB. -Metadata publications may not exceed initial allocation plus rollover count. -One manifest generation incurs at most one extent-page miss per concurrent -reader; the 32-task random baseline observed 25 racing initial misses followed -by 572,996 hits, while replay observed one miss and 6,143 hits. GC must release -at least one complete chunk. +Defaults bound admission at 1,024 requests and 64 MiB, a batch at 64 requests +and a 1 MiB byte target, one append at 64 MiB, and one chunk at 256 MiB with one compressed record +per chunk. Sequential reading uses an 8 MiB window, +eight concurrent physical requests of at most 1 MiB each, and GC uses a 64 MiB +pass budget. Bounds are configurable and validated; record/decode/cache budgets, +and optional checkpoint budgets must also bound their own work. Replica count +is configured, not implied by a physical-byte metric. + +One encoded chunk record is limited to 24 MiB, and its mapping count is bounded +by the physical chunk geometry. Decode requires canonical 256-entry blocks, +except the final partial block. Collection enumerates at most 128 keys and loads +values individually under its byte target, allowing at most one final record +to cross that target; it never buffers every enumerated value at once. + +Lock-free counters and phase histograms distinguish: + +- append admission, queue, capacity preparation, batching, framing, synchronized + mirror RPC completion, in-memory publication, and end-to-end completion; +- chunk-internal layout updates from Stream lifecycle publications, seal, + ownership maintenance, and optional checkpoints; +- total chunk sealing latency and its segment-drain portion, including failures; +- chunk record build, get, scan and conditional publication latency, mapping + counts/encoded bytes, and cache hits/misses; +- recovery chunks/bytes/frames scanned, replica resolution/repair, cursor and + extent reconstruction, and read-only capture; +- logical/physical bytes, queue high-water marks, retained buffers/cache, read + amplification, reclaimed bytes, orphan backlog, and watchdog observations. + +Periodic perf output and cumulative benchmark output retain count, average, +p50, p99, and maximum. Phase scopes and whether counts describe requests, +batches, or replicas are explicit. Nested percentiles are not added. Ordinary +append samples must not conceal control-plane waits inside a mirror stage. +Publication counts and latency include initial attachment and writer adoption; +opening a read-only handle does not count as a publication. + +Performance acceptance uses release artifacts and a matched three-node native +fixture with `mem-block` KV/WAL and retained MemDisk payload bytes. Record the +replica count, buffer/queue settings, checkpoint policy, binary identities, and +fixture parameters. The fixture removes physical-media latency without skipping +framing, transport, synchronization responses, or read validation. It proves +process/reopen behavior while retained media exists, not power-loss durability; +durability fault tests use storage that survives the tested failure. + +The regression workloads and minimum gates are: + +- 4 KiB append, one producer: at least 250 ops/s and p99 at most 10 ms. +- 4 KiB append, 32 producers: at least 3,000 ops/s and p99 at most 20 ms. +- 1 MiB append, eight producers: at least 125 MiB/s and p99 at most 150 ms. +- 4 MiB rollover, eight producers: at least 200 MiB/s and p99 at most 500 ms. +- 4 KiB random reads, 32 producers: at least 30,000 ops/s and p99 at most 5 ms. +- 320 MiB replay, one producer: at least 300 MiB/s and p99 at most 2 s. +- 576 MiB prefix trim: at least 1,000 MiB/s and p99 at most 100 ms. + +These are regression floors, not a claim that multi-millisecond memory-backed +append latency is the performance objective. Matched before/after distributions +must show the effect of removed KV waits and the combined write/sync RPC. +Separately report standalone Stream append, actual Chunk-KV WAL, and client +mutation latency. + +Sentinels require zero errors/watchdogs, bounded admission, less than 256 MiB +peak workload RSS growth, no per-append ChunkDB cursor or chunk record writes, +history-independent normal append work, and at most one tail chunk scanned on +recovery. With optional checkpoints disabled and close outside measurement, +chunk record publications do not exceed initial attachment plus rollover count. Cache +and lookup accounting follow accessed records and configured budgets. GC must +release at least one complete eligible chunk. ## 9. Errors and Invariants -`StreamError` distinguishes invalid requests, admission backpressure, stale -writers, definitely absent appends, ambiguous resolution, stalled writes, -unavailable reads, corruption, and internal invariant failures. - -The central invariants are: - -- returned append ranges form one durable ordered tail; -- append never updates group 0 or stream KV metadata per batch; -- ambiguous data is never resubmitted at a guessed offset; -- one append belongs to one chunk; -- logical offsets never renumber after rollover or trim; -- logical trim publication precedes physical deletion; -- immutable page coverage is gap-free over the retained sealed interval; -- an ownership epoch can advance but cannot regress; -- read-only open never changes writer epoch, manifest generation, active chunk, - or durable cursor; -- a captured read-only tail is finite and cannot silently widen during replay; -- trim and metadata reclamation stay behind every externally owned overlay - pin; and -- watchdogs observe without cancelling, retrying, or completing IO. - -## Open Issues - -None. Metadata-group rebinding, per-stream sharding, and EC conversion are -explicit scale-out non-goals for this design. +`StreamError` distinguishes invalid requests, backpressure, stale writers, +definitely absent operations, unresolved outcomes, stalled writes, unavailable +reads, corruption, and internal invariant failures. Absence cannot be inferred +solely from a stale checkpoint cursor. + +- **I1 — Durable completion.** A successful append has exact ordered logical + ranges and synchronized data on every required mirror replica. Per-append + ChunkDB cursor and chunk record updates are not completion prerequisites. +- **I2 — Bounded record work.** Historical chunk records retain their bytes; + publication touches only the tail and successor, and open/seek loads only + bounded target records. +- **I3 — Publication and ownership.** Data is not admitted to an unattached + chunk; attached records describe complete append groups; epochs never regress; + Group-0 lease expiry or release excludes stale acknowledgments/publications. +- **I4 — Bounded recovery and views.** Only the last referenced chunk can need + parsing. Recovery preserves acknowledged bytes beyond old checkpoints and + establishes a validated contiguous prefix. Read-only ends are finite; an + owner-unavailable checkpoint fallback explicitly permits an older view and + cannot silently satisfy a newer requested cutover. +- **I5 — Retention.** Logical trim precedes deletion and reclamation respects + whole-chunk liveness, logical intervals and every retained consumer pin. +- **I6 — Consumer separation.** Wal and business streams share the same storage + contract. The consumer owns business-record and retry-result semantics. +- **I7 — Logical addressing.** One append stays in one chunk, framing bytes never + enter logical offsets, compatible multi-frame payloads use compact locations, + recovery accepts the entire append or none of it, and rollover/trim never + renumber retained bytes. +- **I8 — Chunk isolation.** Internal chunk layout and `chunkinfo` changes do not + mutate Stream chunk record generations or copy Stream metadata. +- **I9 — Synchronized write RPC.** A synchronized write response follows both + write and flush; it does not require a separate client fsync round trip. +- **I10 — Resource and observation bounds.** Queues, batches, buffers, record + caches, recovery, and GC are bounded; the hot path adds no lock, and watchdogs + observe without taking over IO completion. + +## 10. Open Issues + +There are no unresolved decisions for the append/recovery contract. Verifiable +bounded frame groups, all-required-mirror completion, Group-0 owner leases, and +explicit live-view bounds define the contract. Byte packing, decoder fixtures, +and rollout are implementation work under those constraints. + +Metadata-group rebinding, per-stream sharding, and sealed-chunk EC conversion +remain separate scale-out topics. diff --git a/doc/design/chunkio/design-crowdb-chunkio.md b/doc/design/chunkio/design-crowdb-chunkio.md index 490f97005..50d71193c 100644 --- a/doc/design/chunkio/design-crowdb-chunkio.md +++ b/doc/design/chunkio/design-crowdb-chunkio.md @@ -27,13 +27,31 @@ writer derives its shard scheme from that strip's metadata and retains its separate parity and repair flow. `ChunkReader` likewise dispatches each strip to mirror or EC recovery based on stored geometry. +`MirrorStripFlow` routes both ordinary mirror writes and replacement images +through synchronized `DiskWriter` operations. The production routing and metrics +adapters preserve that completion contract: matching DiskIO peers perform ordered +write and flush before one response. Oversized immutable view lists complete in +byte order and only their final request synchronizes the preceding range. +Missing synchronization proof fails directly. A replacement is synchronized +before ChunkDB publishes its new segment mapping; successful surviving replicas +are retained. Stream and WAL replacement images initialize their complete strip +with a retained prefix and zero suffix in one attempt, without automatic retries. +An uncertain initialization cannot authorize publication or immediate reuse. + +`DiskWriter::drain_segment` preserves the segment ordering base and one absolute +deadline through metrics and routing adapters. Its empty synchronized request +waits for earlier backend writes and disk flush; a missing completion proof fails +directly. The caller must first exclude old producers and late admissible requests. +Recovery and allocation reuse cannot substitute an independent disk fsync for +this ordered barrier. + EC shard capacity is the persisted segment's `unit_count × unit_kb × 1024`, independent of the allocation-unit size. ISA-L consumes only each data shard's actual byte prefix. A partial first shard produces equally short code shards; once a full data shard exists, each code shard spans the full shard width. No logical padding is written. Device alignment belongs to DiskIO. -Writers seal with an exact byte cursor; KiB length fields remain coarse -compatibility fields and must not define the valid data boundary. +Writers seal through one exact-byte RPC, including zero for an empty chunk. +Rounded KiB fields summarize chunk/strip geometry and do not define valid bytes. ## Table of Contents @@ -300,6 +318,16 @@ new `ChunkWriter`. The buffer is re-pushed to the new chunk. The `ProtoLocation` array accumulates one entry per rotated chunk, ordered by `logical_offset`. +Object writers use the common revision-2 ordinary frame codec: 15-byte header, +20-byte footer and at most 65,501 payload bytes per 64 KiB frame. The role is +ordinary; Stream head/follower markers are never added to object payloads. +Unknown-length uploads keep bounded tail buffers and publish an object only +when the complete upload finishes. Native socket buffers reserve the same +header/footer geometry and retain payload ownership through scatter/gather IO. +Readers retain explicit V1 support, including both historical CRC conventions; +revision 2 has canonical header/payload CRC coverage with separate chunk-ID +validation, shared with the C++ codec. + ### 6.1 Location `ProtoLocation` (from `crowdb-protocol::chunkdb::rpc::Location`) records @@ -315,8 +343,10 @@ is flatbuffers via the flatbuffer `Message` trait (`encode_to_vec` / `decode`). Edge cases: - Empty object (size 0) → `Vec` is empty; no chunk allocated. -- `logical_length` may be < `length` in future hole-punch scenarios; the - writer always sets them equal. +- `logical_length` counts object payload bytes. `length` counts physical + bytes including ordinary frame headers and footers. Readers infer the + supported frame geometry from this pair and verify each actual frame kind + and revision before exposing payload views. - `max_chunk_size` not a multiple of strip data capacity → rotation still happens at strip boundaries; the actual chunk size is the multiple-of-strip value at or just above the threshold. @@ -372,7 +402,7 @@ Edge cases: ## 8. Interaction with Neighbors - **chunkdb** — the writer calls allocate / append / seal / delete / - update_chunk_strip / query via `ChunkAllocator`. chunkdb handles + fenced strip-range replacement / query via `ChunkAllocator`. chunkdb handles placement and lifecycle; the writer is unaware of internal placement logic — it receives `Segment` placements and writes to them. - **diskio** — the writer writes data and parity blocks through the single diff --git a/doc/design/console/design-crowdb-console.md b/doc/design/console/design-crowdb-console.md index 9a72e5e86..67a027abc 100644 --- a/doc/design/console/design-crowdb-console.md +++ b/doc/design/console/design-crowdb-console.md @@ -37,14 +37,6 @@ design is detailed in the sub-design `design-crowdb-console-ui.md`. - [6.4 Resolution rules](#64-resolution-rules) - [6.5 Frontend contract](#65-frontend-contract) - [7. CLI Design](#7-cli-design) - - [7.1 Four-Domain Hierarchy](#71-four-domain-hierarchy) - - [7.2 Command Hierarchy](#72-command-hierarchy) - - [7.3 `cluster init` — bootstrap special case](#73-cluster-init--bootstrap-special-case) - - [7.4 `cluster clean` — data wipe boundary](#74-cluster-clean--data-wipe-boundary) - - [7.5 `kv server delete` — graceful + require-empty](#75-kv-server-delete--graceful--require-empty) - - [7.6 Bench subcommand](#76-bench-subcommand) - - [7.7 Bench lifecycle verbs (deploy / prepare / run / teardown)](#77-bench-lifecycle-verbs-deploy--prepare--run--teardown) - - [7.8 S3 mini-clusters and benchmarks](#78-s3-mini-clusters-and-benchmarks) - [8. Error Model and Operation Logging](#8-error-model-and-operation-logging) - [9. Observability](#9-observability) - [10. Hardware mutations](#10-hardware-mutations) @@ -62,15 +54,13 @@ design is detailed in the sub-design `design-crowdb-console-ui.md`. - Multi-tenancy and a general audit-log service. - Making a console-local file authoritative for cluster topology. -Local deployments use one stable directory per logical server below their -runtime namespace. Each server owns its `data/`, `config/`, `log/`, and -`artifacts/` directories. Process IDs are recorded as live-process ownership, -not used as logical directory identities, so restart continues to use the same -server directory and port assignments. - -The default simulated topology is one rack containing all requested nodes. -Benchmarks that need distinct failure domains create additional racks -explicitly; full-stack deployment does not silently move nodes between racks. +Ordinary deployment runs the shared OCI node image. The host lifecycle owns +containers, applied resources and private ownership receipts; each node monitor +owns its service processes and retained data. Local evaluation may place several +logical nodes on one host, with zero physical host failure tolerance. Production +placement uses the confirmed physical failure domains. Test-only measurement +fixtures may own native processes but do not define a second public deployment +lifecycle. ## 2. High-Level Architecture @@ -101,12 +91,15 @@ path through the shared `ops` module: - **Web**: `user → crowdb-web (Axum) → shared (ops module) → group-0 sysdata + crowdb-kv-server` - **CLI**: `user → crowdb-cli → shared (ops module) → group-0 sysdata + crowdb-kv-server mgmt` -Both frontends build an `OpContext` with Group 0 discovery seeds. The CLI -uses `--system-ip` and `--system-port`; production Web uses a versioned process -configuration. Both read confirmed hardware and logical records through Group 0 -and resolve node management endpoints from live registration. The process -launch registry is local to each bare-metal console. Docker process state is -owned by `crowdb-monitor`. +Both frontends build an `OpContext` with Group 0 discovery seeds. Attached CLI +commands select `--cluster` or `--management-endpoint`, verify the confirmed +cluster UUID through the management origin, then verify the same UUID through +native Group 0 before hardware/KV operations and native workloads. Production +Web uses a versioned process configuration. Both read confirmed hardware and logical records through Group 0 +and resolve node management endpoints from live registration. Ordinary OCI nodes +use `crowdb-monitor` for process ownership on both Docker and containerd. A +configured auxiliary Web process may retain a local launch registry; it is not +required for node formation or endpoint-only attachment. The CLI talks directly to group-0 system metadata via `CrowdbSysmdClient` and to individual `crowdb-kv-server` management @@ -145,12 +138,12 @@ call. - The web SPA **does not** reimplement business logic; it calls `shared` via the Axum backend, never `crowdb-kv-server` directly. - Both frontends build an `OpContext` and call `shared`'s `ops` - module directly — the CLI from `--system-ip` / `--system-port` global - flags, the web backend via `AppState::op_context()` (sharing the + module directly — the CLI from its verified application target, + the web backend via `AppState::op_context()` (sharing the cached `CrowdbKvClient` and Group 0 management seeds). - Shared hardware and logical operations provide the same authority and - conditional publication rules to both frontends. Deployment-mode policy - controls which process and hardware mutations Web exposes. + conditional publication rules to both frontends. Node count and runtime do not + select UI permissions; required authority and resource ownership still apply. ## 3. Data Model @@ -356,29 +349,21 @@ child recovery and status to `crowdb-monitor`. ### 6.1 Design Rules -Configured managed Web uses the managed router and a versioned process configuration. -`/api/preview` combines confirmed Group 0 records, live registration, and the -mode-specific process view. `/api/stores/...` provides authenticated logical -mutations through shared operations in both modes. Bare-metal Web additionally -exposes rack, node, disk-group and disk reads and authenticated mutations, plus -registry-backed launch controls and bootstrap. Automatic single-node Docker Web does not expose -hardware or process mutation routes. Manual node Web exposes candidate/rack -preparation before bootstrap and routes subsequent shared management through -Group 0 and authenticated monitor execution. Unknown managed API routes report -unavailable rather than entering an in-memory topology path. - -A mutation is accepted only after the required node-side steps and Group 0 -publication are confirmed. The Console assumes a root operator until UI login is introduced. Automatic single-node container -mode rejects topology, deployment, and disk-management writes at the -backend; logical and Access operations use server-held protocol credentials. -Docker Web is writable by default. The process configuration explicitly sets -`readonly = true` to reject every non-read request with 403; `/api/mode` exposes -this flag and the SPA disables write controls accordingly. The public -single-container profile enables readonly, while internal writable profiles -select the same release image with readonly disabled. Docker physical-management -restrictions apply independently of readonly. -The SPA calls the Axum backend; it does not talk directly to KV management -endpoints. +The ordinary OCI entrypoint starts a node Web with an editable UI in both +one-member and three-member clusters. Automatic preparation and runtime choice +do not restrict management writes. Before bootstrap, candidate/rack preparation +uses retained local admission state; afterward, hardware and logical mutations +use confirmed Group 0 authority and authenticated monitor execution. Configured +managed Web retains its versioned process configuration and requires confirmed +records; unknown managed routes never enter an in-memory topology fallback. + +A mutation returns success only after the required node-side steps and Group 0 +publication are confirmed. The Console assumes a root operator until UI login +is introduced. Explicit `readonly = true` rejects non-read requests with 403; +`/api/mode` exposes that setting and the SPA disables writes accordingly. The +ordinary node default is writable. Credentials remain in the established private +store, and invalid authority, ownership or placement rejects the operation for +that cause. The SPA calls the Axum backend rather than KV management endpoints. ### 6.2 In-process test API @@ -437,221 +422,39 @@ backend-facing contract here: ## 7. CLI Design -- Binary: `crowdb-cli` (four-domain structure: `crowdb-cli `). -- Parser: `clap` derive; one module per domain under `commands/`. -- **Direct-to-system-group call path.** Every verb builds an `OpContext` - seeded with `--system-ip` / `--system-port` (any system-group management - endpoint) and talks directly to system metadata via `CrowdbSysmdClient` and to - individual `crowdb-kv-server` mgmt APIs via `ServerClient`. There is - no `crowdb-web` intermediary. Leader discovery is automatic. The global - connection flags are `--system-ip` (default `127.0.0.1`, env - `CROWDB_SYSTEM_IP`) and `--system-port` (default system-group management - port, env `CROWDB_SYSTEM_PORT`). -- Output is console-first and human-readable. Every invocation identifies the - copyable command and final result. HTTP operations show request/response - headers and body disposition; JSON and XML response bodies are pretty - printed, while binary bodies are represented by type and byte count. ANSI - colors distinguish commands, HTTP sections, body metadata, warnings, errors, - and final success/failure. -- Every non-benchmark command logs only to the console. Each benchmark writes - the same tracing events to the console and to - `/cli-log/bench-/`; a new run replaces that family's previous - directory so benchmark logs do not accumulate. - -The full command hierarchy is defined in the `clap` derive structs; -this section covers design rules only. - -### 7.1 Four-Domain Hierarchy - -`cluster` owns hardware metadata and bootstrap, clean, destroy and status. -`kv` owns KV server launch controls, logical store/group/replica operations -and KV data commands. `chunk` owns storage-service launch controls and -maintenance. `bench` owns workload runners. The CLI connects to Group 0 -directly and shares the authority operations with production Web. - -### 7.2 Command Hierarchy - -The `clap` command enums define the exact verbs and flags. Hardware and -logical commands use Group 0 for identity and membership. Process controls -require `--registry`; `cluster init` additionally requires sealed bootstrap -input for first creation. Development `local-deploy` runs a one-shot loopback -cluster and prints the Group 0 management seed for later CLI invocations. - -### 7.3 `cluster init` — bootstrap special case - -`cluster init` requires `--registry` and a versioned `--bootstrap-file` -for first creation, or a sealed retry intent beside the registry. It takes -`--nodes ` and bootstraps group-0/store-0 on those nodes via -direct node REST calls (the `POST /system/init` mechanism, §4.3), -wires remotes, and writes the hardware + KV-cluster topology into -group-0 sysdata. After `cluster init` completes, subsequent commands -use `--system-ip` / `--system-port` to connect to any node in the newly -created system group. - -Initialization also sends Group 0 management seeds to deployed KV processes -outside the selected member set and waits for exactly one live registration -per node. Those processes retain connection hints locally across restart; -they do not become Group 0 members. Logical operations use the confirmed live -registrations rather than treating launch configuration as a live endpoint. - -### 7.4 `cluster clean` — data wipe boundary - -`cluster clean --store --group ` derives the target replica nodes -from confirmed Group 0 membership and resolves every live KV management -registration. It asks each target to wipe user data, then waits for a new -leader. Group 0 hardware, logical records, and process launch policy remain -intact. A missing group, registration, or acknowledgement fails the operation; -a local launch record cannot justify a wipe. - -`--restart-services` additionally restarts locally configured DiskIO, DiskDB -and ChunkDB processes in dependency order through `LaunchRuntime`. It requires -a validated launch registry before the wipe begins. KV processes stay running -so Group 0 remains available. - -### 7.5 `kv server delete` — graceful + require-empty - -All operations use graceful Paxos reconfiguration — no force-kill -path. `delete` requires the server to be **empty**: no replicas, no -groups, no stores hosted. The operator must delete in bottom-up order -— replicas → groups → stores → server → node — before the server or -node can be removed. The CLI refuses `kv server delete` if the server -still hosts replicas, with an error listing the replicas/groups/stores -that must be removed first. Same policy applies to `cluster node -remove` — the node must have no running servers/services before -removal. - -Verb distinction: -- `kv server stop` — graceful process stop (keeps server entry, can - restart later). No reconfiguration; replicas remain registered on - peers. -- `kv server delete` — graceful removal (requires empty server). - Removes the server entry after confirming emptiness. No cascading - delete — the operator does the cascade manually in bottom-up order. - -### 7.6 Bench subcommand - -- `bench kv ` runs KV workloads against a target - store/group. `bench rpc` measures raw RPC transport throughput. -- Bench discovery starts from the explicit Group 0 management seed and - resolves metrics hosts from confirmed replica membership and live - registrations. It does not load a console topology file. - -### 7.7 Bench lifecycle verbs (deploy / prepare / run / teardown) - -The all-in-one `bench kv` verb deploys a 3-node cluster, pre-populates -keys, runs the workload, and tears down — all in one process. For -regression suites that run many sub-tests against the same cluster -configuration, this pays deploy + pre-pop overhead per sub-test. The -lifecycle verbs split this monolith into discrete steps with persistent -deploy metadata: - -- **`bench deploy --name --kind kv --mode mem`** — provisions a - 3-node cluster via `BenchFixture` (embedded console-web), then - detaches the fixture so the `crowdb-kv-server` processes survive CLI - exit. The deploy metadata (node pids, endpoints, tunables) is - serialized below `.crowdb-runtime/persistent/bench//handle.json` - (`ClusterHandle`). The - `--kind` flag dispatches to kv (default), rpc (spawns - `crowdb-rpc-fb-server`), or chunk/storage (not yet implemented). -- **`bench prepare --target --keys N`** — loads handle, builds a - `CrowdbClient` from the recorded leader endpoint, and writes N keys - via sequential `put`. Reuses the same `format_key` / `value_for` - logic as `bench kv`'s pre-populate path. -- **`bench run --target --workload read ...`** — loads handle, - builds an `AttachedKvTarget` (implements `BenchTarget` with no-op - provision/cleanup), and calls the shared `run_bench` runner. Reports - go to that namespace's `artifacts/runs//`. The cluster stays running - after the run — multiple `bench run` invocations can attach to the - same deploy. -- **`bench teardown --target `** — loads handle, SIGTERMs the node - pids via `stop_pid_with_timeout`, removes `handle.json`. Idempotent: - a second teardown on the same name exits 0 with "already torn down". - -The all-in-one `bench kv` verb is preserved as the quick one-shot path. -Regression scripts use the lifecycle flow: deploy once, prepare when needed, -run compatible sub-tests with reset boundaries, then teardown once. Each -sentinel accepts environment overrides for case selection and duration; KV -read, write, and scan also accept a reduced keyspace. These controls provide a -short structural smoke without changing the default regression matrix. - -`ClusterHandle` is persistent namespace metadata, not a config extension. The -single `.crowdb-runtime/` root is gitignored. The -`crowdb-kv-server` child processes survive CLI exit because -`lifecycle::deploy_local` spawns them with `kill_on_drop(false)`. - -### 7.8 S3 mini-clusters and benchmarks - -`crowdb-cli s3 cluster start --root --container ` creates or -resumes three formal release containers. The named first container and its -`-node2` / `-node3` members share a cluster-owned Docker bridge and fixed service -ports. Docker must already be available. The CLI uses normal node admission, -Group-0 bootstrap and service deployment; clients run outside the containers. -`--image`, `--node-cpus` and `--memory-mib` select the initial image and per-node -limits. Existing clusters keep their recorded immutable image and limits. -`--diskdb-heartbeat-interval-secs` selects the ordinary DiskDB cadence at -creation (default 10 seconds). The owner record retains it across resume and -restart; an existing cluster rejects a different requested value. - -A root path selects the persistent data directory. `--container` (alias -`--name`) can select any recorded member and resolves its owning cluster for -lifecycle, bucket and object commands. Conflicting root/name selections or -foreign container, image, network or mount ownership fail before mutation. -Each node mounts its own data, logs and crash directory below the cluster root. - -The dependency order is KV, DiskDB, DiskIO, ChunkDB, Chunk-KV, then Access. -Readiness waits for usable dependencies before starting their consumers. -`start` resumes confirmed Group-0 and service state after a client interruption; -completed services keep their identities and no local topology is reconstructed. -The initializing marker disappears after successful provisioning. - -`status` reports running containers and endpoints. `stop` stops all members and -preserves containers, network and data. `restart` starts those same containers -with fixed addresses and retained data. `delete` validates ownership, stops and -removes members and their network, then removes the owned data root. Unknown -files and resources are preserved. Host data clients use the persisted Web S3 -proxy endpoint, whose server-held credentials authenticate requests to Access. -S3 errors are preserved without falling back to another mutation. - -The versioned owner record contains cluster/container identity, image, network, -resource limits, DiskDB heartbeat cadence, fixed addresses and provisioning phase. Client records contain -endpoints and Group-0 seeds, without rack, node or logical topology. Cluster -credentials remain in the normal private node credential directories. - -`crowdb-cli bench s3` owns a separate, invocation-scoped memory profile. KV and -WAL blocks use memory backing, DiskIO uses memory disks, and chunk-KV keeps its -readable tree and value path in memory. The memory disks retain a large logical -address space while allocating resident pages on demand; logical capacity is -therefore independent of the benchmark's resident-memory budget. The benchmark -always stops its cluster, while leaving the stopped work directory and logs as -diagnostic artifacts. - -The benchmark has write, read, range-read, list, and deterministic weighted -mix workloads. It prepares a fixed dataset, runs warm-up outside measurement, -then admits requests until either the operation limit or duration limit is -reached. Workers keep local latency and failure accumulators and use only an -atomic admission counter on the request path. Cleanup deletes only prepared or -actually written benchmark objects, with bounded asynchronous concurrency. - -Results report backing choices, measured duration, attempts, successes, -failures, throughput, average latency, p50, p99, prepared bytes, and process -peak resident bytes. Read validates the full response length, range-read -validates the selected interval length, and list validates ordering and -continuation progress. Failures are separated into metadata, protocol, -transport, and resource categories. RSS is sampled asynchronously; exceeding -the configured budget makes the run fail instead of publishing valid -throughput. - -The following invariants apply: - -- **S3-C1 — Recovery boundary:** a validated owner record identifies a resumable - container cluster; the ready phase marks completed provisioning. Stop never - removes its stored data. -- **S3-C2 — Real request path:** every benchmark operation traverses the normal - S3 endpoint and validates the response property relevant to its workload. -- **S3-C3 — Bounded execution:** duration, operation count, concurrency, and - resident memory are explicit limits; exhausting memory is an error. -- **S3-C4 — Simple ownership:** persistent operator data and ephemeral - benchmark data use separate storage profiles and lifecycle ownership. +- `cluster start/stop` own OCI runtime lifecycle through a named local ownership + receipt. `start` accepts one or three nodes and delegates preparation to the + normal node monitor. Runtime inspection, logs and image operations use the + runtime's tools; cleanup belongs to the independent host/Pixi helpers. +- `cluster status/check` observe confirmed application readiness over management + endpoints, without a runtime socket. A finite check returns nonzero when any + required dependency is unmet. Runtime facts unavailable remotely remain unknown. +- Hardware, logical KV, S3 and benchmark commands require `--cluster` or + `--management-endpoint`. Every selected origin must report the same confirmed + UUID; saved bindings pin that UUID. Native consumers additionally verify + transport reachability and the committed Group-0 UUID before data access. + There is no system-address, localhost or launch-registry fallback. +- Hardware and logical KV operations call Group 0 and node management APIs + directly after target verification. Signed S3 clients obtain access credentials + and the advertised public URI from the selected management origin. +- Endpoint-only attachment grants no runtime ownership. It cannot start or stop + containers, create a runtime receipt, change the image or prepare infrastructure. +- The retained workload families are KV, DiskDB, ChunkDB, ChunkIO and S3. + `bench kv prepare` writes its declared dataset into an existing group. + Workloads never initialize groups, deploy services, reset cluster topology or + stop a cluster. Workload-owned data cleanup and ordinary client transport + workers remain allowed. Standalone RPC tests own their server/client lifetime. +- `kv store/group/replica/data` retain normal topology and data operations. + Process deployment and reset commands are absent from the command tree. +- Output identifies the copyable command and final result. HTTP exchanges show + response metadata and format structured bodies; binary bodies are identified + by type and byte count. Status/check support structured JSON output. +- Ordinary commands log only to the console. A benchmark writes the latest run + under the runtime root's `artifacts/cli/bench-` directory; the next run + replaces only that family's previous benchmark log. KV workloads also retain + a structured `report.json` with operation counts, errors and measured latency + alongside the human-readable console report. Measurement tools preserve each + completed invocation before another run replaces the latest directory. ## 8. Error Model and Operation Logging @@ -663,7 +466,8 @@ The following invariants apply: body marker. Text is visible, XML/JSON is formatted, and binary content is summarized by byte count without duplicating it in the trace. - **Retained diagnostics** — only benchmark commands write files. The latest - run for each benchmark family lives beneath `/cli-log/bench-/` + run for each benchmark family lives beneath the runtime root's + `artifacts/cli/bench-/` and replaces the previous run. Benchmark tracing is also mirrored to the console. Routine outbound HTTP calls are debug-level so default output keeps lifecycle events, warnings, and errors without per-request noise. @@ -685,12 +489,12 @@ The following invariants apply: ## 10. Hardware mutations -CLI and bare-metal Web use the shared Group 0 hardware operations. Rack, +CLI and node Web use the shared Group 0 hardware operations. Rack, node, disk-group and disk changes update parent and child records in one conditional batch where membership changes. Matching retries are confirmed; -conflicting concurrent writes preserve the existing record. Automatic single-node Docker Web does -not expose hardware or process mutations. Manual Docker nodes use the same -shared hardware operations after confirmed Group 0 publication. +conflicting concurrent writes preserve the existing record. One-member and +three-member OCI nodes expose the same shared management surface after +confirmed Group 0 publication. The physical Cluster workspace shares one tree and selection across Topology and Capacity. Cluster/Rack node cards and capacity tables reuse bounded observations; @@ -712,16 +516,20 @@ from incomplete state. Secrets remain in the established private credential path ## 11. Cluster teardown and verification -`cluster destroy` requires the local launch registry. It reads confirmed Group -0 membership, removes user stores and groups through shared logical operations, -then removes the system group last. Only after metadata teardown succeeds does -it stop processes named by that console's launch registry. A failed or -unconfirmed step returns an error instead of deleting presumed local topology. - -There is no orphan-guessing reset command. A stopped or unreachable node does -not imply its membership should be deleted. `cluster clean` derives its -replica targets from Group 0, wipes each live target, and waits for a new -leader while preserving topology. +`cluster start`, `cluster stop`, `cluster status` and `cluster check` form the +common CLI lifecycle. Start and stop require matching private runtime ownership; +stop retains accepted inputs and data. Endpoint-only attachment supports current +observation and workloads without granting runtime stop or cleanup authority. +Independent OCI cleanup validates the recorded daemon, exact container and +resource identities before removing owned runtime resources. It preserves data +by default; explicit owned-data deletion is separate from stopping services. + +Application cleanup is a distinct authenticated operation. It validates the +retained bootstrap, retires further setup/recovery, drains owned services and +removes the matching system store through its local deletion endpoint. Failed +members remain pending for explicit retry; an unavailable node is never proof +that its membership or data should be deleted. No cleanup guesses orphan +ownership or reconstructs topology from host-local process records. Manual node discovery and bootstrap follow the [deployment architecture](../deploy/design-crowdb-deploy.md). The Cluster sidebar diff --git a/doc/design/deploy/design-crowdb-deploy.md b/doc/design/deploy/design-crowdb-deploy.md index 1ec997f41..caad22017 100644 --- a/doc/design/deploy/design-crowdb-deploy.md +++ b/doc/design/deploy/design-crowdb-deploy.md @@ -62,7 +62,7 @@ operator browser -> any node's Web UI / shared console operations -> Group 0: committed topology and operation authority -> target node monitor - -> Docker node container and managed server processes + -> OCI node container and managed server processes node monitor <-> management-network discovery <-> peer node monitors ``` @@ -119,6 +119,10 @@ change either identity. Duplicate live UUID claims are identity conflicts. - **I8 — Bootstrap gate:** only Group-0 prerequisite processes and Group-0 initialization run before the shared information platform is ready. Other deployment requires committed initial topology. +- **I9 — Identity-bound address recovery:** a changed address is usable only + after the peer proves possession of the SSH host private key pinned to its + accepted cluster UUID, discovery UUID and numeric node ID. Discovery cannot + change this binding or the accepted voting membership. ## 4. Node discovery @@ -251,7 +255,8 @@ requires an explicit recovery/decommission procedure, not an automatic timeout. 1. Any UI collects compatible candidates and the operator's rack assignments. 2. The operator selects initial Group-0 voting members, or Auto join selects - three authenticated eligible candidates in discovery-UUID order. Candidate + the explicitly selected one or three authenticated eligible candidates in + discovery-UUID order. Candidate count is not voting-member count; application nodes need not all vote. 3. Submit the bootstrap identity, immutable initial manifest and initial data group choice to the selected member with the smallest discovery UUID. Its @@ -299,27 +304,32 @@ them separately and do not automatically merge their logs or topology. ### 6.3 Recovery and subsequent admission -Standard node workspaces persist an enabled-by-default Auto join policy. The Web -backend evaluates it without a browser connection, using the existing authenticated -admission operation and configured rack assignments. A workspace with no rack waits -for one. Missing key access remains Needs access; automatic admission does not -supply a password. Multiple advertised clusters without a configured local draft -require target selection rather than selecting an arbitrary advertisement. - -Before bootstrap, fewer than three eligible nodes remain pending. The first three -in UUID order form the fixed manifest, and the smallest UUID member's monitor -executes initialization. Automatic initialization creates only Store 0 / Group 0. -After publication, admission can add nonvoting nodes but does not replace or expand -the initial voters. Quorum loss waits for authority recovery. A persisted target -without its binding is a recovery condition, not permission to create a cluster. -Turning Auto join off stops new scheduling while accepted monitor work continues. - -Startup mode is durable node policy. Standard and explicit single mode cannot -replace one another by changing an environment variable, including before the -first successful bootstrap. Concurrent first starts publish only one mode. -Existing single-profile manifests retain their identity and profile/configuration -validation. A single discovered node does not imply single mode, and an accepted -standard bootstrap cannot be reinterpreted as single-profile state. +Node workspaces persist an enabled-by-default Auto join policy. The Web backend +evaluates it without a browser connection through authenticated admission and +accepted deployment inputs. Local policy supplies a test rack and credentials; +Production requires configured rack assignments and access. Missing Production +credentials remain Needs access, without a Local fallback. Multiple bound cluster +UUIDs require explicit target selection rather than an arbitrary advertisement. + +The accepted initial voter count is one or three. An incomplete three-node set +remains pending; an observed overfull fresh set reports conflict before acceptance. +Eligible UUID ordering selects the coordinator within that set, while monitor and +KV-server accepted-manifest fencing prevent conflicting execution. Fixed voters +never expand from later advertisements. Turning Auto join off stops new scheduling +while already accepted monitor work continues. + +After Group-0 publication, one committed default preparation intent owns the +ordinary metadata group, replica mapping and shared access credentials. Authorized +workers resume that intent across Web/coordinator loss. Each monitor interleaves +disk preparation with dependent service startup; DiskIO may start before Online +because its ownership proof enables Online. Explicit later service stops or group +retirement supersede historical setup and prevent automatic recreation. + +The versioned deployment policy is durable and independent of UI permissions. +Local and Production inputs cannot replace one another through an environment +change. Initial voter count, storage, listeners and scope are sealed before node +identity or setup effects. Retained values outrank new image defaults; corrupt, +conflicting or obsolete populated roots fail without implicit reset. Restart resumes accepted identity and replica WAL state. Lost multicast does not erase stored Group-0 contacts. An interrupted bootstrap reuses its operation @@ -353,7 +363,81 @@ SSH connection/authentication has a ten-second bound; bidirectional proof has a twenty-second bound. Passwords are discarded, sessions are reused during setup, and only public keys cross nodes. -### 6.4 Voting replica admission +### 6.4 Authenticated member address recovery + +Node identity survives changes to management and data addresses. Each accepted +member has a durable binding between cluster UUID, discovery UUID, numeric node +ID and its Ed25519 SSH host public key. Admission retains the canonical public host key in its Group-0 node record; +normal address/rack updates cannot replace that pin. The SSH login key authenticates a caller; +the SSH host key authenticates the peer. Both private keys remain on their owning +node's persistent volume. Address recovery reuses SSH and does not introduce a +CA, certificate service or custom signature protocol. + +Initial trust is established by the existing bootstrap/admission procedure on +the deployment's trusted management network. Local test credentials are a +convenience, not production identity verification. An mDNS announcement alone +cannot establish trust. Before bootstrap replicas start, each selected member +durably accepts the same member-to-host-key bindings alongside its fixed +manifest. Subsequent admissions and revocations use committed Group-0 authority; +participants retain the applicable bindings locally for recovery without quorum. +A pending non-voter seals the initial manifest's authenticated pins and its own +admission pin before its first KV start. This does not authorize recovery: only +confirmation distributes `ConfirmedMemberTrust`, whose registry and pins each +monitor verifies through current linearizable authority. The retained snapshot +adds contact hints while preserving the original voting/store configuration. +The independently running contact worker refreshes it when authority returns; +its cached confirmed pins permit SSH identity proofs during quorum loss. +Current authority retirement durably fences the local admission before owned +KV/storage services stop. Missing or corrupt later-member snapshots cannot +resume the member or answer a recovery identity request. Discovery cannot promote a pending admission or revive a removed member. Missing +or corrupt trust records fail closed; a new address never triggers first-use +trust or automatic key replacement for an existing identity. + +Recovery proceeds independently of Group-0 availability: + +1. mDNS/DNS-SD or a configured seed reveals a candidate's current management + endpoint. Its advertised UUIDs and node ID are untrusted lookup hints. +2. The monitor selects an already accepted member binding and opens SSH to that + address with the expected host key, rather than trusting the IP's previous + occupant or accepting a new key. SSH proves possession of the private key. +3. Through that authenticated connection, the monitor reads the peer's durable + discovery identity and cluster/bootstrap binding. All must match the accepted + member, and the verified reply supplies that member's current endpoints. + The SSH identity command reads private retained state and verifies the owning + monitor's independent liveness endpoint. It does not enter the serialized + lifecycle executor or wait for service readiness: otherwise slow service + startup could suppress peer proofs, fence routes and prevent its own quorum + recovery. Offline roots and changed or retired identities still fail closed. +4. The local transport uses the verified endpoints for the same replica IDs in + the existing groups. It does not change votes, membership, WAL, rack placement + or bootstrap identity. Endpoint replacement invalidates old connections and + applies to subsequent reconnects; mappings use node/replica identity, not a + global old-IP-to-new-IP substitution. + Restored replicas fence old routes before their election drivers start. + Consensus requests identify their intended cluster and replica; a reachable + process at a reused address cannot contribute another replica's response. +5. Once quorum returns, normal authorized operations publish updated addresses + in Group 0. Local endpoint evidence never authorizes a management mutation. + Committed replica-to-node ownership is retained locally for subsequent + recovery. Internal service listeners and management contacts change through + conditional new service operations; later stop/delete commands win. Public + client URIs are independent of these internal addresses. + +This permits all members to restart with changed IPs at once. Neither stable DNS +names nor fixed IPs nor a complete peer-address list is required. Multicast must +reach the selected management scope; across discovery boundaries, seed discovery +remains necessary. Docker bridge DNS is optional and host networking uses the +same monitor discovery and SSH verification path. + +A changed key, foreign cluster/bootstrap, unknown member or conflicting live +claims is rejected and reported. An unverified advertisement cannot overwrite a +verified endpoint. Duplicate authenticated live claimants for one identity stop +automatic selection; a copied data volume is not a new node. Recovery has bounded +handshake timeouts and retains the last accepted identity on failure. Key rotation, +lost-identity replacement and removal remain explicit authority-controlled work; +they are never inferred from address changes or discovery expiry. + +### 6.5 Voting replica admission Node admission grants hardware/service management access; it does not add a voter. Adding a replica uses `JoinGroupRequest` to import a snapshot from a confirmed @@ -369,7 +453,7 @@ Generic join cannot bypass an accepted system bootstrap. Independent clusters remain separate: operators explicitly clean one cluster, wait for every node's cleanup, admit the released nodes to the retained cluster and add voting replicas. -### 6.5 Accepted node setup and storage +### 6.6 Accepted node setup and storage Node access review reads authenticated identity and inventory without installing peer authorization or granting admission. The browser keeps an initial password @@ -379,8 +463,9 @@ operation and sends the selected storage/services to the target monitor. The monitor persists an immutable `NodeSetupRequest` and sealed placement before starting work. Browser disconnect, another Console session and monitor restart reuse that intent. Provisional disk discovery grants no membership: only ready -Group 0 can register disks. Standard default storage is one 16-GiB preallocated -file with a 1-GiB reserve; supplied or diskless choices never silently fall back. +Group 0 can register disks. Local default storage is one 16-GiB sparse file with +a 1-GiB reserve; Production automatic storage requires preallocation. Supplied +or diskless choices never silently fall back. Different media classes use compatible independent placement. A chosen missing ordinary metadata group remains waiting instead of creating an unrelated group. @@ -413,8 +498,16 @@ and unassigning their parents require a separate explicit operator flow; deletio must not emulate that flow with an implicit cascade. Explicit service Stop/Restart/Delete supersedes old setup intent. Consecutive -startup/readiness failures have a durable five-attempt budget per operation; -readiness success resets it. Explicit cluster cleanup retires and drains accepted +process launches have a durable five-attempt budget per operation; readiness +success resets it. A bounded readiness observation that expires while the +same operation's process is alive retains that process and its attempt count. +Further observations continue startup without stopping or launching it again, +including on the fifth launch. Recovery verifies the committed local dependency +intents and their live probes before launching a dependent process. Consumers +that need the complete DiskIO topology also wait for its current authoritative +owner view, including other nodes. Waiting for these dependencies consumes no +process-launch attempt. A new explicit operation can supersede it. +Explicit cluster cleanup retires and drains accepted setup/provisioning before deleting system metadata. Cleanup dispatches to at most four retained nodes concurrently and saves each result; unreachable nodes stay pending under the original confirmed cleanup operation. @@ -513,12 +606,15 @@ Publish OCI images to Docker Hub or another OCI-compatible registry. The same image for a supported CPU architecture contains monitor/UI/server binaries and their dependencies. Node-local state and cluster identity are external mounts. Single-node and multi-node operation use the same image, monitor and UI. -Default manual startup awaits candidate selection and bootstrap for one or more -nodes. Explicit automatic single-node startup creates virtual disks and initializes -the cluster, then disables UI editing. Startup policy and the UI editing capability differ, -while identity, discovery, authority and managed-service behavior remain shared. -Read-only single-node mode rejects UI management mutations in the backend as -well as disabling editing controls. +Fresh Local startup prepares an explicitly one- or three-voter cluster through the +shared authenticated node lifecycle. The default is one voter with file-backed +test storage and zero physical host failure tolerance. Production selects explicit +credentials, resources and protected placement. Both expose editable management; +automatic preparation and node count do not determine authorization. Aggregate +status distinguishes Group-0 authority, historical setup and current full-cluster +readiness. Host helpers create runtime resources and observe this lifecycle; +they do not provision application topology through a second bootstrap path. + Tags identify releases for humans; production plans pin immutable image digests. Architecture-specific builds may share a multi-platform image index only when their dependencies have been validated. @@ -584,7 +680,7 @@ plans explicitly assign host resources; monitors validate and execute, rather than autonomously choose global placement. - **CPU/memory:** the production launcher validates CPU quotas and memory limits - against host capacity, then verifies Docker applied them. Memory is at least + against host capacity, then verifies the selected runtime applied them. Memory is at least 512 MiB. Hardware observations respect container limits. CPU affinity, NUMA and IRQ tuning remain host policy. - **Disk:** stable device references, exclusive intended ownership and supported @@ -599,7 +695,7 @@ than autonomously choose global placement. - **Monitoring:** container and process health are separate observations. Existing monitor supervision can recover eligible process exits; it must not silently override a deliberate stop, expired authorization or upgrade drain. -- **Runtime access:** the launcher uses the system Docker daemon. Node containers +- **Runtime access:** the host selects system Docker or rootful containerd. Node containers supervise local processes and receive no Docker or containerd socket. Explicit permitted block devices are passed through by the host launcher. @@ -627,6 +723,47 @@ Expose: - Desired deployment inputs, actual image digest/resources, process/container health, drift and operation outcomes. +`cluster status` reads the node's aggregate HTTP observation without a local +runtime socket. Its collector has an six-second monotonic deadline after the +bounded local monitor observation. Current linearizable Group-0 publication, +accepted ordinary metadata replicas and ordinary-group authority, physical disk +placement/bindings, live DiskDB owner leases and allocatable capacity are +independent dependencies. Automatic file storage uses its accepted initial group +reservation. Supplied storage resolves every accepted disk ID from a fresh, +identity-checked member setup observation through its unique Group-0 disk and +group membership. All resulting media-class groups must satisfy metadata +binding, current owner and space checks; a missing, duplicated or conflicting +medium stays unknown. Observation never changes sealed placement or creates the +automatic reservation for a supplied-media node. Capacity summaries must be no older than the service +registry's fifteen-second liveness window. Per-member service evidence matches +the current committed operation; required S3, Iceberg and Dataset listeners each +supply their own read-only protocol proof. Historical setup completion cannot +substitute for these observations. Missing or stale evidence remains unknown. +A bounded observation may wait for a slightly faster peer's timestamp within +its existing monotonic deadline and that evidence's freshness window, capped +at three seconds. It then evaluates the unmodified timestamps against the +actual local clock. It never treats future evidence as current, extends a +request deadline or refreshes old evidence to compensate for clock offsets. +Runtime/image facts remain unknown when runtime access was not supplied. + +Access dependency evidence comes from one backend sampler, independently of +status reads and business request outcomes. ChunkDB evidence uses one routed +point read in each of the system and user-data maintenance domains. An +authoritative absent key is healthy evidence; routing, storage, ownership and +protocol failures remain unhealthy. These observations never create probe +objects or enumerate the chunk catalog. Hardware discovery and bounded header +reads on online disks share the same overall probe deadline. Phase latency +histograms retain failed and cancelled observations and are readable without +issuing new probes. + +The aggregate route bypasses deployment recovery middleware and never reserves +preparation, restores configuration or repairs topology. Valid unhealthy status +returns success; `cluster check` returns one while readiness is unmet and zero +only for current ready evidence. Invalid or unobtainable reports return two. +Checks support bounded waiting, and JSON stdout contains only the report while +diagnostics use stderr. Explicit origins must agree with the selected cluster +UUID and name; attaching never grants runtime cleanup ownership. + Keep local logs/crash artifacts persistent and collect asynchronously. Do not replicate every log line into Group 0 or expose credentials in discovery/UI diagnostics. diff --git a/doc/design/deploy/design-crowdb-oci-image.md b/doc/design/deploy/design-crowdb-oci-image.md index aad800f79..67ff938c0 100644 --- a/doc/design/deploy/design-crowdb-oci-image.md +++ b/doc/design/deploy/design-crowdb-oci-image.md @@ -31,8 +31,8 @@ macOS implementation and Kubernetes deployment automation are deferred. prerequisites can compile, stage, construct and validate an OCI image without Docker CLI, Docker daemon, Docker socket or containerd daemon. - **I2 — One image**: for a given architecture and release, Docker and containerd - consume the same published image digest. Single-node initialization and manual - multi-node admission are startup policies, not separate image builds. + consume the same published image digest. Local and Production formation use + accepted startup policies rather than separate image builds. - **I3 — External state**: UUIDs, SSH keys, cluster bindings, server data and logs remain outside image layers. Replacing a runtime retains persistent state. - **I4 — Enforced resources**: production startup fails if requested CPU @@ -62,21 +62,21 @@ Tokio child status. This uses Tini's standard reaping/signal behavior as documen in its [upstream implementation](https://github.com/krallin/tini). The image contains the same monitor, UI, Rust/C++ servers, shared libraries, -templates and entrypoint used by both deployment modes. The common -image defaults to `CROWDB_STARTUP_MODE=manual`: discovery, SSH and UI start, then -the node waits for preparation and Group-0 bootstrap. Manual mode supports one -or multiple nodes. An operator can select the local candidate, admit it into the -prepared cluster and initialize a one-member Group 0; this is an ordinary -editable cluster and can later admit more nodes. It still requires configured -storage and passes the same preparation/bootstrap validation. - -Explicit `CROWDB_STARTUP_MODE=single` selects the existing convenience policy: -automatic virtual-disk creation and cluster initialization, followed by disabled -UI editing and backend mutation rejection. Node count alone never activates this -policy. Initialization always happens at startup or through management actions, -never during image construction. No additional single-node image is needed for -manual one-node bootstrap. The production policy remains -`CROWDB_DEPLOYMENT_MODE=production` with explicit credentials and resources. +templates and entrypoint for Local and Production deployment. Fresh Local startup +defaults to one initial voter, automatic file-backed test storage and all three +access protocols. Three-container Local startup explicitly selects three voters +on one dedicated discovery scope; missing peers remain waiting. Local supplies +clearly scoped test admission credentials and reports zero physical host failure +tolerance. Production requires configured credentials, explicit resources and +valid protected placement. It never falls back to Local credentials or geometry. + +The monitor seals the versioned deployment policy before identity, admission or +storage effects. Retained accepted values supply omitted inputs on restart; +conflicting overrides or populated obsolete roots fail without conversion or +reset. Both policies expose the same editable UI and authorized management +operations. Initialization occurs through the shared durable node lifecycle, +independently of host helpers or browser connections. Image construction creates +no node identity or initialized storage. BuildKit's Dockerfile frontend remains usable, including existing bind mounts in `RUN` instructions. The recipe name does not imply a Docker daemon dependency. @@ -88,6 +88,17 @@ Docker-specific `HEALTHCHECK` is not a portable OCI configuration field. Runtime health checks invoke `crowdb-monitor liveness` and `crowdb-monitor readiness` explicitly. Container restart policy and application readiness are distinct: unready Group 0 must not trigger destructive reinitialization or restart loops. +On retained startup, the monitor validates sealed member trust and starts the +owned KV process without waiting for quorum health. Management discovery and +authenticated member identity remain available while the replica recovers its +peers' current addresses. Independent HTTP health observations determine KV +readiness; a live unready process does not consume another restart attempt. +Explicit admission/start actions retain their bounded readiness wait. +Background protocol observations obtain the local node's current address and +service intents from Group 0 within one shared authority deadline. Cluster, +node and discovery identities must match a confirmed, non-cancelled record. +The retained UI topology is not the address authority for these local probes; +status readers only consume the resulting observations. ## 3. Pixi-managed construction @@ -153,16 +164,11 @@ separate construct, runtime verification and publish jobs: graph using digest-preserving transfer; failure to preserve it is an error. Verify the registry digest before signing or updating moving tags. -The canonical Docker Hub repository is `crowdb/crowdb-node`. The Iceberg, S3 and Dataset -repositories provide single-node usage names for the same image digest. Their -startup instructions explicitly select `CROWDB_STARTUP_MODE=single`; repository -names do not alter the default manual mode. The Dataset name reserves its usage -entry point while Dataset access remains unavailable. Each -repository receives version and `latest` tags; there is no runtime-specific or -service-specific image build. -Changing the published default from automatic single to manual is a usage -change: existing automatic examples/helpers must explicitly select `single`. -Do not silently change the startup policy of an existing populated data root. +The canonical Docker Hub repository is `crowdb/crowdb-node`. The Iceberg, S3 and +Dataset repositories are usage names for the same image digest and version tags. +Repository names change neither the accepted deployment policy nor UI permissions. +All three access protocols use the ordinary node lifecycle. Existing populated +state retains its accepted policy through image replacement. Keep signing, SBOM and provenance. Generate provenance from the source/tool/base receipt and attach attestations to the verified digest without rebuilding the @@ -177,10 +183,10 @@ digest, then start it with the existing entrypoint/environment/mount contract. Production uses `--network host`, explicit CPU policy and `--memory`, a persistent data directory, physical host identity, management interface and credential mounts. Bridge-mode single-host simulation remains available for tests. -Ordinary one-node usage needs only the usual ports/persistent data mount and UI -bootstrap. Automatic development usage adds `-e CROWDB_STARTUP_MODE=single`; -production resource/device arguments are not prerequisites for the simple -development example. Both use the same image. +Fresh Local one-node usage needs only the published client/UI ports and persistent +data mount. Three-node Local usage selects three voters without peer addresses or +fixed IPs. Both use automatic authenticated preparation and remain editable. +Production resource/device arguments are separate explicit host inputs. The standard monitor accepts private `PrepareDiskFile` requests only for its confirmed node/cluster binding with available Group-0 authority. It records the @@ -209,9 +215,9 @@ The persistent data mount contains node identity, bootstrap/service journals and managed disk files. Supplied files and whole block devices use the same container disks directory in Docker and containerd, with launcher-verified exact mappings. Replacement retains the original sources and identities; invalid inputs never -trigger managed-file fallback. Standard setup defaults to one 16-GiB preallocated -file and a 1-GiB host reserve; explicit single mode retains four sparse development -files. File preallocation failure is not permission for sparse fallback. +trigger managed-file fallback. Local defaults to one 16-GiB sparse managed file +and a 1-GiB host reserve. Production automatic storage uses preallocation and +rejects sparse files. File preallocation failure never authorizes sparse fallback. Supplied provisioning seals the canonical physical host/resource and physical capacity separately from whole-zone usable geometry. A full-file loop device @@ -267,6 +273,17 @@ for explicit stop, restart and cleanup. Host limits are upper bounds, not guaranteed CPU reservations. Block-device access stays explicit; no blanket privileged container is needed. +The common cluster CLI supports Docker and containerd host lifecycle through +these shared validators. A private receipt is sealed before runtime effects; +it records the stable daemon identity, Unix endpoint, context or namespace, +snapshotter, owner token, immutable image, resources, media lineage and input +file hashes. Resume accepts the same inputs and exact owned container, or +recreates that container against the retained data. A foreign same-name object, +changed daemon or changed configuration fails before start/stop effects. +Application UUID binding is separate and can be sealed only after observing +that cluster. Independent host cleanup verifies the same ownership and removes +only the container, preserving media, data and receipts. + Explicit monitor liveness/readiness probes report node health. A dedicated systemd unit restarts the containerd daemon, and its restart plugin applies the node container’s `unless-stopped` policy. Monitor supervises child services. diff --git a/doc/design/diskdb/design-crowdb-diskdb.md b/doc/design/diskdb/design-crowdb-diskdb.md index 666915f2f..8b8ee84d5 100644 --- a/doc/design/diskdb/design-crowdb-diskdb.md +++ b/doc/design/diskdb/design-crowdb-diskdb.md @@ -273,12 +273,9 @@ routing (§3.2); it is not stored in record keys or `Segment`. ### Protocol -Schema definitions are split across multiple files in -`lib/crowdb-protocol/src/fbs/` (`error_code`, `common_type`, -`diskdb_type`, `diskdb_op`, `diskdb_sys_op`, `diskdb_service`, -`diskdb_sys_service`, `chunkdb_type`, `chunkdb_op`, `chunkdb_service`, -`diskio_op`, `diskio_service`). See the schema files for field-level -detail. +`diskdb.fbs`, `chunkdb.fbs` and `diskio.fbs` define the storage services, +with shared identities and message types. Rust and C++ use generated definitions +from the same current schemas. The storage path uses three RPC services: - **`DiskdbService`** (served by the diskdb server): `AllocateBlocks` @@ -290,24 +287,20 @@ The storage path uses three RPC services: caller uses this operation only after frame parse or write-frame checksum proves corruption; the server checks owner, geometry, allocation timestamp, and committed state before a revision-fenced `Corrupt` update. Ordinary I/O - failures do not change the block state. The service ships - with allocate/free returning `Unimplemented`; the rest are later - requirements. + failures do not change the block state. Allocation and ownership-qualified + free update the durable allocator records. - **Hardware admin** (no rpc surface): rack/node/disk-group/disk add/remove and `set_*_status` are writes to group-0 sysdata performed through `HardwareClient` in `crowdb-kv-client`, invoked by - the console (`crowdb-web` / `crowdb-cli`). The previous - `DiskdbAdminService` schema (`diskdb_sys_service.fbs` / - `diskdb_sys_op.fbs`) is removed — `FetchHardware` is replaced by - `HardwareClient` prefix scans, `Keepalive` by - `ServiceRegistryClient.heartbeat`, and the add/remove/status ops by - `HardwareClient` methods. The diskdb server reads hardware state + the console (`crowdb-web` / `crowdb-cli`). Hardware reads use + `HardwareClient` prefix scans; liveness uses `ServiceRegistryClient.heartbeat`. + The diskdb server reads hardware state from group 0 via `HardwareClient` in its sync loop; it does not serve hardware admin. See `doc/design/kv/design-crowdb-kv-group0.md` §2.8. - **`ChunkdbService`**: `AllocateChunk`, `AppendChunk`, `QueryChunk`, `SealChunk`, `DeleteChunk`, - `DeleteChunkRange`, `UpdateChunkStrip`, `ListChunks`, exact-segment owner + `DeleteChunkRange`, `ReplaceChunkStripRange`, `ListChunks`, exact-segment owner queries, and relocation handoff. - **`DiskioService`**: `DiskWrite`, `DiskRead`, and `DiskFsync`. @@ -455,6 +448,27 @@ Transitions: - Offline ↔ Maintenance (operator). - Offline → Up (operator). +An unknown or cancelled durable metadata write closes its `DiskWriteGate`. +The local disk then reports Offline even if Group 0 still records Up; this +is unresolved persistence, not evidence of exhausted capacity. `/ready` returns +503 with the affected DiskGroup, disk and `metadata-write-outcome-unknown` reason. +Health reads only observe this state. + +The process recovery watcher detects that uncertainty, drains the service and +stops. The monitor supervises the existing Start/Restart intent and launches a +new process. Its process session advances the owner generation and claims the +data-group fence before reconstructing allocation state from durable records. +Old process writes remain fenced and old disk snapshots retain their closed +gates. Recovery never clears an uncertain gate or treats uncertain occupied +space as free. Administrative Offline without an unknown write does not trigger +this process recovery; its explicit Online authorization remains required. + +Usage reports derive allocatable disk count and bytes from current disk gates, +independently of the cached allocation membership. Allocation failures with no +currently usable disks return Unavailable, including group/disk state, unknown +write state and capacity counters. NoSpace describes insufficient eligible +space or placement on usable disks. + diskdb's first major component: - **Sync with group 0** (fixed 10 s interval, same on success and failure — no error back-off in v1): fetch latest metadata; update diff --git a/doc/design/diskio/design-crowdb-diskio.md b/doc/design/diskio/design-crowdb-diskio.md index 870ca4819..e4e3eed4d 100644 --- a/doc/design/diskio/design-crowdb-diskio.md +++ b/doc/design/diskio/design-crowdb-diskio.md @@ -285,26 +285,42 @@ resolves the request and calls `crowdb_rpc_server_submit_response`. disk's `IoEngine`. Every write traverses it. `NullDisk` and `MemDisk` use a block size of one, so the same path delegates their writes without padding. -For disks with larger blocks, the stage aligns the physical range and owns an -aligned buffer until the backend completion callback runs. An aligned start -with a partial tail copies the request bytes into a zero-initialized block and -writes the padded range immediately. The zero source is a process-lifetime, -pre-prepared buffer. The RPC completion reports the logical byte count only -after the padded backend write completes in full. +For disks with larger blocks, the stage aligns the physical range and retains +one aligned buffer through backend completion. It preserves every byte outside +the requested range. Partial head and tail blocks come from valid cache entries +or exact boundary-block reads; a range spanning both uncached boundaries needs +at most two reads. Request bytes are overlaid only after those reads complete. +The RPC reports the logical count only after the complete aligned write succeeds. The stage caches the last partial physical block by disk and block offset. -A sequential write beginning inside that block merges into the cached image. -When the entry is absent, `AlignedWriter` warns, reads the physical block, -merges the request, and writes it back. Failed and completed full-block writes -invalidate affected partial entries. The bounded cache may evict an entry; -the read-merge path preserves correctness after eviction or process restart. - -Writes are assigned to 64 shards by disk. Each shard has a bounded lock-free -MPSC queue and one active consumer. This preserves submission and completion -order for every overlapping range on one disk without a mutex. Different disks -remain independently executable apart from hash collisions, which only -serialize them. Queue saturation fails the request with `-EAGAIN`; it never -drops a write silently. +The bounded cache is an optimization: eviction, invalidation and process restart +do not change range-write semantics. Failed writes invalidate affected entries; +full-block writes invalidate prior partial images. An aligned start with a +partial tail also preserves the rest of that physical block, so a delayed +immutable retry cannot erase bytes written by a later request. + +Writes are assigned to 64 shards by disk and segment ordering base. Each shard +has a bounded lock-free MPSC queue and one active consumer through write +completion. Related partial-block writes remain ordered without a mutex. +Independent segment bases and disks can execute separately apart from hash +collisions. Queue saturation fails the request with `-EAGAIN`. + +Byte-addressed writes use the same ordering queue. A zero-length ordered request +waits for preceding write completions without issuing a media write; when its +sync flag is set, the RPC handler then flushes the disk before proving completion. +This is a segment drain after the caller has excluded future producers. A flush +alone, or a drain while new producers remain authorized, cannot establish a final +tail or authorize physical reuse. + +At admission, a write timestamp must fall within the bounded request age and +future clock-skew allowance. The server derives a conservative monotonic deadline +and checks it when dequeuing and before submitting an aligned merge after boundary +reads. An expired queued request returns `OldRequest` without a media write. An +already submitted backend operation retains its queue position through completion, +so a later drain waits for it even when its kernel execution outlasts request age. +The shared admission ceilings are 30 seconds of request age and one second of +clock skew. A server may use shorter windows; startup rejects larger values so +it cannot extend the late-write interval assumed by recovery and reclamation. Concurrent overlapping writes are not a supported caller contract. As with raw `pwrite`, their final contents would otherwise depend on completion order. @@ -656,14 +672,20 @@ handle three message types. Message type IDs are in the diskio range Each request's control message is a flatbuffer with `{disk_id, zone_index, zone_offset, size}` (read also has `test_pattern_offset`); the write request additionally carries an -`ordering_zone_offset` and a raw data payload of `size` bytes. The handler: +`ordering_zone_offset`, `write_create_time_ms`, a `sync` option, and a raw data +payload of exactly `size` bytes. The write handler validates the FlatBuffer, +zone bounds and payload length before IO. The handler: 1. Resolves `disk_id` to a `Disk` via `DiskSet`. 2. Computes the physical offset: `zone.base_offset + zone_offset`. -3. Calls `AlignedWriter::submit` for writes or `IoEngine::read`/`fsync` for +3. Calls `AlignedWriter::submit_ordered` for writes or `IoEngine::read`/`fsync` for the other operations, with the disk handle, physical offset, buffer, and a completion callback. -4. The completion callback builds the response and calls - `crowdb_rpc_server_submit_response`. +4. A synchronized write submits its disk flush only after a complete successful + write. Its sole response follows that flush; buffered writes reply after the + write. Failed or short writes never flush or report synchronized success. +5. Completion retains the disk, shared connection and request buffer through + both operations, then submits the response through the transport. Rejected + submission releases the caller-owned response frame. The data payload is passed from the crowdb-rpc frame decoder to `AlignedWriter`. Block-size-one writes remain zero-copy on the server; padded @@ -672,13 +694,13 @@ payload. Related writes are serialized by `ordering_zone_offset` within `AlignedWriter`. This ordering protects partial-block read-modify-write state; -it does not validate allocation ownership. The retired `allocation_ts` wire -field is ignored and retained only for schema compatibility. +it does not validate allocation ownership. Write requests carry only the +current addressing, ordering, request-age and synchronization contract. Flatbuffer schemas (`diskio.fbs`): - `DiskWriteRequest { disk_id, zone_index, zone_offset, size, - ordering_zone_offset }` -- `DiskWriteResponse { ret_code }` + ordering_zone_offset, write_create_time_ms, sync }` +- `DiskWriteResponse { ret_code, sync_completed }` - `DiskReadRequest { disk_id, zone_index, zone_offset, size, test_pattern_offset }` (`test_pattern_offset` used by NullDisk for deterministic content; default 0 = start of disk/object) @@ -722,27 +744,32 @@ The internal wire transport retains the caller's owned `Bytes` allocation in the RPC buffer until completion. This avoids a client-side `Bytes` to `Vec` payload copy. On the server, block-size-one and already aligned payloads can continue directly into `IoEngine::write`. Other writes are copied once into -the server-owned aligned buffer that supplies padding and satisfies the +the server-owned aligned buffer that preserves boundary bytes and satisfies the `O_DIRECT` address-alignment requirement. `write_views` splits oversized view lists into bounded RPC frames without copying their payload. Frames within one strip's disk block complete serially -in byte order, including aligned boundaries. Logical non-overlap does not -permit concurrent submission: connections can reorder arrivals, and the -aligned writer's partial-tail padding can overwrite a later frame. Separate +in byte order, including aligned boundaries. Connections can reorder arrivals; +serial completion preserves a contiguous prefix before the final flush. Separate caller operations on that block must obey the same ordering. Independently owned strip blocks and different disks holding EC shards or mirror replicas -may run concurrently. Requested fsync runs once after every frame succeeds; +may run concurrently. Only the final frame requests synchronization, after +preceding frames have completed. Its flush covers the whole preceding range; a frame error stops the remaining submissions. Input arithmetic, segment bounds, and wire-size limits are checked before RPC admission. Reads retry transient transport outcomes within the original deadline. Writes can retry only the same immutable bytes at the same allocated -location; partial writes and permanent disk errors return immediately, while -an unresolved admitted write is reported as ambiguous. Durable writes are not -acknowledged until a successful disk fsync; an fsync can be repeated after an -ambiguous response. The lifecycle no-reuse window is what makes an identical -write retry safe. +location and retain their original creation time; partial writes and permanent +disk errors return immediately. Expired operations submit no new request. +Fresh-storage initialization disables automatic retries, and an unresolved +admitted write is reported as ambiguous. Durable writes require +`sync_completed=true` after write and flush, using one request/reply. Missing +proof fails directly. A failed +flush returns `FsyncFailed`, exposed as `DurabilityFailure`; no failure is a +proof of synchronization. Independent fsync remains available and can be +repeated after an ambiguous response. The lifecycle no-reuse window is what +makes an identical write retry safe. Status exposes route generation and age, topology and pool counts, healthy connections, operation/retry/backpressure/ambiguity counters, and operation @@ -757,8 +784,10 @@ the crate's `test-util` feature. followed by a `read(segment, None)` on the same `Segment` returns the same bytes. Holds for `UringEngine`, `BlockingEngine`, and `DummyEngine` (with rule-based read content). -- **I2 (durability)**: A `write` followed by `fsync` followed by - process restart returns the written data on re-read. +- **I2 (durability)**: A successful synchronized write, or a write followed + by successful explicit fsync, retains the written data across process restart + when its backing media remains available. A synchronized response follows + both write and flush, with exactly one completion. - **I3 (offset correctness)**: The physical offset for an I/O is `zone.base_offset + zone_offset`, where `zone` is looked up by `zone_index` from the `Disk`'s zone records. @@ -779,9 +808,9 @@ the crate's `test-util` feature. decides the next action. - **I8 (partial-block preservation)**: Sequential writes to the same physical block complete in submission order. An unaligned continuation merges with - the cached acknowledged block image. A cache miss reads and merges the - physical block before writing. A partial tail from an aligned start is - zero-padded and cached for the next continuation. + the cached acknowledged block image. Each uncached partial boundary is read + before the request bytes are overlaid. Alignment preserves all bytes outside + the requested range, including tails after aligned starts and late retries. - **I9 (backend alignment)**: Every backend write to a disk with block size greater than one has aligned address, offset, and length. diff --git a/doc/design/kv/design-crowdb-kv-server.md b/doc/design/kv/design-crowdb-kv-server.md index cf0699df4..8c50b9504 100644 --- a/doc/design/kv/design-crowdb-kv-server.md +++ b/doc/design/kv/design-crowdb-kv-server.md @@ -216,8 +216,9 @@ same `(store_id, group_id)`. `POST /stores/{sid}/groups/{gid}/wipe-user-data` drops and recreates WAL + engine user data for a group, preserving group0 sysdata + -topology. Used by `cluster clean` to reset a cluster between -write-regression sub-tests without a full redeploy. The handler +topology. Explicit test-only measurement fixtures use this endpoint between +write-regression sub-tests while retaining their owned topology. Attached +product benchmarks do not reset groups or manage servers. The handler steps down if leader, deletes the WAL + engine dirs, recreates the group via `create_group_with_wal` (replays empty WAL → fresh state), and does `remove_group` + `add_group` (remove-before-add is critical: diff --git a/doc/design/protocol/design-crowdb-protocol-key.md b/doc/design/protocol/design-crowdb-protocol-key.md index 61b7b4553..2f746ab7c 100644 --- a/doc/design/protocol/design-crowdb-protocol-key.md +++ b/doc/design/protocol/design-crowdb-protocol-key.md @@ -261,6 +261,33 @@ change and is not allowed; add a new key kind instead (§6). Header for every binary key: `magic:1 | type_tag:2`. +- **StreamDirectoryPageKey** — `stream_name:16 bytes | page_id:32 bytes`. + Total 51 bytes. Tag `0x0016`. The page identity is its canonical record's + SHA-256 digest; ownership epoch and head generation do not enter this key. + Scan prefix `magic|0x0016|stream_name` selects one stream's independent pages. +- **StreamDirectoryRootKey** — `stream_name:16 bytes | writer_epoch:u64 BE | + generation:u64 BE`. Total 35 bytes. Tag `0x0017`. + Scan prefix `magic|0x0017|stream_name` selects retained published roots. Root + records commit atomically with the stable head and retention control. +- **StreamPublicationGuardKey** — `stream_name:16 bytes | publication_id:32 + bytes`. Total 51 bytes. Tag `0x0018`. Ordinary publication identity is SHA-256 + of the canonical indexed head record. The value protects the base root + and a bounded new page set until + publication is resolved. +- **StreamRootPinKey** — `stream_name:16 bytes | pin_id:16 bytes`. + Total 35 bytes. Tag `0x0019`. Scan prefix `magic|0x0019|stream_name` + selects bounded consumer and reader pins. Each value carries the complete + bounded directory head and its captured logical interval, independently of + the retained-root index. Pin mutations also change the retention control. +- **StreamSealedChunkSpanKey** — `stream_name:16 bytes | chunk_id:16 bytes BE`. + Total 35 bytes. Tag `0x001a`. Scan prefix `magic|0x001a|stream_name` + selects exact whole-chunk bounds. Each span commits atomically with the head + that ends its original active attachment; root generation does not enter the + span identity. +- **StreamDirectoryControlKey** — `stream_name:16 bytes`. Total 19 bytes. + Tag `0x001b`. The fixed scope record's KV revision changes atomically with + guard/root/pin visibility; conditional cleanup uses it to reject stale + reachability observations. - **RackKey** — `rack_id:u64 BE`. Total 11 bytes. Tag `0x0002`. Scan prefix `magic|0x0002` = all racks. - **NodeKey** — `rack_id:u64 BE | node_id:u64 BE`. Total 19 bytes. diff --git a/doc/design/rpc/design-crowdb-rpc-tcp.md b/doc/design/rpc/design-crowdb-rpc-tcp.md index 907585320..71475a1e1 100644 --- a/doc/design/rpc/design-crowdb-rpc-tcp.md +++ b/doc/design/rpc/design-crowdb-rpc-tcp.md @@ -36,8 +36,8 @@ Per worker thread: loop: events = engine.wait(timeout) for event in events: - if event.fd == notify_fd: // shutdown wake only - no-op (no cross-thread submit queue) + if event.fd == notify_fd: + drain notified cross-thread sends elif event.fd == timer_fd: run due scheduled tasks → reset timer to next deadline elif event is READABLE: @@ -50,12 +50,24 @@ Per worker thread: elif event is ERROR/HUP: conn->close() → remove_connection(fd) → trigger reconnect + drain owning worker's explicit-close queue → remove descriptors // send aggregation: batch writev pending responses for conn in pending_write_conns: on_writable(conn) if partial/EAGAIN: engine.arm_write(fd) ``` +Explicit close first fences admission and completes pending RPC failures. It +also publishes an identity-retaining close request to the owning worker through +an atomic queue and wakes that worker's engine. The worker removes both read +and duplicate write descriptors even when the peer is idle and produces no +further I/O. A single atomic state fences new socket-use grants at close and +counts in-flight reads, writes and event re-arms. Descriptor cleanup waits until +those uses retire, so an old syscall cannot touch a newly reused descriptor. +Queued close requests survive this deferral. The worker releases its +connection-map entry only after flushing pending responses. Descriptor cleanup remains single-owner; stale kernel events resolve +through non-reused connection IDs. Shutdown drains retained close requests. + The transport creates N independent engines (`io_engines`), with `io_workers` total workers (per-engine M = `io_workers / io_engines`). Connections are partitioned round-robin across all workers; each diff --git a/doc/design/tree/design-crowdb-tree-chunk-storage.md b/doc/design/tree/design-crowdb-tree-chunk-storage.md index b20312ddd..a320b9d92 100644 --- a/doc/design/tree/design-crowdb-tree-chunk-storage.md +++ b/doc/design/tree/design-crowdb-tree-chunk-storage.md @@ -54,7 +54,8 @@ exposes construction, transport routes, statistics, and orphan accounting. The tree writes a logical byte image. The chunk backend groups its dirty ranges into immutable packs of at most `pack_bytes`. Every pack has a logical offset and a `ChunkPageRef` containing chunk identity, physical offset, logical length, -and checksum. +and frame revision. Frame footers own payload integrity and expected chunk +identity; high-cardinality reference entries do not duplicate their checksums. Mapping-table slots do not store physical chunk offsets. Their tagged 64-bit page-reference words name a logical location whose covering pack is resolved @@ -62,13 +63,22 @@ through immutable segmented reference tables. Untagged words remain valid for local stores and legacy mapping images. Mixed mapping images use their versioned codec so the two forms cannot be confused. -Manifest format 6 requires concrete BtreePage/PageIndex chunk identities. -Unsupported legacy chunk manifests are rejected rather than silently retagged. -The manifest retains the WAL replay offset; format 4 introduced the WAL replay offset associated with the tree snapshot; -format 3 permits sorted, non-overlapping sparse pack layouts. A gap -means that no live page references that logical pack range. Recovery and page -reference decoding validate arithmetic bounds, reference-segment checksums, and -complete pack coverage for every addressed page. +Manifest formats 6 and 7 require concrete BtreePage/PageIndex chunk identities, +retain the WAL replay offset and permit sorted, non-overlapping sparse pack +layouts. A gap means that no live page references that logical pack range. +Recovery validates arithmetic bounds, reference-segment checksums and complete +pack coverage for every addressed page. Unsupported formats fail visibly. + +New manifests use format 7 and packs use ordinary revision-2 common frames. +Each frame has a 15-byte header and 20-byte footer, with at most 65,501 payload +bytes; Stream grouping roles do not apply. Revision-1 packs remain readable. +The serialized 28-byte reference uses its u32 length's high bit as the revision-2 +marker. Valid logical pack lengths are bounded by 256 MiB, so retained +revision-1 lengths have that bit clear. In-memory logical lengths exclude the +marker. Manifest and reference-segment CRCs cover the encoded marker, and +format-6 manifests accept only legacy references. Exact reads derive physical +length from the reference's revision without a separate header probe. Reused +packs and immutable reference segments retain their original revisions. The backend allocates three-way mirror strips. One B+tree chunk carries at most 256 MiB of logical pack data. A pack never crosses that limit: the current chunk @@ -118,7 +128,8 @@ nonblocking pipe. Rust polls the descriptor through its existing async reactor. `ChunkManifest` records tree identity, generation, owner epoch, logical size, the snapshot's WAL replay offset, pack entries, reference-segment directory, sharing counters, and checksums. Manifest directories and reference-segment -images are immutable. Only the current typed manifest format is accepted by this backend. +images are immutable. The backend accepts supported typed manifests and +rejects unknown formats before using their references. A checkpoint captures the current catalog generation before it constructs any packs. Its new manifest is numbered `expected_generation + 1`. After pack and diff --git a/doc/doc_index.md b/doc/doc_index.md index 3e8680a81..37753703a 100644 --- a/doc/doc_index.md +++ b/doc/doc_index.md @@ -110,7 +110,7 @@ Temporary plans live under `doc/working/`; flow analyses live under | `doc/design/chunkdb/chunkdb-allocate-flow-analysis.md` | EC allocation benchmark and bottlenecks. | | `doc/design/chunkio/design-crowdb-chunkio-small-object-writer.md` | Shared-chunk admission, routing, recovery, elasticity. | | `doc/design/chunkio/design-crowdb-chunkio-reader.md` | Mirror/EC reads, recovery, fencing, repair. | -| `doc/design/chunkds/design-crowdb-chunk-stream.md` | Logical streams, append, rollover, trim, transfer. | +| `doc/design/chunkds/design-crowdb-chunk-stream.md` | Frame groups, mirrored append, ordered chunk records. | | `doc/design/diskdb/design-crowdb-diskdb-zone-management.md` | Zone allocation, free, compaction, recovery. | | `doc/design/diskdb/design-crowdb-diskdb-space-metrics.md` | Capacity counters, reporting, aggregation. | | `doc/design/diskdb/diskdb-allocate-flow-analysis.md` | Durable allocation flow and benchmarks. | diff --git a/e2e-test/README.md b/e2e-test/README.md index eca4625a6..3768220a8 100644 --- a/e2e-test/README.md +++ b/e2e-test/README.md @@ -45,10 +45,10 @@ pixi run test unit core pixi run test e2e s3 --case s3.sdk::java pixi run test profile containerd-cluster # Explicit Linux host-network admission/recovery; run on an exclusive host: -pixi run test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 -# Explicit native containerd in isolated privileged Linux hosts: -pixi run test-container-e2e --profile containerd-host --concurrency 1 --cpus 3 --memory-mib 2048 -pixi run test-container-e2e --profile containerd-cluster --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 +pixi run sudo "$(command -v pixi)" run --as-is test-container-e2e --profile host --concurrency 1 --cpus 4 --memory-mib 4096 +# Native containerd on an exclusive Docker-free Linux host: +pixi run sudo "$(command -v pixi)" run --as-is test-container-e2e --profile containerd-host --concurrency 1 --cpus 3 --memory-mib 3072 +pixi run test-container-e2e --profile containerd-cluster --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 8192 # Manual discovery/SSH/bootstrap and four host browser acceptance flows: pixi run bash -c 'cd e2e-test/clients/console-ui && npm ci --ignore-scripts' pixi run test-container-e2e --profile bridge-admission --concurrency 1 --node-cpus 1 --cpus 4 --memory-mib 6144 @@ -67,14 +67,31 @@ chunk-stream, Chunk-KV, S3, Iceberg, Dataset, Console and deployment cases. The host profile retains its owned bind-mounted data/log/crash root and an exact image/source record; fixed ports require an exclusive Linux host. It is an explicit deployment acceptance and is excluded from ordinary bridge domains. +The Docker host and native containerd host profiles qualify two explicit policies. +A one-voter production diskless node exercises the common CLI's accepted inputs, +authenticated admission, current metadata authority and retained recreation. This +metadata-only configuration does not claim whole-cluster storage readiness. +A Local one-voter node exercises all three access protocols and retained data, +identity and credentials through ordinary container recreation, with zero node +failure tolerance. Each node has a 1-CPU quota and 2048 MiB memory limit. +Protected production storage requires independent voters; the separate three-host +acceptance verifies that contract. The Docker host profile also retains file/raw +fsync, Offline, Remove/Rejoin, exclusive-descriptor and physical-tail assertions. +These profiles require an explicitly privileged disposable Linux host for owned +fixture setup; the node container receives no blanket privilege. + Containerd profiles require the installed `container/oci-image` runtime Pixi -environment and a verified OCI archive/receipt. They create one or three -privileged Linux host containers, with no Docker socket inside them. Each host -runs native containerd and a formal node; clients execute outside the formal -node image. Private data/log/crash mounts survive teardown. The host profile -uses 1024 CPU shares per host/node container and retains its 1024 MiB node -memory limit. Cluster admission uses a shared CPU allowance and sums hard -memory limits, including its host containers and client. Artifact and image identities are verified before startup. +environment and a verified OCI archive/receipt. `containerd-host` starts its own +rootful native-snapshotter daemon and rejects a Docker client, socket, daemon or +container-provisioned host. `containerd-cluster` separately labels its three +Docker-provisioned privileged Linux hosts; it proves adapter/discovery/quorum +behavior and does not substitute for the native-host gate. Both retain private +data and complete logs. Clients execute outside the formal node image. +The Local host fixture selects an 8-GiB sparse managed file with a 1-GiB +host reserve. Sparse files are confined to the explicit Local policy. `CROWDB_E2E_CLI_BUNDLE` supplies a prebuilt host CLI and +`lib/` directory when the host cannot build clients; Docker media probes can +likewise use `CROWDB_E2E_DOMAIN_BUNDLE` with `domain-test` and `lib/`. + DiskIO process-exit faults retain their network namespace, so the original 15-second stream/reader/repair bounds still apply. diff --git a/e2e-test/access_dataset_fixture.py b/e2e-test/access_dataset_fixture.py index e0f36841a..ad73996ca 100644 --- a/e2e-test/access_dataset_fixture.py +++ b/e2e-test/access_dataset_fixture.py @@ -49,7 +49,7 @@ def control(self, body): if body["node"] == 12: self.ready_dataset() elif body["node"] == self.chunk_kv_index: - self.ready_chunk_kv() + self.ready_chunk_kv(recovering=True) return result def sdk_roundtrip(self): diff --git a/e2e-test/access_iceberg_fixture.py b/e2e-test/access_iceberg_fixture.py index 9304ecd30..8fc0e46ae 100644 --- a/e2e-test/access_iceberg_fixture.py +++ b/e2e-test/access_iceberg_fixture.py @@ -80,5 +80,5 @@ def control(self, body): if body["node"] in (12, 13): self.ready_iceberg(body["node"]) elif body["node"] == self.chunk_kv_index: - self.ready_chunk_kv() + self.ready_chunk_kv(recovering=True) return result diff --git a/e2e-test/admission_profile.py b/e2e-test/admission_profile.py index a6df0b88f..22401ba03 100644 --- a/e2e-test/admission_profile.py +++ b/e2e-test/admission_profile.py @@ -12,6 +12,7 @@ import uuid from fixture import docker, redact from image_identity import verify_server_sources +from native_containerd_profile import client_bundle ROOT = Path(__file__).resolve().parents[1] @@ -74,15 +75,17 @@ def run_profile(args, parser): data = (args.data_root / run_root.name / "admission" if args.data_root else ROOT / ".crowdb-runtime/test-data/container-e2e" / run_root.name / "admission").resolve() data.mkdir(parents=True, mode=0o700) - prefix = "crowdb-e2e-admission-" + run_root.name.removeprefix("run-") + prefix = "crowdb-admit-" + run_root.name.removeprefix("run-") (run_root / "identity.json").write_text(json.dumps({"profile":"bridge-admission", "id":identity["Id"], "labels":labels, "sources":source, "cpus":cpus, "memory_mib":memory, "node_cpus":args.node_cpus, "host_data":str(data), "container_data":"/opt/crowdb/data", "ports":{"web":9090,"ssh":2222}, "core_policy":Path("/proc/sys/kernel/core_pattern").read_text().strip()}, indent=2)) + bundle = client_bundle(run_root / "host-client") env = os.environ.copy() env.update(CROWDB_CONTAINER_IMAGE=identity["Id"], CROWDB_E2E_PROFILE_DATA=str(data), - CROWDB_E2E_PROFILE_NAME=prefix, CROWDB_E2E_NODE_CPUS=str(args.node_cpus)) + CROWDB_E2E_PROFILE_NAME=prefix, CROWDB_E2E_NODE_CPUS=str(args.node_cpus), + CROWDB_E2E_CLI_BUNDLE=str(bundle)) print(f"Bridge admission: exact image {identity['Id']}; artifacts {run_root}", flush=True) started, status, code = time.monotonic(), "failed", None child = subprocess.Popen([sys.executable, str(ROOT / "e2e-test/profiles/node-containers.py")], diff --git a/e2e-test/browser_batch.py b/e2e-test/browser_batch.py index a1a5220b3..afa738623 100644 --- a/e2e-test/browser_batch.py +++ b/e2e-test/browser_batch.py @@ -22,7 +22,7 @@ def __exit__(self, kind, error, traceback): return self.cluster.__exit__(kind, error, traceback) -def run_browser_batches(cases, slots, prepare, execute): +def run_browser_batches(cases, slots, prepare, execute, fail_fast=False): errors = [] for offset in range(0, len(cases), slots): batch = cases[offset:offset + slots] @@ -43,4 +43,6 @@ def run_browser_batches(cases, slots, prepare, execute): errors.append(error) except Exception as error: errors.append(error) + if fail_fast and errors: + break return errors diff --git a/e2e-test/chunk_kv_fixture.py b/e2e-test/chunk_kv_fixture.py index 548b3274d..1dd729e3f 100644 --- a/e2e-test/chunk_kv_fixture.py +++ b/e2e-test/chunk_kv_fixture.py @@ -66,7 +66,7 @@ def start(self): "ip": address, "rpc_port": 7402, "http_port": 7401, "node": index}, indent=2)) return self - def ready_chunk_kv(self, index=None): + def ready_chunk_kv(self, index=None, *, recovering=False): index = self.chunk_kv_index if index is None else index address = self.addresses[index] def probe(): @@ -80,12 +80,17 @@ def probe(): if health["ready"] or health["hosted_partitions"] != 0 or health["catalog_generation"] != 0: raise AssertionError(f"unpublished Chunk-KV catalog unexpectedly serves data: {health}") return response.status == 200 - wait(probe, bool, "production Chunk-KV catalog and partition readiness") + # Catalog recovery has a 60-second service deadline. An ungraceful + # owner replacement must exclude its lease and late IO before binding + # the serving endpoint; the ordinary 30-second startup probe cannot + # cover that required safety boundary. + wait(probe, bool, "production Chunk-KV catalog and partition readiness", + seconds=60 if recovering else 30) def crash_restart(self): started = time.monotonic() self.command("kill", "--signal", "KILL", self.servers[self.chunk_kv_index]) super().crash_restart() self.command("start", self.servers[self.chunk_kv_index]) - self.ready_chunk_kv() + self.ready_chunk_kv(recovering=True) self.timings["restart"] = time.monotonic() - started diff --git a/e2e-test/client.py b/e2e-test/client.py index ddc3902b9..adea1aeef 100644 --- a/e2e-test/client.py +++ b/e2e-test/client.py @@ -157,6 +157,23 @@ def bundle_cli(output): shutil.copy2(binaries[0], Path(output) / "crowdb-cli") +def bundle_cli_workload(output): + """Compile the opt-in regression for explicit execution against an owned node.""" + result = subprocess.run(["cargo", "test", "--locked", "-p", "crowdb-cli", "--test", + "cluster_workloads_test", "--no-run", "--message-format=json"], + capture_output=True, text=True, check=False, timeout=600) + print(result.stderr, end="", flush=True) + if result.returncode: + print(result.stdout, end="", flush=True) + raise RuntimeError("CLI workload test compilation failed") + binaries = [item["executable"] for line in result.stdout.splitlines() if line.startswith("{") + and (item := json.loads(line)).get("target", {}).get("name") == "cluster_workloads_test" + and item.get("executable")] + if len(binaries) != 1: + raise RuntimeError("missing or ambiguous CLI workload test executable") + shutil.copy2(binaries[0], Path(output) / "cli-workload-test") + + def bundle_iceberg_rck(output): """Verify immutable upstream source and isolate Gradle outputs per run.""" revision = "6976e020b894f6a6777704df2b8c4458cb291ae9" diff --git a/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts b/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts index 23d71fea2..c93ec1036 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/candidateSetup.ts @@ -4,7 +4,7 @@ import type { Locator, APIRequestContext } from '@playwright/test'; import { expect } from './realBackend'; -/** Bootstrap-only fixtures explicitly opt out of automatic storage/services. */ +/** Bootstrap fixtures review their target's fixed diskless startup settings. */ export async function reviewDisklessCandidate(dialog: Locator, request?: APIRequestContext, origin = '', discoveryId?: string) { await dialog.getByRole('button', { name: 'Verify and review', exact: true }).click(); await expect(dialog.getByText(/Access verified/)).toBeVisible(); @@ -13,13 +13,13 @@ export async function reviewDisklessCandidate(dialog: Locator, request?: APIRequ expect(response.ok(), await response.text()).toBe(true); expect((await response.json()).some((node: { discovery_id: string }) => node.discovery_id === discoveryId)).toBe(false); } - await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toHaveValue('16'); - await expect(dialog.getByLabel('Sparse file', { exact: true })).not.toBeChecked(); - await dialog.getByLabel('Storage setup', { exact: true }).selectOption('diskless'); + await expect(dialog.getByLabel('Storage setup', { exact: true })).toHaveValue('diskless'); + await expect(dialog.getByLabel('Storage setup', { exact: true })).toBeDisabled(); + await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toHaveCount(0); for (const name of ['crowdb-paxos-kv', 'crowdb-disk-db', 'crowdb-disk-io', 'crowdb-chunk-db', 'crowdb-chunk-kv', 'crowdb-access-server']) { const service = dialog.getByRole('checkbox', { name, exact: true }); - await expect(service).toBeChecked(); - await service.uncheck(); + await expect(service).not.toBeChecked(); + await expect(service).toBeDisabled(); } await dialog.getByRole('button', { name: 'Join and start', exact: true }).click(); } diff --git a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts index b23ea2fe6..47189ce07 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/defaultDeployments.ts @@ -7,7 +7,7 @@ import { dirname } from 'node:path'; import { expect, test } from './realBackend'; import { deploymentState, observeDeploymentDiagnostics } from './deploymentDiagnostics'; -/** Observe actual automatic deployment mutations before checking durable progress. */ +/** Observe accepted mutations and target-owned asynchronous deployment completion. */ export function observeDefaultDeployments(page: Page) { const replies = new Map(); const attempts = observeDeploymentDiagnostics(page); @@ -23,10 +23,29 @@ export function observeDefaultDeployments(page: Page) { const identity = key(response); if (identity) replies.set(identity, [...(replies.get(identity) ?? []), response]); }); - async function wait(nodes: number[], kind: string, predicate: (response: Response) => boolean) { + const expectedWait = (response: Response, kind: string) => response.status() === 409 && ['chunkdb', 'chunk-kv'].includes(kind); + async function completed(nodes: number[], kind: string) { const timeout = process.platform === 'darwin' ? 30_000 : 10_000; + let found: { node: number; response: Response } | undefined; try { - return await page.waitForResponse(predicate, { timeout }); + await expect.poll(async () => { + for (const node of nodes) { + const unexpected = replies.get(`${node}:${kind}`)?.find(response => response.status() !== 201 && !expectedWait(response, kind)); + if (unexpected) { found = { node, response: unexpected }; return true; } + const response = replies.get(`${node}:${kind}`)?.find(response => response.status() === 201); + if (response) { found = { node, response }; return true; } + } + const waiting = nodes.filter(node => replies.get(`${node}:${kind}`)?.some(response => expectedWait(response, kind))); + if (!waiting.length) return false; + const response = await page.request.get('/api/service-plans'); + expect(response.ok(), await response.text()).toBe(true); + const plans = await response.json(); + const node = waiting.find(node => plans[node]?.steps[kind]?.state === 'deployed'); + if (node === undefined) return false; + found = { node, response: replies.get(`${node}:${kind}`)!.find(response => expectedWait(response, kind))! }; + return true; + }, { timeout, intervals: [100] }).toBe(true); + return found!; } catch (error) { const evidence = { nodes, service: kind, failed_at: new Date().toISOString(), timeout_ms: timeout, attempts: attempts(), observations: await deploymentState(page) }; @@ -40,36 +59,27 @@ export function observeDefaultDeployments(page: Page) { } catch (diagnosticError) { console.log(`[DEPLOYMENT DIAGNOSTIC WRITE FAILED] ${String(diagnosticError)}`); } - if (error instanceof Error) error.message = `Deployment response missing: nodes=${nodes.join(',')} service=${kind}\n${error.message}`; + if (error instanceof Error) error.message = `Deployment completion missing: nodes=${nodes.join(',')} service=${kind}\n${error.message}`; throw error; } } + async function verifyResponse(response: Response, kind: string) { + if (expectedWait(response, kind)) { + expect((await response.json()).error).toMatch(/^Waiting: /); + } else { + expect(response.status(), await response.text()).toBe(201); + } + } return { async nextCompleted(nodes: number[], kind: string) { - const pending = new Set(nodes.map(node => `${node}:${kind}`)); - const expectedWait = (response: Response) => response.status() === 409 && ['chunkdb', 'chunk-kv'].includes(kind); - const rejected = nodes.flatMap(node => replies.get(`${node}:${kind}`) ?? []) - .find(response => response.status() !== 201 && !expectedWait(response)); - if (rejected) expect(rejected.status(), await rejected.text()).toBe(201); - const completed = nodes.find(node => replies.get(`${node}:${kind}`)?.some(response => response.status() === 201)); - if (completed !== undefined) return completed; - const response = await wait(nodes, kind, response => pending.has(key(response) ?? '') - && (response.status() === 201 || !expectedWait(response))); - expect(response.status(), await response.text()).toBe(201); - return Number(key(response)!.split(':')[0]); + const result = await completed(nodes, kind); + await verifyResponse(result.response, kind); + return result.node; }, async verify(node: number, kind: string) { - const identity = `${node}:${kind}`; - const response = replies.get(identity)?.find(response => response.status() === 201) - ?? await wait([node], kind, response => key(response) === identity && response.status() === 201); - for (const rejected of replies.get(identity) ?? []) { - if (rejected === response) break; - const body = await rejected.json(); - expect(rejected.status(), JSON.stringify(body)).toBe(409); - expect(['chunkdb', 'chunk-kv']).toContain(kind); - expect(body.error).toMatch(/^Waiting: /); - } - expect(response.status(), `Node ${node} ${kind}: ${await response.text()}`).toBe(201); + const result = await completed([node], kind); + for (const response of replies.get(`${node}:${kind}`) ?? []) await verifyResponse(response, kind); + await verifyResponse(result.response, kind); }, }; } diff --git a/e2e-test/clients/console-ui/e2e/fixtures/disklessExpansion.ts b/e2e-test/clients/console-ui/e2e/fixtures/disklessExpansion.ts index 4e1db5d24..64401ddee 100644 --- a/e2e-test/clients/console-ui/e2e/fixtures/disklessExpansion.ts +++ b/e2e-test/clients/console-ui/e2e/fixtures/disklessExpansion.ts @@ -15,6 +15,12 @@ const disklessNodes = [4, 5, 6]; export async function disklessExpansion(context: UIContext) { const { page, request, baseURL, deployments } = context; const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); + type Process = { id: string; node_id: number; pid: number }; + const observed = await request.get('/api/servers'); + expect(observed.ok(), await observed.text()).toBe(true); + const original: Process[] = (await observed.json()).map(({ id, node_id, pid }: Process) => ({ id, node_id, pid })); + expect(original).toHaveLength(18); + expect(original.every(server => server.pid > 0)).toBe(true); await step('diskless: admit three release candidates through UI', async () => { await page.getByTestId('domain-cluster').click(); const candidates: { node_id: number; discovery_id: string }[] = JSON.parse(process.env.CROWDB_NATIVE_PENDING_CANDIDATES!); @@ -57,6 +63,11 @@ export async function disklessExpansion(context: UIContext) { await expect(sidebar.getByTestId(`tree-node-SERVICE-${server.id}`).getByTitle('Healthy', { exact: true })).toBeVisible(); } expect(servers.filter((server: { pid: number }) => server.pid)).toHaveLength(36); + const processes: Process[] = servers.map(({ id, node_id, pid }: Process) => ({ id, node_id, pid })); + for (const process of original) { + expect(processes.find(server => server.id === process.id && server.node_id === process.node_id), + `Adding nodes preserves the healthy process ${process.id}`).toEqual(process); + } // New local KV processes need not host Group 0. DiskDB still registers // through the cluster seeds while owning no disk groups. await expect.poll(async () => { diff --git a/e2e-test/clients/console-ui/e2e/flows/10-cluster-rack-node.spec.ts b/e2e-test/clients/console-ui/e2e/flows/10-cluster-rack-node.spec.ts index 5206c2a9d..8b960f0ed 100644 --- a/e2e-test/clients/console-ui/e2e/flows/10-cluster-rack-node.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/10-cluster-rack-node.spec.ts @@ -29,6 +29,7 @@ test.describe('cluster · registry browser', () => { expect(jsErrors, jsErrors.join('\n')).toEqual([]); }); + // Baseline: reference three-node OCI 7.2s (2026-10-22). test('datacenter root wraps racks; Add Rack from datacenter context menu', async ({ page, baseURL }) => { await step('dc-root: createRack 60', () => createRack(baseURL!, { id: 60, name: 'Rack Sixty' })); @@ -45,21 +46,31 @@ test.describe('cluster · registry browser', () => { // The rack appears as the datacenter's child. await expect(aside.getByText('R-60 (Rack Sixty)')).toBeVisible({ timeout: 3_000 }); - await page.getByRole('button', { name: 'Show diagram', exact: true }).click(); - // The topology canvas also renders the datacenter node at layer 0. - await expect(page.locator('.react-flow__node').filter({ hasText: 'datacenter' })).toBeVisible({ timeout: 5_000 }); + await step('dc-root: diagram refresh', async () => { + await page.getByRole('button', { name: 'Show diagram', exact: true }).click(); + // The topology canvas also renders the datacenter node at layer 0. + await expect(page.locator('.react-flow__node').filter({ hasText: 'datacenter' })).toBeVisible({ timeout: 3_000 }); + }); // Right-click the datacenter → Add Rack (the only menu item). - await aside.getByText('datacenter', { exact: true }).click({ button: 'right' }); - await page.getByRole('menuitem', { name: /add rack/i }).click(); + await step('dc-root: open rack editor', async () => { + await aside.getByText('datacenter', { exact: true }).click({ button: 'right' }); + await page.getByRole('menuitem', { name: /add rack/i }).click(); + }); await expect(page.getByRole('dialog', { name: 'Add Rack' })).toBeVisible(); await page.getByLabel('Rack ID').fill('61'); await page.getByLabel('Name (optional)').fill('Rack Sixty-One'); - await page.getByRole('button', { name: /create rack/i }).click(); - - // The new rack appears as another child of the datacenter. - await expect(aside.getByText('R-61 (Rack Sixty-One)')).toBeVisible({ timeout: 3_000 }); + await step('dc-root: mutation response', async () => { + const created = page.waitForResponse(response => response.request().method() === 'POST' + && new URL(response.url()).pathname === '/api/racks', { timeout: 3000 }); + await page.getByRole('button', { name: /create rack/i }).click(); + expect((await created).status()).toBe(201); + }); + await step('dc-root: DOM refresh', async () => { + // The new rack appears as another child of the datacenter. + await expect(aside.getByText('R-61 (Rack Sixty-One)')).toBeVisible({ timeout: 3_000 }); + }); }); const api = await apiContext(baseURL!); diff --git a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts index 453bf98ba..746e51d58 100644 --- a/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/11-cluster-server-lifecycle.spec.ts @@ -94,7 +94,7 @@ test('native diagnostics: auxiliary dialogs reopen with unused IDs ports and val await expect(dialog.getByLabel('Metadata store', { exact: true })).toHaveValue('1'); await expect(dialog.getByLabel('Journal metadata group', { exact: true })).toHaveValue('1'); } - await expect(dialog.getByLabel('Single-node test deployment (reduced redundancy)')).not.toBeChecked(); + await expect(dialog.getByLabel('Local deployment (reduced redundancy)')).not.toBeChecked(); await dialog.getByRole('button', { name: 'Cancel', exact: true }).click(); await expect(dialog).toHaveCount(0); } @@ -179,6 +179,12 @@ test('native diagnostics: waiting plan resumes when DiskIO prerequisites arrive' page.on('request', current => { if (current.method() === 'POST' && /\/api\/nodes\/\d+\/(server|diskdb|services)\/deploy$/.test(new URL(current.url()).pathname)) deploys.push(current.postDataJSON().kind); }); + const acceptedDeploys: string[] = []; + page.on('response', response => { + const current = response.request(); + if (response.status() === 201 && current.method() === 'POST' + && /\/api\/nodes\/\d+\/(server|diskdb|services)\/deploy$/.test(new URL(current.url()).pathname)) acceptedDeploys.push(current.postDataJSON().kind); + }); const dialog = page.getByRole('dialog', { name: 'Node 1 services', exact: true }); const open = async () => { const aside = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); @@ -224,7 +230,8 @@ test('native diagnostics: waiting plan resumes when DiskIO prerequisites arrive' return Object.values((await response.json())['1'].steps).map((entry: any) => entry.state); }, { timeout: 3000, intervals: [100] }).toEqual(kinds.map(() => 'deployed')); await expect(dialog.getByRole('listitem').filter({ hasText: 'deployed' })).toHaveCount(6); - expect(deploys).toEqual(['chunk-kv', 'access-server']); + // A prerequisite may still return Waiting; only accepted requests create services. + expect(acceptedDeploys).toEqual(['chunk-kv', 'access-server']); const current = await list(); expect(current).toHaveLength(14); for (const kind of kinds) expect(current.filter((row: { node_id: number; service_type: string }) => row.node_id === 1 && row.service_type === kind)).toHaveLength(1); diff --git a/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts b/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts index 68dd38508..51bc30184 100644 --- a/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/12-bridge-discovery.spec.ts @@ -23,6 +23,9 @@ test('Docker default setup: submit storage and services then close the initiatin await expect(dialog.getByLabel('Initial SSH password', { exact: true })).toHaveValue(''); for (const checkbox of await dialog.getByRole('region', { name: 'Node service choices' }).getByRole('checkbox').all()) await expect(checkbox).toBeChecked(); await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toHaveValue('16'); + await expect(dialog.getByLabel('Managed file capacity (GiB)', { exact: true })).toBeDisabled(); + await expect(dialog.getByLabel('crowdb-disk-db RPC port', { exact: true })).toHaveValue('7100'); + await expect(dialog.getByLabel('crowdb-disk-db RPC port', { exact: true })).toBeDisabled(); await dialog.getByRole('button', { name: 'Join and start', exact: true }).click(); await expect(dialog).toHaveCount(0, { timeout: 10_000 }); } @@ -165,5 +168,7 @@ test('Docker node update: authenticated rack and endpoint changes preserve alloc expect(after.node_id).toBe(before.node_id); expect(after.physical_host_id).toBe(before.physical_host_id); expect(after.rack_id).toBe(2); - expect(after.host).not.toBe(before.host); + // Independent monitor recovery may publish the new address before the dialog. + expect(after.host).toBe(process.env.CROWDB_NODE_UI_NEW_HOST); + expect(after.host).not.toBe(process.env.CROWDB_NODE_UI_OLD_HOST); }); diff --git a/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts b/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts index 156448cc4..86b9fc18b 100644 --- a/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/13-node-service-plans.spec.ts @@ -17,14 +17,20 @@ async function occupyDiskdbPort(request: APIRequestContext) { expect(group.status(), await group.text()).toBe(201); const added = await request.post('/api/nodes/1/disk-groups', { data: { id: 1, store_id: 0, group_id: 1 } }); expect(added.status(), await added.text()).toBe(201); - // Port ownership needs the actual DiskIO listener, without admitting media. + // The real DiskIO listener requires a live local DiskDB, even without media. + const dependency = await request.post('/api/nodes/1/diskdb/deploy', { data: { rpc_port: 29920 } }); + expect(dependency.status(), await dependency.text()).toBe(201); + const diskdb = await dependency.json(); const defaults = await json(request, '/api/deployment-defaults?node_id=1'); const deployed = await request.post('/api/nodes/1/services/deploy', { data: { - ...defaults.diskio, kind: 'diskio', instance_id: '99', rpc_port: 7102, disk_group_id: 1, test_single_node: false, + ...defaults.diskio, kind: 'diskio', instance_id: '99', rpc_port: 7102, disk_group_id: 1, } }); expect(deployed.status(), await deployed.text()).toBe(201); const guard = await deployed.json(); expect(await nativeProcessAlive(0, guard.pid)).toBe(true); + const removed = await request.delete('/api/services/diskdb-1'); + expect(removed.ok(), await removed.text()).toBe(true); + await expect.poll(() => nativeProcessAlive(0, diskdb.pid), { intervals: [100] }).toBe(false); return guard; } diff --git a/e2e-test/clients/console-ui/e2e/flows/50-chunk-capacity-disk-group.spec.ts b/e2e-test/clients/console-ui/e2e/flows/50-chunk-capacity-disk-group.spec.ts index 4651168ba..306b7033e 100644 --- a/e2e-test/clients/console-ui/e2e/flows/50-chunk-capacity-disk-group.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/50-chunk-capacity-disk-group.spec.ts @@ -33,6 +33,7 @@ test('native capacity: disk-group CRUD and explicit owner binding use mounted st await page.goto('/?domain=Capacity'); const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); const node = sidebar.getByTestId('tree-node-N-1').locator(':scope > [role="treeitem"]'); + await expect(sidebar.getByRole('button', { name: 'Add Rack', exact: true })).toBeEnabled(); await diskMenu(page, node, 'Add Disk Group'); const create = page.getByRole('dialog', { name: 'Add Disk Group', exact: true }); const id = create.getByLabel('Disk Group ID (auto-assigned)'); @@ -80,7 +81,7 @@ test('native capacity: disk-group CRUD and explicit owner binding use mounted st const defaults = (await json(request, '/api/deployment-defaults?node_id=1')).diskio; const response = await request.post('/api/nodes/1/services/deploy', { data: { ...defaults, kind: 'diskio', instance_id: '1', disk_group_id: 520, - rpc_port: 7200, test_single_node: false, + rpc_port: 7200, } }); expect(response.status(), await response.text()).toBe(201); }); @@ -156,6 +157,7 @@ test('KV binding recommends the least loaded group and initialization creates Gr await page.getByTestId('domain-capacity').click(); const sidebar = page.getByRole('complementary', { name: 'Cluster tree sidebar' }); const node = sidebar.getByTestId('tree-node-N-1').locator(':scope > [role="treeitem"]'); + await expect(sidebar.getByRole('button', { name: 'Add Rack', exact: true })).toBeEnabled(); await diskMenu(page, node, 'Add Disk Group'); const dialog = page.getByRole('dialog', { name: 'Add Disk Group', exact: true }); const binding = dialog.getByRole('combobox', { name: 'KV group', exact: true }); diff --git a/e2e-test/clients/console-ui/e2e/flows/52-chunk-capacity-zone.spec.ts b/e2e-test/clients/console-ui/e2e/flows/52-chunk-capacity-zone.spec.ts index 1a03532e2..a20767041 100644 --- a/e2e-test/clients/console-ui/e2e/flows/52-chunk-capacity-zone.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/52-chunk-capacity-zone.spec.ts @@ -17,7 +17,7 @@ test('native diagnostics: Zone canvas colors match every actual allocation bit', expect(zone.busy_block_count).toBeGreaterThan(0); expect(zone.free_block_count).toBeGreaterThan(0); await step('native zone DOM setup', async () => { - await page.goto('/?domain=Capacity'); + await page.goto('/?domain=Cluster'); await expect(page.getByRole('complementary', { name: 'Cluster tree sidebar' })).toBeVisible(); }); // Open the actual Storage branch before its DiskGroup and disk. diff --git a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts index 51a8c93b9..2b049afd0 100644 --- a/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/55-chunk-kv-catalog.spec.ts @@ -17,16 +17,16 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i expect(response.ok(), await response.text()).toBe(true); const { partition, observed } = await selectActiveJournal(page, request, await response.json()); const runtime = await observed.json(); - expect(runtime.journal.active).not.toBeNull(); + expect(runtime.journal.chunk_record).not.toBeNull(); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); const journal = page.getByRole('tabpanel', { name: 'Journal', exact: true }); - await expect(journal).toContainText(runtime.journal.active.chunk_id); - for (const [name, value] of [['Stream manifest generation', runtime.journal.generation], ['Stream writer epoch', runtime.journal.writer_epoch], ['Metadata group', runtime.journal.metadata_group_id]]) { + await expect(journal).toContainText(runtime.journal.chunk_record.chunk_id); + for (const [name, value] of [['Stream chunk record generation', runtime.journal.generation], ['Stream writer epoch', runtime.journal.writer_epoch], ['Metadata group', runtime.journal.metadata_group_id]]) { await expect(journal.locator('dt').filter({ hasText: new RegExp(`^${name}$`) }).locator('..').locator('dd')).toHaveText(value); } - await expect(journal.getByLabel('Extent page map').getByRole('button')).toHaveCount(runtime.journal.extent_pages.length); - await journal.getByRole('button', { name: 'Inspect active Chunk', exact: true }).click(); - await expect(page.getByLabel('Exact Chunk ID')).toHaveValue(runtime.journal.active.chunk_id); + await expect(journal.getByLabel('Stream chunk record', { exact: true })).toContainText(runtime.journal.chunk_record.logical_end); + await journal.getByRole('button', { name: 'Inspect Chunk', exact: true }).click(); + await expect(page.getByLabel('Exact Chunk ID')).toHaveValue(runtime.journal.chunk_record.chunk_id); await page.goBack(); await expect(journal).toBeVisible(); // Back navigation preserves the selection, but normal placement may have @@ -46,8 +46,8 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i const observation = page.getByRole('region', { name: 'Partition runtime', exact: true }); await observation.getByRole('button', { name: 'Refresh runtime', exact: true }).click(); await expect(observation).toContainText('Runtime unavailable'); - await expect(journal.getByLabel('Extent page map')).toHaveCount(0); - await expect(journal.getByRole('button', { name: 'Inspect active Chunk', exact: true })).toHaveCount(0); + await expect(journal.getByLabel('Stream chunk record', { exact: true })).toHaveCount(0); + await expect(journal.getByRole('button', { name: 'Inspect Chunk', exact: true })).toHaveCount(0); } finally { expect((await request.post(`/api/services/${owner}/restart`, { data: {} })).ok()).toBe(true); } @@ -60,7 +60,7 @@ test('native diagnostics: journal identity survives Chunk navigation and owner i await page.getByRole('tab', { name: 'Journal', exact: true }).click(); const observation = page.getByRole('region', { name: 'Partition runtime', exact: true }); await expect(observation).not.toContainText('Runtime unavailable'); - await expect(journal.getByRole('heading', { name: 'Published extent index', exact: true })).toBeVisible(); + await expect(journal.getByRole('heading', { name: 'Stream chunk record', exact: true })).toBeVisible(); const stale = new URL(observed.url()); stale.searchParams.set('stream_generation', runtime.journal.generation); stale.searchParams.set('stream_offset', '0'); @@ -187,7 +187,7 @@ async function selectActiveJournal(page: Page, request: APIRequestContext, initi }; const changed = async (response: APIResponse | Response) => { if (response.status() !== 409) return false; - expect((await response.json()).error).toMatch(/^(Chunk-KV catalog changed; refresh the range map|Catalog changed during runtime observation|Owner, catalog or tree page observation changed; refresh the catalog or page root)$/); + expect((await response.json()).error).toMatch(/^(Chunk-KV catalog changed(?: during observation)?; refresh the range map|Catalog changed during runtime observation|Owner, catalog or tree page observation changed; refresh the catalog or page root)$/); await refresh(() => page.getByRole('button', { name: 'Refresh catalog', exact: true }).click()); return true; }; @@ -201,7 +201,7 @@ async function selectActiveJournal(page: Page, request: APIRequestContext, initi const response = await request.get(`/api/chunk-kv/runtime?${query}`); if (await changed(response)) return false; expect(response.ok(), await response.text()).toBe(true); - if (!(await response.json()).journal.active) continue; + if (!(await response.json()).journal.chunk_record) continue; await page.getByLabel('Partition range map').getByRole('button', { name: `Partition ${entry.id}`, exact: true }).click(); // Selecting the same partition leaves its identity unchanged. Explicitly // refresh runtime so owner restart is observed even without a new catalog. @@ -221,8 +221,8 @@ async function selectActiveJournal(page: Page, request: APIRequestContext, initi return selected as { partition: Partition; observed: Response }; } -// Baseline: actual large native Journal directory (2026-10-04). -test('native diagnostics: large Journal replaces 100 extent fences with its remainder', async ({ page, request }) => { +// Baseline: 3.9s (2026-10-24), including owner restart and stale-generation checks. +test('native diagnostics: large Journal exposes a bounded tail chunk record across restart', async ({ page, request }) => { test.skip(!process.env.CROWDB_NATIVE_JOURNAL_WINDOWS, 'requires actual small-geometry native stream writes'); const response = await request.get('/api/chunk-kv/catalog?page=0&offset=0'); expect(response.ok(), await response.text()).toBe(true); @@ -234,49 +234,33 @@ test('native diagnostics: large Journal replaces 100 extent fences with its rema const observed = await request.get(`/api/chunk-kv/runtime?${query}`); expect(observed.ok(), await observed.text()).toBe(true); const runtime = await observed.json(); - if (runtime.journal.next_offset === 100) { target = entry; first = runtime.journal; break; } + if (BigInt(runtime.journal.chunk_record.logical_start) > 0n) { target = entry; first = runtime.journal; break; } } expect(target).not.toBeNull(); - expect(first.extent_pages).toHaveLength(100); - const query = new URLSearchParams({ id: target.id, epoch: target.epoch, generation: catalog.generation, page: '0', offset: '0', stream_generation: first.generation, stream_offset: '100' }); - const remainderResponse = await request.get(`/api/chunk-kv/runtime?${query}`); - expect(remainderResponse.ok(), await remainderResponse.text()).toBe(true); - const remainder = (await remainderResponse.json()).journal; - expect(remainder.extent_pages.length).toBeGreaterThan(0); - expect(remainder.extent_pages.length).toBeLessThan(100); - expect(remainder.next_offset).toBeNull(); + expect(first.next_offset).toBeNull(); + expect(first.chunk_record.state).toBe('Active'); + expect(BigInt(first.chunk_record.logical_end)).toBeGreaterThanOrEqual(BigInt(first.chunk_record.logical_start)); + const query = new URLSearchParams({ id: target.id, epoch: target.epoch, generation: catalog.generation, page: '0', offset: '0', stream_generation: first.generation, stream_offset: '0' }); await page.goto('/?domain=Chunk-KV'); await page.getByLabel('Partition range map').getByRole('button', { name: `Partition ${target.id}`, exact: true }).click(); await page.getByRole('tab', { name: 'Journal', exact: true }).click(); const journal = page.getByRole('tabpanel', { name: 'Journal', exact: true }); - const fences = journal.getByLabel('Extent page map').getByRole('button'); - await expect(fences).toHaveCount(100); - await journal.getByRole('button', { name: 'Next extent pages', exact: true }).click(); - await expect(fences).toHaveCount(remainder.extent_pages.length); - await expect(fences).toHaveText(remainder.extent_pages.map((entry: { page_index: string; first_logical: string; end_logical: string }) => `Extent page ${entry.page_index}[${entry.first_logical}, ${entry.end_logical})`)); - await expect(journal.getByRole('button', { name: 'Next extent pages', exact: true })).toBeDisabled(); - const selected = remainder.extent_pages[0]; - await journal.getByLabel('Extent page map').getByRole('button', { name: new RegExp(`^Extent page ${selected.page_index}\\b`) }).click(); - await expect(page.getByRole('complementary', { name: 'Selected extent page', exact: true })).toContainText(selected.page_index); - await journal.getByRole('button', { name: 'Inspect active Chunk', exact: true }).click(); - await expect(page.getByLabel('Exact Chunk ID')).toHaveValue(first.active.chunk_id); + await expect(journal.getByLabel('Stream chunk record', { exact: true })).toContainText(first.chunk_record.chunk_id); + await journal.getByRole('button', { name: 'Select stream chunk record', exact: true }).click(); + await expect(page.getByRole('complementary', { name: 'Selected stream chunk record', exact: true })).toContainText(first.chunk_record.logical_start); + await journal.getByRole('button', { name: 'Inspect Chunk', exact: true }).click(); + await expect(page.getByLabel('Exact Chunk ID')).toHaveValue(first.chunk_record.chunk_id); await page.goBack(); - await expect(fences).toHaveCount(remainder.extent_pages.length); - await expect(page.getByRole('complementary', { name: 'Selected extent page', exact: true })).toContainText(selected.page_index); + await expect(page.getByRole('complementary', { name: 'Selected stream chunk record', exact: true })).toContainText(first.chunk_record.logical_end); await page.goForward(); - await expect(page.getByLabel('Exact Chunk ID')).toHaveValue(first.active.chunk_id); + await expect(page.getByLabel('Exact Chunk ID')).toHaveValue(first.chunk_record.chunk_id); await page.goBack(); - await expect(fences).toHaveCount(remainder.extent_pages.length); - await journal.getByRole('button', { name: 'Previous extent pages', exact: true }).click(); - await expect(fences).toHaveCount(100); - await journal.getByRole('button', { name: 'Next extent pages', exact: true }).click(); - await expect(fences).toHaveCount(remainder.extent_pages.length); const owner = `chunk-kv-${target.owner_id}`; expect((await request.post(`/api/services/${owner}/restart`, { data: {} })).ok()).toBe(true); expect((await request.get(`/api/chunk-kv/runtime?${query}`)).status()).toBe(409); await page.getByRole('region', { name: 'Partition runtime', exact: true }).getByRole('button', { name: 'Refresh runtime', exact: true }).click(); - await expect(fences).toHaveCount(100); - await expect(journal).toContainText('offset 0'); + await expect(journal.getByLabel('Stream chunk record', { exact: true })).toBeVisible(); + await expect(page.getByRole('complementary', { name: 'Selected stream chunk record', exact: true })).toHaveCount(0); }); // The slow fixture uses the one-minute production split cooldown. @@ -325,7 +309,7 @@ test('native diagnostics: production split displays actual inherited and current await expect(journal).toContainText(identity(overlay.source_stream_name)); await expect(journal).toContainText(identity(partition.artifact.stream_name)); expect(identity(overlay.source_stream_name)).not.toBe(identity(partition.artifact.stream_name)); - for (const [label, value] of [['Replay offset (bytes)', overlay.replay_offset], ['Cutover offset (bytes)', overlay.cutover_offset], ['Manifest generation', overlay.source_stream_manifest_generation]]) { + for (const [label, value] of [['Replay offset (bytes)', overlay.replay_offset], ['Cutover offset (bytes)', overlay.cutover_offset], ['Stream chunk record generation', overlay.source_stream_generation]]) { await expect(inherited.locator('dt').filter({ hasText: label }).locator('..').locator('dd')).toHaveText(value); } await page.getByRole('tab', { name: 'Dependencies', exact: true }).click(); diff --git a/e2e-test/clients/console-ui/e2e/flows/60-iceberg-catalog.spec.ts b/e2e-test/clients/console-ui/e2e/flows/60-iceberg-catalog.spec.ts index 6b1293873..045a25211 100644 --- a/e2e-test/clients/console-ui/e2e/flows/60-iceberg-catalog.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/60-iceberg-catalog.spec.ts @@ -202,7 +202,13 @@ test('native diagnostics: unavailable catalog retries the actual Access services expect(response.ok(), await response.text()).toBe(true); }); } + const retried = Promise.all([ + page.waitForResponse(response => new URL(response.url()).pathname === '/api/access/iceberg/v1/config' && response.request().method() === 'GET'), + page.waitForResponse(response => new URL(response.url()).pathname === '/api/access/iceberg/v1/namespaces' && response.request().method() === 'GET'), + ]); await page.getByRole('button', { name: 'Retry catalog', exact: true }).click(); + for (const response of await retried) expect(response.ok(), await response.text()).toBe(true); + await expect(page.getByRole('navigation', { name: 'Iceberg tree', exact: true })).toBeVisible(); await expect(page.getByRole('button', { name: 'Retry catalog', exact: true })).toHaveCount(0); await page.getByText('Catalog actions', { exact: true }).click(); await expect(page.getByRole('button', { name: 'Create metadata demo', exact: true })).toBeEnabled(); diff --git a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts index d574ecf43..1411c15de 100644 --- a/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/71-s3-native.spec.ts @@ -46,7 +46,17 @@ test('S3 native multipart upload, HEAD, bounded preview and full round trip', as await page.getByLabel('Object file').setInputFiles({ name: 'multipart.txt', mimeType: 'text/plain', buffer: bytes }); }); await step('native 9 MiB multipart mutation and DOM refresh', async () => { - await page.getByRole('button', { name: 'Upload', exact: true }).click(); + const [completed] = await Promise.all([ + page.waitForResponse(response => { + const url = new URL(response.url()); + return response.request().method() === 'POST' + && url.pathname === objectPath && url.searchParams.has('uploadId'); + }), + page.getByRole('button', { name: 'Upload', exact: true }).click(), + ]); + const completion = await completed.text(); + expect(completed.status(), completion).toBe(200); + expect(completion).toContain(' { diff --git a/e2e-test/clients/console-ui/e2e/flows/72-managed-native.spec.ts b/e2e-test/clients/console-ui/e2e/flows/72-managed-native.spec.ts index 66affbced..398742438 100644 --- a/e2e-test/clients/console-ui/e2e/flows/72-managed-native.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/72-managed-native.spec.ts @@ -10,15 +10,14 @@ test.use({ actionTimeout: 3000 }); // Baseline: 6.0s (2026-10-04); actual monitor and all native services. test('Managed native KV, Iceberg, S3 and Chunk operations retain hardware boundaries', async ({ page, request }) => { - await step('managed hardware capability rejection', async () => { + await step('managed node authority and editing capabilities', async () => { await page.goto('/'); - await expect(page.getByTestId('managed-monitor-phase')).toContainText('ready', { timeout: 3000 }); - await expect(page.getByRole('button', { name: 'Add Rack' })).toHaveCount(0); + await expect(page.getByTestId('deployment-phase')).toHaveText('Active cluster'); + const capabilities = await request.get('/api/mode'); + expect(capabilities.ok(), await capabilities.text()).toBeTruthy(); + const actual = await capabilities.json(); + expect(actual).toEqual({ mode: 'node', readonly: false }); await expect(page.getByLabel('Management token')).toHaveCount(0); - for (const path of ['/api/racks', '/api/cluster/init', '/internal/reset', '/api/diskdb/scan', '/api/diskdb/recalc', '/api/diskdb/compact', '/api/diskdb/rebuild']) { - const response = await request.post(path, { data: {} }); - expect(response.status(), path).toBe(503); - } }); await step('managed user KV read', async () => { await page.getByTestId('domain-kv').click(); @@ -82,7 +81,8 @@ test('Managed native KV, Iceberg, S3 and Chunk operations retain hardware bounda await page.getByTestId('domain-chunk').click(); await page.getByLabel('Chunk type').selectOption('5'); - await page.getByRole('table', { name: 'Chunks' }).getByRole('button', { name: chunkId, exact: true }).click(); + await page.getByLabel('Exact Chunk ID', { exact: true }).fill(chunkId); + await page.getByRole('button', { name: 'Lookup ID', exact: true }).click(); const card = page.getByLabel('Chunk strips').getByTestId('chunk-strip').filter({ has: page.getByRole('button', { name: new RegExp(`^Sequence ${strip.strip_sequence} ·`) }) }); await card.getByRole('button', { name: /^(Mirror 1|Data 0)( · unavailable)?$/, exact: true }).click(); const properties = page.getByLabel('Chunk properties'); @@ -97,30 +97,34 @@ test('Managed native KV, Iceberg, S3 and Chunk operations retain hardware bounda }); }); -test('Managed native authority outage hides stale topology and retains monitor observation', async ({ page }) => { +test('Node authority outage rejects mutations and marks retained observations stale', async ({ page, request }) => { const pid = Number(process.env.CROWDB_NATIVE_KV_PID); expect(pid, 'Run the owned release-container managed runner').toBeGreaterThan(0); await page.goto('/'); - await expect(page.getByTestId('managed-source')).toHaveText('Source: Group 0'); + await expect(page.getByTestId('deployment-phase')).toHaveText('Active cluster'); await page.clock.install(); await signalService(pid, 'STOP'); try { - // One 3-second UI poll plus the backend's 3-second authority budget. - const failed = page.waitForResponse(response => response.url().endsWith('/api/preview') && response.status() === 503, { timeout: 10000 }); + const observed = page.waitForResponse(response => response.url().endsWith('/api/node/status') + && response.status() === 200, { timeout: 10000 }); await page.clock.runFor(3001); - const response = await failed; - const body = await response.json(); - expect(body.reason).toBe('group0_unavailable'); - expect(body.monitor.services.kv.pid).toBe(pid); - await expect(page.getByTestId('managed-unavailable')).toContainText('Group 0 is unavailable'); - await expect(page.getByRole('complementary', { name: 'Cluster tree sidebar' }).getByRole('button', { name: 'N-1', exact: true })).toHaveCount(0); - await expect(page.getByRole('region', { name: 'Monitor status' })).toBeVisible(); + expect((await (await observed).json()).phase).toBe('authority_unavailable'); + await expect(page.getByTestId('deployment-phase')).toHaveText('Group 0 unavailable · confirmed view is stale'); + // The API fixture covers the server's bounded authority check. + const mutation = await request.post('/api/racks', { data: { id: 2, name: 'must-not-commit' } }); + expect(mutation.status()).toBe(502); + await expect(page.getByRole('region', { name: 'Candidate nodes' })).toBeVisible(); + await expect(page.getByRole('complementary', { name: 'Cluster tree sidebar' }) + .getByRole('button', { name: 'N-1', exact: true })).toBeVisible(); } finally { await signalService(pid, 'CONT'); + await page.clock.resume(); } - await page.clock.runFor(3001); - await expect(page.getByTestId('managed-unavailable')).toHaveCount(0); - await expect(page.getByRole('complementary', { name: 'Cluster tree sidebar' })).toBeVisible(); + await expect(page.getByTestId('deployment-phase')).toHaveText('Active cluster'); + const racks = await request.get('/api/racks'); + expect(racks.ok(), await racks.text()).toBe(true); + expect((await racks.json()).some((rack: { id: number }) => rack.id === 2)).toBe(false); + }); async function signalService(pid: number, signal: 'STOP' | 'CONT') { diff --git a/e2e-test/clients/console-ui/e2e/flows/80-dataset-browse.spec.ts b/e2e-test/clients/console-ui/e2e/flows/80-dataset-browse.spec.ts index 368a280b2..4cddf85ec 100644 --- a/e2e-test/clients/console-ui/e2e/flows/80-dataset-browse.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/80-dataset-browse.spec.ts @@ -51,7 +51,12 @@ test('Dataset native: capabilities', async ({ page, request, baseURL }) => { await page.getByRole('button', { name: 'Release Snapshot' }).click(); await expect.poll(async () => (await call(request, `${datasetPath(name)}/snapshots/${snapshot}/reclaim`)).retained, { intervals: [100], timeout: 3000 }).toBe(false); const proxy = await consoleProxy(baseURL!); const seen: string[] = []; - page.on('request', value => { if (value.url().includes('/dataset/')) seen.push(value.url()); }); + let observedModules = 'activity'; + page.on('request', value => { + const document = new URL(page.url()); + if (document.origin === new URL(proxy.url).origin && document.searchParams.get('disableModules') === observedModules + && value.url().includes('/dataset/')) seen.push(value.url()); + }); try { await page.goto(`${proxy.url}/?domain=Dataset&readonly=1&disableModules=activity&apiPrefix=/proxy/api`); await page.getByRole('navigation', { name: 'Dataset tree' }).getByRole('button', { name, exact: true }).click(); @@ -61,7 +66,7 @@ test('Dataset native: capabilities', async ({ page, request, baseURL }) => { await expect(page.getByRole('button', { name: 'Create child Snapshot' })).toHaveCount(0); await expect(page.getByText('Session activity', { exact: true })).toHaveCount(0); expect(seen.length).toBeGreaterThan(0); expect(seen.every(url => new URL(url).pathname.startsWith('/proxy/api/'))).toBe(true); - seen.length = 0; + seen.length = 0; observedModules = 'dataset'; await page.goto(`${proxy.url}/?domain=Dataset&disableModules=dataset&apiPrefix=/proxy/api`); await expect(page.getByTestId('domain-cluster')).toHaveAttribute('aria-pressed', 'true'); await expect(page.getByTestId('domain-dataset')).toHaveCount(0); expect(seen).toEqual([]); diff --git a/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts b/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts index 7baf284f4..72ded0f27 100644 --- a/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts +++ b/e2e-test/clients/console-ui/e2e/flows/92-three-node-data.spec.ts @@ -125,7 +125,6 @@ async function addDisks({ page, request }: UIContext) { async function verifyServices({ page, request, deployments }: UIContext) { await step('three-node: all six services resume without Retry', async () => { - await expect.poll(async () => (await (await request.get('/api/chunk-storage-readiness')).json()), { intervals: [100] }).toMatchObject({ ready: true }); for (const kind of ['diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server']) { const pending = new Set([1, 2, 3]); while (pending.size) { @@ -141,6 +140,8 @@ async function verifyServices({ page, request, deployments }: UIContext) { pending.delete(id); } } + // Completion observes the asynchronous startup before checking readiness. + await expect.poll(async () => (await (await request.get('/api/chunk-storage-readiness')).json()), { intervals: [100] }).toMatchObject({ ready: true }); const servers = await (await request.get('/api/servers')).json(); expect(servers.filter((server: { pid: number }) => server.pid)).toHaveLength(18); const bindings = await (await request.get('/api/hardware/disk-group-bindings')).json(); diff --git a/e2e-test/clients/console-ui/release-web.cjs b/e2e-test/clients/console-ui/release-web.cjs index 19591d6af..210f3bcb8 100644 --- a/e2e-test/clients/console-ui/release-web.cjs +++ b/e2e-test/clients/console-ui/release-web.cjs @@ -14,8 +14,25 @@ async function main() { ].find(existsSync); if (!executablePath) throw new Error('Container acceptance requires an installed system browser'); const browser = await chromium.launch({ executablePath, headless: true }); + const requests = new Map(); try { const page = await browser.newPage(); + page.on('request', request => { + const path = new URL(request.url()).pathname; + if (path.startsWith('/api/') && requests.size < 100) requests.set(request, { path, method: request.method(), started: Date.now() }); + }); + page.on('response', response => { + const observed = requests.get(response.request()); + if (observed) Object.assign(observed, { status: response.status(), response_ms: Date.now() - observed.started }); + }); + page.on('requestfinished', request => { + const observed = requests.get(request); + if (observed) observed.completed_ms = Date.now() - observed.started; + }); + page.on('requestfailed', request => { + const observed = requests.get(request); + if (observed) observed.failure = request.failure()?.errorText; + }); const pageErrors = []; page.on('pageerror', error => pageErrors.push(error.message)); // Exercise the HTTP-origin path even when acceptance uses localhost. @@ -27,7 +44,11 @@ async function main() { for (const domain of ['cluster', 'kv', 'capacity', 'chunk', 'chunk-kv', 'iceberg', 's3']) { await expect(page.getByTestId(`domain-${domain}`)).toBeVisible({ timeout: 3000 }); } - await expect(page.getByRole('button', { name: 'Add Rack' })).toHaveCount(0); + await expect(page.getByRole('complementary', { name: 'Cluster tree sidebar' }) + .getByRole('button', { name: 'Add Rack', exact: true })).toBeEnabled({ timeout: 3000 }); + const capabilities = await page.request.get(new URL('/api/mode', process.argv[2]).href); + expect(capabilities.ok()).toBe(true); + expect(await capabilities.json()).toEqual({ mode: 'node', readonly: false }); if (process.argv[3]) { await verifyAuthorityOutage(page, process.argv[3]); } @@ -37,6 +58,11 @@ async function main() { } expect(pageErrors).toEqual([]); console.log('Container managed Web browser acceptance passed'); + } catch (error) { + console.error('[MANAGED WEB REQUESTS]', JSON.stringify([...requests.values()].map(observed => ({ + ...observed, elapsed_ms: Date.now() - observed.started, + })))); + throw error; } finally { await browser.close(); } @@ -47,20 +73,25 @@ async function verifyAuthorityOutage(page, container) { await page.clock.install(); await signalService('STOP', pid); try { - const failedSnapshot = page.waitForResponse(response => - response.url().endsWith('/api/preview') && response.status() === 503, + const observed = page.waitForResponse(response => + response.url().endsWith('/api/node/status') && response.status() === 200, { timeout: 10000 }); await page.clock.runFor(3001); - const response = await failedSnapshot; - const body = await response.json(); - expect(body.reason).toBe('group0_unavailable'); - expect(body.monitor.services.kv.pid).toBe(Number(pid)); - await expect(page.getByRole('complementary', { name: 'Cluster tree sidebar' }).getByRole('button', { name: 'N-1', exact: true })).toHaveCount(0, { timeout: 3000 }); + expect((await (await observed).json()).phase).toBe('authority_unavailable'); + await expect(page.getByTestId('deployment-phase')).toHaveText('Group 0 unavailable · confirmed view is stale', { timeout: 3000 }); + const mutation = await page.request.post(new URL('/api/racks', process.argv[2]).href, + { data: { id: 2, name: 'must-not-commit' } }); + expect(mutation.status()).toBe(502); + await expect(page.getByRole('complementary', { name: 'Cluster tree sidebar' }) + .getByRole('button', { name: 'N-1', exact: true })).toBeVisible({ timeout: 3000 }); } finally { await signalService('CONT', pid); + await page.clock.resume(); } - await page.clock.runFor(3001); - await expect(page.getByRole('complementary', { name: 'Cluster tree sidebar' })).toBeVisible({ timeout: 3000 }); + await expect(page.getByTestId('deployment-phase')).toHaveText('Active cluster', { timeout: 3000 }); + const racks = await page.request.get(new URL('/api/racks', process.argv[2]).href); + expect(racks.ok()).toBe(true); + expect((await racks.json()).some(rack => rack.id === 2)).toBe(false); } async function signalService(signal, pid) { diff --git a/e2e-test/clients/ecosystem/README.md b/e2e-test/clients/ecosystem/README.md index b3cd757d7..2da77e958 100644 --- a/e2e-test/clients/ecosystem/README.md +++ b/e2e-test/clients/ecosystem/README.md @@ -8,7 +8,7 @@ - Run the **Iceberg container ecosystem** GitHub Actions workflow manually. Its `all` profile is the complete verified matrix; `python` is a focused Python/DuckDB diagnostic and cannot certify distributed engines. -- Locally, run `pixi run build-single-node-container`, then +- Locally, run `pixi run oci-build --load`, then `pixi run -e iceberg-ecosystem test-iceberg-container-ecosystem`. No default task or default CI job depends on this environment or test. - Every run owns an isolated bridge network and normal release node with private diff --git a/e2e-test/clients/s3/base.py b/e2e-test/clients/s3/base.py index 9c2e1ec61..f0406b2d9 100644 --- a/e2e-test/clients/s3/base.py +++ b/e2e-test/clients/s3/base.py @@ -24,6 +24,8 @@ from botocore.credentials import Credentials from botocore.exceptions import ClientError +from http_exchange import request_response + class FragmentedBody(BytesIO): def __init__(self, payload, seed): @@ -110,9 +112,10 @@ def signed_http( for offset in range(0, len(body), slow_chunk_size): connection.send(body[offset : offset + slow_chunk_size]) time.sleep(0.002) + response = connection.getresponse() else: - connection.request(method, path, body=body, headers=dict(request.headers.items())) - response = connection.getresponse() + response = request_response(connection, method, path, body, + dict(request.headers.items())) return response.status, dict(response.getheaders()), response.read() finally: connection.close() diff --git a/e2e-test/clients/s3/clients.py b/e2e-test/clients/s3/clients.py index 6ac91d052..d83512df1 100644 --- a/e2e-test/clients/s3/clients.py +++ b/e2e-test/clients/s3/clients.py @@ -30,7 +30,7 @@ def run(client, args, env): binary = Path(sys.executable).parent / client result = subprocess.run([str(binary), *args], env=env, capture_output=True, timeout=120) if result.returncode: - diagnostic = result.stderr.decode(errors="replace") + diagnostic = f"command: {client} {args!r}\nstdout:\n{result.stdout.decode(errors='replace')}\nstderr:\n{result.stderr.decode(errors='replace')}" for name in ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]: if env.get(name): diagnostic = diagnostic.replace(env[name], "") @@ -39,6 +39,17 @@ def run(client, args, env): return result.stdout.decode() +def aws_failure_snapshot(endpoint, env, error, probes): + for probe in probes: + try: + detail = run("aws", ["--endpoint-url", endpoint, "--no-cli-pager", + "--cli-connect-timeout", "2", "--cli-read-timeout", "2", + "s3api", *probe], env | {"AWS_MAX_ATTEMPTS": "1"}) + except (AssertionError, subprocess.SubprocessError, OSError) as diagnostic: + detail = str(diagnostic) + error.add_note(f"post-failure {probe}: {detail}") + + def aws_workflow(endpoint, phase="all"): env = client_env(endpoint) bucket = "crowdb-aws-cli" @@ -58,10 +69,28 @@ def aws(*args): (source / "small.txt").write_bytes(b"CLI exact bytes") if phase in ["all", "write"]: aws("s3api", "create-bucket", "--bucket", bucket) - assert bucket in aws("s3", "ls") + try: + buckets = aws("s3", "ls") + assert bucket in buckets, f"new bucket missing from CLI listing: {buckets!r}" + except AssertionError as error: + aws_failure_snapshot(endpoint, env, error, [ + ["head-bucket", "--bucket", bucket], ["list-buckets"], + ]) + raise aws("s3", "cp", str(source / "large.bin"), f"s3://{bucket}/prefix/large.bin") aws("s3", "cp", str(source / "small.txt"), f"s3://{bucket}/prefix/small.txt") - assert "large.bin" in aws("s3", "ls", f"s3://{bucket}/prefix/", "--recursive") + try: + listed = aws("s3", "ls", f"s3://{bucket}/prefix/", "--recursive") + except AssertionError as error: + probes = [ + ["list-objects-v2", "--bucket", bucket], + ["list-objects-v2", "--bucket", bucket, "--prefix", "prefix/"], + ["head-object", "--bucket", bucket, "--key", "prefix/large.bin"], + ["head-object", "--bucket", bucket, "--key", "prefix/small.txt"], + ] + aws_failure_snapshot(endpoint, env, error, probes) + raise + assert "large.bin" in listed aws("s3", "cp", f"s3://{bucket}/prefix/small.txt", f"s3://{bucket}/copied.txt", "--metadata-directive", "COPY") aws("s3", "cp", f"s3://{bucket}/prefix/large.bin", f"s3://{bucket}/copied-large.bin", "--metadata-directive", "COPY") aws("s3", "sync", str(source), f"s3://{bucket}/synced/", "--delete") diff --git a/e2e-test/clients/s3/http_exchange.py b/e2e-test/clients/s3/http_exchange.py new file mode 100644 index 000000000..b68aca194 --- /dev/null +++ b/e2e-test/clients/s3/http_exchange.py @@ -0,0 +1,15 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""Read an early HTTP rejection even when its peer has stopped accepting body bytes.""" + + +def request_response(connection, method, path, body, headers): + try: + connection.request(method, path, body=body, headers=headers) + except (BrokenPipeError, ConnectionResetError): + response = connection.getresponse() + if response.status < 400: + raise + return response + return connection.getresponse() diff --git a/e2e-test/console_cli_client.py b/e2e-test/console_cli_client.py new file mode 100644 index 000000000..ef6f578df --- /dev/null +++ b/e2e-test/console_cli_client.py @@ -0,0 +1,57 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Explicit CLI attachment and bounded, redacted host invocation artifacts.""" + +import json +import os +import subprocess +import time +import urllib.error +import urllib.request + +from fixture import redact + + +class CliClient: + def diagnostics(self): + # Export cached counters before teardown, including a failed probe's + # latency. These reads never issue dependency health probes. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + for index, address in enumerate(self.addresses): + try: + with opener.open(f"http://{address}:9091/_crowdb/metrics", timeout=2) as response: + metrics = response.read(1024 * 1024 + 1) + if len(metrics) > 1024 * 1024: + raise ValueError("Access metrics exceeded the diagnostic limit") + (self.artifacts / f"access-metrics-{index}.prom").write_text(redact(metrics.decode())) + except (OSError, ValueError, urllib.error.HTTPError) as error: + (self.artifacts / f"access-metrics-{index}.error").write_text(redact(str(error))) + super().diagnostics() + + def environment(self): + environment = dict(os.environ, CROWDB_RUNTIME_ROOT=str(self.root), NO_PROXY="*", no_proxy="*") + for name in ("CROWDB_CLUSTER", "CROWDB_SYSTEM_IP", "CROWDB_SYSTEM_PORT"): + environment.pop(name, None) + environment.update(getattr(self, "extra_environment", {})) + return environment + + def arguments(self, *arguments): + return [str(self.bundle / "crowdb-cli"), "--cluster", self.run_id, *arguments] + + def invoke(self, *arguments, endpoint=None, expected_exit=0): + self.phase_number += 1 + started = time.monotonic() + command = self.arguments(*arguments) if endpoint is None else [ + str(self.bundle / "crowdb-cli"), "--management-endpoint", endpoint, *arguments] + result = subprocess.run(command, env=self.environment(), + capture_output=True, text=True, timeout=210) + output = result.stdout + result.stderr + (self.artifacts / f"client-cli-{self.phase_number}.log").write_text(redact(output)) + (self.artifacts / f"client-cli-{self.phase_number}.json").write_text(json.dumps({ + "case": self.case_name, "phase": ".".join(arguments[:2]), "execution": "host", + "seconds": time.monotonic() - started, "exit_code": result.returncode, + "expected_exit_code": expected_exit}, indent=2)) + if result.returncode != expected_exit: + print(output, flush=True) + raise AssertionError(output) + return result.stdout diff --git a/e2e-test/console_cli_fixture.py b/e2e-test/console_cli_fixture.py index 17b7b9308..60668362e 100644 --- a/e2e-test/console_cli_fixture.py +++ b/e2e-test/console_cli_fixture.py @@ -1,82 +1,74 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Host CLI recovery against three owned formal release containers.""" +"""Common CLI interruption/recovery against ordinary OCI node startup.""" import json -import tomllib import os +from pathlib import Path import signal import subprocess import time from fixture import Cluster, redact, request +from console_cli_recovery import terminal_writer_recovery +from console_cli_client import CliClient -LABEL = "org.crowdb.cluster.id" -ROOT_LABEL = "org.crowdb.cluster.root" +LABEL = "org.crowdb.cluster" +OWNER_LABEL = "org.crowdb.cluster.owner" +ROOT = Path(__file__).resolve().parents[1] -class CliContainerCluster(Cluster): +class CliContainerCluster(CliClient, Cluster): def __init__(self, image, artifacts, bundle, checkpoint=None, **kwargs): super().__init__(image, artifacts, node_count=3, wire_replicas=False, **kwargs) - self.bundle = bundle - self.root = (self.data_root / "cluster").resolve() + self.bundle = Path(bundle).resolve() + self.root = (self.data_root / "runtime").resolve() self.phase_number = 0 self.checkpoint = checkpoint - self.delete_after_replay = False + self.port_base = None - def start(self): - self.root.mkdir(parents=True) - if self.checkpoint is not None: - self.interrupt(self.checkpoint) - self.refresh() - return self - - def arguments(self, *arguments): - return [str(self.bundle / "crowdb-cli"), "s3", *arguments] - - def environment(self): - return dict(os.environ, NO_PROXY="*", no_proxy="*") + def state(self): + return json.loads((self.root / "persistent/oci-clusters" / f"{self.run_id}.json").read_text()) - def invoke(self, *arguments): - self.phase_number += 1 - started = time.monotonic() - result = subprocess.run(self.arguments(*arguments), env=self.environment(), - capture_output=True, text=True, timeout=180) - output = result.stdout + result.stderr - (self.artifacts / f"client-cli-{self.phase_number}.log").write_text(redact(output)) - (self.artifacts / f"client-cli-{self.phase_number}.json").write_text(json.dumps({ - "case": self.case_name, "phase": ".".join(arguments[:2]), "execution": "host", - "seconds": time.monotonic() - started, "exit_code": result.returncode}, indent=2)) - if result.returncode: - raise AssertionError(output) - return result.stdout + def start_arguments(self): + return ("cluster", "start", "--nodes", "3", "--image", self.image, + "--cpus", str(self.node_cpus), "--memory-mib", "2048", + "--port-base", str(self.port_base), "--wait", "180", "--json") - def state(self): - return json.loads((self.root / "container-cluster.json").read_text()) + def start(self): + self.root.mkdir(parents=True, mode=0o700) + # Claims belong to this live runner in the shared allocator. Receipt roots + # remain per case; separate roots must not allocate overlapping host ports. + result = subprocess.run([str(self.bundle / "crowdb-cli"), "port-alloc", + "--service", "web", "--count", "24", "--owner-pid", str(os.getpid())], + capture_output=True, text=True, timeout=10, check=True) + ports = [int(port) for port in result.stdout.split()] + if len(ports) != 24 or ports != list(range(ports[0], ports[0] + 24)): + raise AssertionError("CLI fixture needs 24 contiguous published ports") + self.port_base = ports[0] + self.interrupt("storage" if self.checkpoint == "terminal-writer" else self.checkpoint) + self.refresh() + return self def interrupt(self, checkpoint): - relative = os.path.relpath(self.root, os.getcwd()) - arguments = self.arguments("cluster", "start", "--root", relative, - "--container", self.run_id, "--image", self.image, - "--node-cpus", str(self.node_cpus), "--memory-mib", "2048", - "--diskdb-heartbeat-interval-secs", "3") + if checkpoint not in ("bound", "storage"): + raise ValueError("unsupported common CLI recovery checkpoint") log = self.artifacts / "client-interrupted-start.log" interrupted_at = time.monotonic() with log.open("w") as output: - process = subprocess.Popen(arguments, env=self.environment(), stdout=output, stderr=output) + process = subprocess.Popen(self.arguments(*self.start_arguments()), env=self.environment(), + stdout=output, stderr=output) try: - deadline = time.monotonic() + 90 + deadline = time.monotonic() + 180 while True: if process.poll() is not None: raise AssertionError(f"CLI ended before {checkpoint}: {log.read_text()}") - path = self.root / "container-cluster.json" - state = json.loads(path.read_text()) if path.exists() else {} - if state.get("phase") == checkpoint: + if self.checkpoint_reached(checkpoint): process.send_signal(signal.SIGSTOP) break if time.monotonic() >= deadline: - raise AssertionError(f"CLI never reached {checkpoint}: {state}") - time.sleep(0.01) + raise AssertionError(f"CLI never reached {checkpoint}: {log.read_text()}") + time.sleep(0.02) finally: if process.poll() is None: process.kill() @@ -87,142 +79,175 @@ def interrupt(self, checkpoint): "ok": True, "result": {"checkpoint": checkpoint, "signal": "SIGKILL", "exit_code": process.returncode}, "seconds": time.monotonic() - interrupted_at}, indent=2)) + def checkpoint_reached(self, checkpoint): + receipt = self.root / "persistent/oci-clusters" / f"{self.run_id}.json" + if not receipt.exists(): + return False + self.refresh() + if len(self.servers) != 3: + return False + path = "node-binding.json" if checkpoint == "bound" else "diskdb-execution" + try: + value = self.read_owned(path) + except RuntimeError: + return False # The monitor has not published this checkpoint yet. + return bool(value.get("bootstrap")) if checkpoint == "bound" else bool(value.get("complete")) + def replay(self, checkpoint): if checkpoint != self.checkpoint: raise ValueError("CLI recovery checkpoint was not prepared before client dispatch") - assert (self.root / "s3-mini-cluster.initializing.json").is_file() - assert (self.root / "s3-local-state.toml").is_file() - assert not (self.root / "console.toml").exists() before = self.state() - self.refresh() binding = self.read_owned("node-binding.json") - diskdb = self.read_owned("services/diskdb-1/execution.json") if checkpoint == "storage" else None - if diskdb is not None: - assert diskdb["complete"] and diskdb["pid"] > 1 - self.command("exec", self.servers[0], "/bin/kill", "-0", str(diskdb["pid"])) - ids = [json.loads(self.command("inspect", name))[0]["Id"] for name in self.servers] - self.invoke("cluster", "start", "--container", self.servers[1]) - after = self.state() - assert before["id"] == after["id"] and before["addresses"] == after["addresses"] - assert before["diskdb_heartbeat_interval_secs"] == after["diskdb_heartbeat_interval_secs"] == 3 - self.assert_diskdb_cadence() - self.file_disks() - assert binding == self.read_owned("node-binding.json") - if diskdb is not None: - assert diskdb == self.read_owned("services/diskdb-1/execution.json"), "confirmed Diskdb execution must survive resume" - self.command("exec", self.servers[0], "/bin/kill", "-0", str(diskdb["pid"])) - assert ids == [json.loads(self.command("inspect", name))[0]["Id"] for name in self.servers] - assert not (self.root / "s3-mini-cluster.initializing.json").exists() - assert not (self.root / "bootstrap-intent.toml").exists() - assert "[[rack]]" not in (self.root / "s3-local-state.toml").read_text() - assert "Containers: 3/3" in self.invoke("cluster", "status", "--container", self.servers[2]) - if checkpoint == "bound": - self.lifecycle() - # The batch retains peer networks until every client has ended. - # Teardown still verifies the ordinary CLI delete operation itself. - self.delete_after_replay = True - - def lifecycle(self): + container_ids = [self.inspect_owned(name)["Id"] for name in self.servers] + execution = self.read_owned("diskdb-execution") if checkpoint == "storage" else None + report = json.loads(self.invoke("cluster", "start", "--wait", "180", "--json")) + assert report["phase"] == "ready", report + assert before == self.state(), "resume must retain accepted launch inputs" + assert binding == self.read_owned("node-binding.json"), "resume must retain accepted binding" + assert container_ids == [self.inspect_owned(name)["Id"] for name in self.servers] + if execution is not None: + assert execution == self.read_owned("diskdb-execution"), "confirmed storage must survive CLI interruption" + if checkpoint == "terminal-writer": + terminal_writer_recovery(self, report["observation"]["cluster_id"]) + return + self.lifecycle(report["observation"]["cluster_id"]) + self.current_readiness(report["observation"]["cluster_id"]) + + def lifecycle(self, cluster_id): files = self.file_disks() - self.invoke("bucket", "put", "--container", self.run_id, "cli-replay") - self.invoke("object", "put", "--container", self.run_id, "cli-replay", "witness", "--text", "durable-container-witness") - self.invoke("cluster", "stop", "--container", self.servers[1]) - assert "Containers: 0/3" in self.invoke("cluster", "status", "--root", str(self.root)) - self.invoke("cluster", "restart", "--container", self.run_id) - self.assert_diskdb_cadence() + self.invoke("s3", "bucket", "put", "cli-replay") + self.invoke("s3", "object", "put", "cli-replay", "witness", "--text", "durable-container-witness") + self.invoke("cluster", "stop") + assert all(not self.inspect_owned(name)["State"]["Running"] for name in self.servers) + report = json.loads(self.invoke("cluster", "start", "--wait", "180", "--json")) + assert report["phase"] == "ready" and report["observation"]["cluster_id"] == cluster_id assert self.file_disks() == files, "restart must retain the accepted file disk identities" - assert self.invoke("object", "get", "--container", self.servers[2], "cli-replay", "witness") == "durable-container-witness" - + assert self.invoke("s3", "object", "get", "cli-replay", "witness") == "durable-container-witness" + # Alternate-member endpoint attachment uses no local runtime selector. + report = json.loads(self.invoke("cluster", "check", "--json", + endpoint=f"http://{self.addresses[2]}:9090")) + assert report["phase"] == "ready" and report["observation"]["cluster_id"] == cluster_id + + def current_readiness(self, cluster_id): + origin = f"http://{self.addresses[0]}:9090" + node_id = self.read_owned("node-binding.json")["node_id"] + services = request(origin, "/api/servers") + service = next(item for item in services if item["node_id"] == node_id + and item.get("service_type") == "diskio") + service_id = service["id"] + dependency = f"node/{node_id}/service/{service_id}" + # Restart belongs to the fixture's explicit mutation, never to check. + try: + request(origin, f"/api/services/{service_id}/stop", {}) + started = time.monotonic() + stopped = json.loads(self.invoke("cluster", "check", "--wait", "2", "--json", + endpoint=origin, expected_exit=1)) + assert time.monotonic() - started < 15, "endpoint-only check exceeded its bounded budget" + if stopped["phase"] == "unavailable": + # The two-second endpoint budget can also expire while collecting + # authority. Its higher-priority reason must remain explicit. + assert any(reason["dependency"] == "cluster/authority" + and reason["code"] == "management-origin-unavailable" + for reason in stopped["reasons"]), stopped["reasons"] + else: + assert stopped["phase"] == "degraded", stopped["reasons"] + assert stopped["observation"]["cluster_id"] == cluster_id + fact = next(item for item in stopped["observation"]["dependencies"] if item["id"] == dependency) + assert fact["state"] == "stopped" and fact["current_operation_id"], fact + assert fact["reason"]["code"] == "required-service-not-ready", fact + assert "current committed action is stop or delete" in fact["detail"], fact + again = request(origin, "/api/cluster/status") + current = next(item for item in again["observation"]["dependencies"] if item["id"] == dependency) + assert current["state"] == "stopped", current + assert current["current_operation_id"] == fact["current_operation_id"], "check changed service intent" + finally: + request(origin, f"/api/services/{service_id}/restart", {}) + recovered = json.loads(self.invoke("cluster", "check", "--wait", "180", "--json", endpoint=origin)) + assert recovered["phase"] == "ready" and recovered["observation"]["cluster_id"] == cluster_id + + def inspect_owned(self, name): + info = json.loads(self.command("inspect", name))[0] + record = self.state() + labels = info["Config"]["Labels"] + if (labels.get(LABEL) != self.run_id or labels.get(OWNER_LABEL) != record["owner"] + or info["Image"] != record["image_id"]): + raise ValueError("CLI recovery observation ownership or image changed") + return info def file_disks(self): observed = [] for index, address in enumerate(self.addresses): - policy = request(f"http://{address}:9090", "/api/node/auto-join") - assert policy["enabled"] is False, policy - page = request(f"http://{self.addresses[0]}:9090", f"/api/nodes/{index + 1}/disk-files") + node_id = self.read_owned("node-binding.json", index)["node_id"] + page = request(f"http://{address}:9090", f"/api/nodes/{node_id}/disk-files") assert page["next"] is None and len(page["items"]) == 1, page disk = page["items"][0] - assert disk["phase"] == "registered", disk - assert disk["request"]["admission"]["disk_group_id"] == index + 1, disk - assert disk["request"]["sparse"] is True, disk - assert disk["preparation_complete"] and disk["backing_identity"], disk + assert disk["phase"] == "registered" and disk["preparation_complete"] and disk["backing_identity"], disk observed.append({key: disk[key] for key in ("request", "path", "backing_identity")}) return observed - def assert_diskdb_cadence(self): - for index in range(3): - execution = self.read_owned(f"services/diskdb-{index + 1}/execution.json", index) - assert execution["complete"] and execution["pid"] > 1 - configuration = tomllib.loads(execution["intent"]["configuration"]) - assert configuration["heartbeat"]["interval_secs"] == 3, "running DiskDB must use the recorded cadence" - def refresh(self): - state = self.state() - self.servers = [state["name"], state["name"] + "-node2", state["name"] + "-node3"] + self.servers, self.addresses = [], [] + network = "crowdb-" + self.run_id + for index in range(1, 4): + name = f"{network}-{index}" + try: + info = self.inspect_owned(name) + except RuntimeError: + continue + self.servers.append(name) + self.addresses.append(info["NetworkSettings"]["Networks"][network]["IPAddress"]) self.containers = self.servers.copy() - self.addresses = state["addresses"] self.origins = [f"http://{address}:10000" for address in self.addresses] - self.data_paths = [self.root / "nodes" / f"node{index}" / "data" for index in range(3)] - (self.artifacts / "mounts.json").write_text(json.dumps({ - "container_data": "/opt/crowdb/data", "host_data": [str(path) for path in self.data_paths]}, indent=2)) (self.artifacts / "endpoints.json").write_text(json.dumps({ - "addresses": self.addresses, "web_port": 9090, "kv_port": 10000, - "diskdb_ports": [7100, 7101, 7102], "diskio_port": 7200, - "chunkdb_ports": [7300, 7301], "chunk_kv_ports": [7400, 7401], - "access_ports": [9000, 9001, 9002]}, indent=2)) + "addresses": self.addresses, "web_port": 9090, + "published_console_ports": [self.port_base + index * 10 for index in range(3)]}, indent=2)) def read_owned(self, path, index=0): - if type(index) is not int or index < 0: - raise ValueError("CLI recovery node is outside the owned cluster") - if path != "node-binding.json" and path != f"services/diskdb-{index + 1}/execution.json": + if path not in ("node-binding.json", "diskdb-execution"): raise ValueError("unsupported CLI recovery observation") - if index >= len(getattr(self, "servers", ())): + if type(index) is not int or not 0 <= index < len(getattr(self, "servers", ())): raise ValueError("CLI recovery node is outside the owned cluster") - info = json.loads(self.command("inspect", self.servers[index]))[0] - labels = info["Config"]["Labels"] - if labels.get(LABEL) != self.state()["id"] or labels.get(ROOT_LABEL) != str(self.root) or info["Image"] != self.image: - raise ValueError("CLI recovery observation ownership changed") + info = self.inspect_owned(self.servers[index]) + if path == "diskdb-execution": + binding = self.read_owned("node-binding.json", index) + node_id = binding["node_id"] + if type(node_id) is not int or node_id < 1: + raise ValueError("invalid owned node binding") + path = f"services/diskdb-{node_id}/execution.json" return json.loads(self.command("exec", info["Id"], "cat", "/opt/crowdb/data/" + path)) def diagnostics(self): - if (self.root / "container-cluster.json").exists(): - self.refresh() - for index, name in enumerate(self.servers): - destination = self.artifacts / "runtime-logs" / f"node{index}" - destination.mkdir(parents=True, exist_ok=True) + if not (self.root / "persistent/oci-clusters" / f"{self.run_id}.json").exists(): + return + self.refresh() + for index, address in enumerate(self.addresses): + for path in ("/api/cluster/status", "/api/disk-groups", "/api/disks", "/api/servers"): + name = path.removeprefix("/api/").replace("/", "-") try: - self.read_owned("node-binding.json", index) - self.command("cp", f"{name}:/opt/crowdb/data/log", str(destination)) - except (RuntimeError, OSError) as error: - (destination / "collection.error").write_text(redact(str(error))) + value = request(f"http://{address}:9090", path) + (self.artifacts / f"{name}-{index}.json").write_text(redact(value)) + except (OSError, ValueError) as error: + (self.artifacts / f"{name}-{index}.error").write_text(redact(str(error))) + for index, name in enumerate(self.servers): + destination = self.artifacts / "runtime-logs" / f"node{index}" + destination.mkdir(parents=True, exist_ok=True) + self.inspect_owned(name) + self.command("cp", f"{name}:/opt/crowdb/data/log", str(destination)) super().diagnostics() def close(self): if self.closed: return started = time.monotonic() - if self.delete_after_replay: - self.invoke("cluster", "delete", "--container", self.servers[2]) - assert not self.root.exists() - self.closed = True - self.timings["stop"] = time.monotonic() - started - self.timings["total"] = time.monotonic() - self.started - self.write_timings() - return - if (self.root / "container-cluster.json").exists(): - state = self.state() - for kind in ("container", "network"): - listing = ("-a",) if kind == "container" else () - names = self.command(kind, "ls", *listing, "--filter", f"label={LABEL}={state['id']}", - "--filter", f"label={ROOT_LABEL}={self.root}", "--format", "{{.Names}}" if kind == "container" else "{{.Name}}") - for name in names.splitlines(): - info = json.loads(self.command("inspect", name) if kind == "container" - else self.command("network", "inspect", name))[0] - labels = info["Config"]["Labels"] if kind == "container" else info["Labels"] - if labels[LABEL] != state["id"] or labels[ROOT_LABEL] != str(self.root): - raise AssertionError("CLI cleanup ownership changed") - self.command(kind, "rm", *(("-f", name) if kind == "container" else (name,))) + # Cleanup is an independent host operation, never a product CLI command. + deletion = ["--delete-data"] if self.timings["status"] == "pass" else [] + result = subprocess.run(["python", str(ROOT / "container/oci-image/docker-local.py"), + "clean", "--cluster", self.run_id, *deletion], env=self.environment(), + capture_output=True, text=True, timeout=180) + if result.returncode: + raise RuntimeError(f"CLI fixture owned cleanup failed: {result.stdout}\n{result.stderr}") self.closed = True self.timings["stop"] = time.monotonic() - started - self.timings["total"] = time.monotonic() - self.started + if self.started is not None: + self.timings["total"] = time.monotonic() - self.started self.write_timings() diff --git a/e2e-test/console_cli_operations.py b/e2e-test/console_cli_operations.py new file mode 100644 index 000000000..3933cc0f9 --- /dev/null +++ b/e2e-test/console_cli_operations.py @@ -0,0 +1,64 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Retained CLI metadata and data semantics against normal admitted nodes.""" + +import json + + +def observation(cluster): + obsolete = cluster.root / "obsolete-console.toml" + obsolete.write_text("invalid legacy configuration") + cluster.extra_environment = {"CROWDB_CLI_STATE": str(obsolete)} + for index in range(3): + endpoint = cluster.endpoint(index) + status = json.loads(cluster.invoke("cluster", "status", "--json", endpoint=endpoint)) + assert status["phase"] == "ready" and status["observation"]["cluster_id"] == cluster.cluster_id + for domain in ("rack", "node"): + assert "1" in cluster.invoke("cluster", domain, "list", endpoint=endpoint) + assert "store 0" in cluster.invoke("cluster", "topology", "--node", "1", endpoint=endpoint) + assert obsolete.read_text() == "invalid legacy configuration" + + +def hardware(cluster): + cluster.invoke("cluster", "rack", "add", "--id", "71", "--name", "cli-rack") + assert "cli-rack" in cluster.invoke("cluster", "rack", "list", endpoint=cluster.endpoint(1)) + cluster.invoke("cluster", "node", "add", "--id", "701", "--rack", "71", + "--host", "192.0.2.71", "--ssh-credential-ref", "fixture-reference") + assert "192.0.2.71" in cluster.invoke("cluster", "node", "list", endpoint=cluster.endpoint(2)) + cluster.invoke("cluster", "node", "remove", "--id", "701", endpoint=cluster.endpoint(1)) + assert "192.0.2.71" not in cluster.invoke("cluster", "node", "list") + cluster.invoke("cluster", "rack", "remove", "--id", "71") + assert "cli-rack" not in cluster.invoke("cluster", "rack", "list") + + +def logical(cluster): + cluster.invoke("kv", "store", "add", "--id", "9", "--nodes", "1") + stores = cluster.invoke("kv", "store", "list", endpoint=cluster.endpoint(1)) + assert any(line.split()[0] == "9" for line in stores.splitlines() if line.split()), stores + for group in (90, 91): + cluster.invoke("kv", "group", "add", "--store", "9", "--group", str(group), + "--replica-id", str(group * 10), "--nodes", "1") + groups = cluster.invoke("kv", "group", "list", "--store", "9", endpoint=cluster.endpoint(2)) + assert "90" in groups and "91" in groups, groups + cluster.invoke("kv", "data", "put", "--store", "9", "--group", "90", "--key", "witness", "--value", "owned") + assert "owned" in cluster.invoke("kv", "data", "get", "--store", "9", "--group", "90", "--key", "witness") + cluster.invoke("kv", "group", "remove", "--store", "9", "--group", "91") + groups = cluster.invoke("kv", "group", "list", "--store", "9") + assert "90" in groups and "91" not in groups, groups + cluster.invoke("kv", "store", "remove", "--id", "9") + stores = cluster.invoke("kv", "store", "list") + assert not any(line.split()[0] == "9" for line in stores.splitlines() if line.split()), stores + + +def data(cluster): + prefix = "cli-data/" + cluster.run_id + for suffix in ("a", "b"): + cluster.invoke("kv", "data", "put", "--store", "0", "--group", "0", + "--key", prefix + suffix, "--value", suffix) + scanned = cluster.invoke("kv", "data", "scan", "--store", "0", "--group", "0", + "--prefix", prefix, endpoint=cluster.endpoint(2)) + assert prefix + "a" in scanned and prefix + "b" in scanned, scanned + for suffix in ("a", "b"): + cluster.invoke("kv", "data", "delete", "--store", "0", "--group", "0", "--key", prefix + suffix) + assert "not found" in cluster.invoke("kv", "data", "get", "--store", "0", "--group", "0", + "--key", prefix + suffix, endpoint=cluster.endpoint(1)) diff --git a/e2e-test/console_cli_recovery.py b/e2e-test/console_cli_recovery.py new file mode 100644 index 000000000..0d1b35c07 --- /dev/null +++ b/e2e-test/console_cli_recovery.py @@ -0,0 +1,143 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Sustained physical outage with current Access recovery and retained S3 reads.""" + +import json +import subprocess +import time + +from fixture import request + + +def signal_diskio(cluster, identity, pid, signal): + expected = "/opt/crowdb/bin/crowdb-diskio" + check = f'test "$(readlink /proc/{pid}/exe)" = {expected} && kill -{signal} {pid}' + if signal == "CONT": + check = f"if test -e /proc/{pid}/exe; then {check}; fi" + cluster.command("exec", "--user", "crowdb", identity, "sh", "-c", check) + + +def health_snapshot(cluster, label): + observed = [] + for index, name in enumerate(cluster.servers): + identity = cluster.inspect_owned(name)["Id"] + url = f"http://{cluster.addresses[index]}:9094/_crowdb/health/ready" + try: + value = cluster.command("exec", identity, "curl", "--silent", "--show-error", + "--max-time", "3", url) + observed.append(json.loads(value)) + except (RuntimeError, ValueError) as error: + observed.append({"error": str(error)}) + (cluster.artifacts / f"access-health-{label}.json").write_text(json.dumps(observed, indent=2)) + return observed + + +def pause_and_write(cluster, pauses): + with (cluster.artifacts / "fault-s3.log").open("w") as output: + clients = [] + try: + for identity, pid in pauses: + signal_diskio(cluster, identity, pid, "STOP") + status = cluster.command("exec", identity, "cat", f"/proc/{pid}/status") + assert "State:\tT" in status, status + print("Paused three verified DiskIO processes across the 30-second I/O deadline", flush=True) + for number in range(3): + clients.append(subprocess.Popen(cluster.arguments("s3", "object", "put", + "writer-recovery", f"during-{number}", "--text", "during-fault"), + env=cluster.environment(), stdout=output, stderr=output)) + time.sleep(40) + unavailable = health_snapshot(cluster, "paused") + assert all(value.get("ready") is False for value in unavailable), unavailable + assert all(value.get("observation", {}).get("state") == "current" + for value in unavailable), unavailable + finally: + for identity, pid in pauses: + signal_diskio(cluster, identity, pid, "CONT") + for client in clients: + try: + client.wait(timeout=60) + except subprocess.TimeoutExpired: + client.kill() + client.wait() + + +def managed_processes(cluster): + processes = {} + for index, name in enumerate(cluster.servers): + identity = cluster.inspect_owned(name)["Id"] + node = cluster.read_owned("node-binding.json", index)["node_id"] + status = json.loads(cluster.command("exec", identity, "cat", + "/opt/crowdb/run/status/monitor.json")) + for kind in ("chunk-kv", "access"): + service = f"{kind}-{node}" + current = status["services"][service] + assert type(current.get("pid")) is int and current["pid"] > 1, current + processes[service] = {key: current[key] for key in ("pid", "generation")} + return processes + + +def terminal_writer_recovery(cluster, cluster_id): + origin = f"http://{cluster.addresses[0]}:9090" + inputs = cluster.state() + identities = [cluster.inspect_owned(name)["Id"] for name in cluster.servers] + cluster.invoke("s3", "bucket", "put", "writer-recovery") + cluster.invoke("s3", "object", "put", "writer-recovery", "before", + "--text", "acknowledged-before-fault") + before = request(origin, "/api/servers") + processes_before = managed_processes(cluster) + services = [service for service in before if service.get("service_type") == "diskio"] + assert len(services) == 3, services + pauses = [] + for service in services: + index = next(index for index in range(3) + if cluster.read_owned("node-binding.json", index)["node_id"] == service["node_id"]) + pid = service["pid"] + assert type(pid) is int and pid > 1, service + pauses.append((identities[index], pid)) + (cluster.artifacts / "fault-injection.json").write_text(json.dumps({ + "diskio_processes": pauses, "pause_seconds": 40}, indent=2)) + pause_and_write(cluster, pauses) + # No application traffic until independent health observation reports recovery. + started = time.monotonic() + try: + report = json.loads(cluster.invoke("cluster", "check", "--wait", "180", "--json")) + finally: + health = health_snapshot(cluster, "checked") + assert report["phase"] == "ready" and report["observation"]["cluster_id"] == cluster_id, report + recovery_seconds = time.monotonic() - started + assert all(value.get("ready") is True for value in health), health + assert all(value.get("observation", {}).get("state") == "current" for value in health), health + assert inputs == cluster.state(), "fault recovery changed accepted launch inputs" + assert identities == [cluster.inspect_owned(name)["Id"] for name in cluster.servers] + processes_after = managed_processes(cluster) + (cluster.artifacts / "process-recovery.json").write_text(json.dumps({ + "before": processes_before, "after": processes_after}, indent=2)) + old = {key: value["pid"] for key, value in processes_before.items() if key.startswith("chunk-kv-")} + new = {key: value["pid"] for key, value in processes_after.items() if key.startswith("chunk-kv-")} + restarted = [service for service in old if old[service] != new.get(service)] + assert restarted, (old, new) + access = lambda rows: {key: value for key, value in rows.items() if key.startswith("access-")} + assert len(access(processes_before)) == 3 and access(processes_before) == access(processes_after), ( + "healthy Access processes restarted", processes_before, processes_after) + recovery_logs = [] + for index, name in enumerate(cluster.servers): + node = cluster.read_owned("node-binding.json", index)["node_id"] + log = cluster.command("exec", identities[index], "sh", "-c", + f"cat /opt/crowdb/data/log/chunk-kv-{node}/crowdb-chunk-kv-server-[0-9]*.log") + (cluster.artifacts / f"chunk-session-{node}.log").write_text(log) + recovery_logs.append(log) + assert any("writer outcome requires durable replay; stopping for supervisor restart" in log + for log in recovery_logs), "fault did not exercise terminal writer replay" + cluster.invoke("s3", "object", "put", "writer-recovery", "after", + "--text", "acknowledged-after-fault") + for address in cluster.addresses: + for key in ("before", "after"): + value = cluster.invoke("s3", "object", "get", "writer-recovery", key, + endpoint=f"http://{address}:9090") + assert value == f"acknowledged-{key}-fault", value + (cluster.artifacts / "recovery-proof.json").write_text(json.dumps({ + "cluster_id": cluster_id, "old_pids": old, "new_pids": new, "restarted": restarted, + "access_processes": access(processes_after), "recovery_seconds": recovery_seconds, + "retained_reads": 3, "new_reads": 3}, indent=2)) + cluster.current_readiness(cluster_id) + print("PASS automatic durable replay, cached Access recovery and operator stop fencing", restarted, flush=True) diff --git a/e2e-test/console_cli_target.py b/e2e-test/console_cli_target.py new file mode 100644 index 000000000..4f24af1a5 --- /dev/null +++ b/e2e-test/console_cli_target.py @@ -0,0 +1,117 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Attach a host CLI to ordinary Docker nodes without a launch receipt.""" + +import json +from pathlib import Path + +from allocation import create_network, node_data, start_node +from console_cli_client import CliClient +import console_cli_operations +from fixture import Cluster, LABEL, request + + +class CliAttachedCluster(CliClient, Cluster): + def __init__(self, image, artifacts, bundle, **kwargs): + super().__init__(image, artifacts, node_count=3, wire_replicas=False, + advertise=True, **kwargs) + self.bundle = Path(bundle).resolve() + self.root = (self.data_root / "client-runtime").resolve() + self.phase_number = 0 + + def start(self): + subnet = create_network(self) + host = json.loads(self.command("info", "--format", "{{json .}}"))["ID"] + if not isinstance(host, str) or not host: + raise AssertionError("Docker physical host identity is absent") + peers = [str(subnet.network_address + index + 2) for index in range(3)] + for index, address in enumerate(peers): + data = node_data(self, index, prepare_logs=False) + preparer = f"{self.run_id}-prepare{index}" + self.containers.append(preparer) + self.command("run", "--rm", "--name", preparer, "--network", "none", "--label", f"{LABEL}={self.run_id}", + "--cpu-shares", "1024", "--memory", "64m", "--pids-limit", "32", + "--mount", f"type=bind,source={data},target=/opt/crowdb/data", + "--entrypoint", "/bin/chown", self.image, + "-R", "crowdb:crowdb", "/opt/crowdb/data") + self.containers.remove(preparer) + self.addresses.append(start_node(self, index, data, (), address=address, + memory="2048m", pids_limit=512, environment=( + f"CROWDB_CLUSTER={self.run_id}", "CROWDB_INITIAL_VOTERS=3", + "CROWDB_DEPLOYMENT_MODE=local", + f"CROWDB_PHYSICAL_HOST_ID={host}", + f"CROWDB_S3_PUBLIC_URI=http://{address}:9091", + f"CROWDB_ICEBERG_PUBLIC_URI=http://{address}:9092", + f"CROWDB_DATASET_PUBLIC_URI=http://{address}:9093", + "CROWDB_DISCOVERY_SEEDS=" + ",".join(f"http://{peer}:9095" for peer in peers)))) + self.origins = [f"http://{address}:10000" for address in self.addresses] + for name in self.servers: + if not json.loads(self.command("inspect", name))[0]["State"]["Running"]: + raise AssertionError("plain Docker node exited: " + self.command("logs", name)) + report = json.loads(self.invoke("cluster", "check", "--wait", "180", "--json", + endpoint=self.endpoint(0))) + assert report["phase"] == "ready", report + self.cluster_id = report["observation"]["cluster_id"] + binding = self.root / "persistent/clusters" / self.run_id / "target.json" + binding.parent.mkdir(parents=True, mode=0o700) + binding.touch(mode=0o600) + binding.write_text(json.dumps({"version": 1, "name": self.run_id, + "cluster_id": self.cluster_id, "management_endpoints": [self.endpoint(0)], "runtime": None})) + return self + + def endpoint(self, index): + return f"http://{self.addresses[index]}:9090" + + def inventory(self): + containers = [json.loads(self.command("inspect", name))[0] for name in self.servers] + assert all(item["Config"]["Labels"].get(LABEL) == self.run_id for item in containers) + services = request(self.endpoint(0), "/api/servers") + return {"containers": [(item["Id"], item["Image"]) for item in containers], + "processes": sorted((item["id"], item["pid"]) for item in services)} + + def replay(self, checkpoint): + if checkpoint != "attachment": + before = self.inventory() + operation = {"observation": console_cli_operations.observation, + "hardware": console_cli_operations.hardware, + "logical": console_cli_operations.logical, + "data": console_cli_operations.data}[checkpoint] + operation(self) + assert self.inventory() == before, "logical CLI operation changed runtime lifecycle" + report = json.loads(self.invoke("cluster", "check", "--json")) + assert report["phase"] == "ready" and report["observation"]["cluster_id"] == self.cluster_id + (self.artifacts / "operation-proof.json").write_text(json.dumps({ + "cluster_id": self.cluster_id, "operation": checkpoint, "inventory": before}, indent=2)) + print("PASS selected-cluster CLI", checkpoint, flush=True) + return + self.attachment() + + def attachment(self): + before = self.inventory() + for domain in ("rack", "node", "disk-group", "disk"): + assert self.invoke("cluster", domain, "list", endpoint=self.endpoint(1)).strip() + assert self.invoke("kv", "store", "list").strip() + assert self.invoke("kv", "group", "list", "--store", "0").strip() + key, value = "cli-attach/" + self.run_id, "verified-selected-cluster" + self.invoke("kv", "data", "put", + "--store", "0", "--group", "0", "--key", key, "--value", value) + for index in range(3): + found = self.invoke("kv", "data", "get", "--store", "0", "--group", "0", "--key", key, + endpoint=self.endpoint(index)) + assert value in found, found + self.invoke("kv", "data", "delete", "--store", "0", "--group", "0", "--key", key) + assert "not found" in self.invoke("kv", "data", "get", "--store", "0", "--group", "0", "--key", key) + self.invoke("s3", "bucket", "put", "cli-attach", endpoint=self.endpoint(2)) + self.invoke("s3", "object", "put", "cli-attach", "witness", "--text", value) + for index in range(3): + assert self.invoke("s3", "object", "get", "cli-attach", "witness", + endpoint=self.endpoint(index)) == value + self.invoke("cluster", "stop", endpoint=self.endpoint(0), expected_exit=2) + report = json.loads(self.invoke("cluster", "check", "--json", endpoint=self.endpoint(2))) + assert report["phase"] == "ready" and report["observation"]["cluster_id"] == self.cluster_id + assert self.inventory() == before, "attachment changed container or service lifecycle" + assert not (self.root / "persistent/oci-clusters").exists(), "attachment created a runtime receipt" + (self.artifacts / "attachment-proof.json").write_text(json.dumps({ + "cluster_id": self.cluster_id, "inventory": before, + "kv_reads": 3, "s3_reads": 3, "runtime_receipt": False}, indent=2)) + print("PASS named and endpoint-only attachment to plain Docker nodes", flush=True) diff --git a/e2e-test/console_discovery_browser_fixture.py b/e2e-test/console_discovery_browser_fixture.py index e53997df5..c55b35bd7 100644 --- a/e2e-test/console_discovery_browser_fixture.py +++ b/e2e-test/console_discovery_browser_fixture.py @@ -27,7 +27,7 @@ def start(self): "--ui-root", "/opt/crowdb/ui", "--log-dir", "/opt/crowdb/data/log/web", "--node-monitor", monitor), address=provider, environment=("CROWDB_RUNTIME_ROOT=/opt/crowdb/data",))) - data = node_data(self, 1) + data = node_data(self, 1, prepare_logs=False) preparation = self.run_id + "-provider-owner" self.containers.append(preparation) self.command("run", "--rm", "--name", preparation, "--label", f"{LABEL}={self.run_id}", @@ -36,7 +36,9 @@ def start(self): "--mount", f"type=bind,source={data},target=/opt/crowdb/data", self.image, "-R", "crowdb:crowdb", "/opt/crowdb/data") self.addresses.append(start_node(self, 1, data, (), address=provider, - environment=("CROWDB_STARTUP_MODE=manual", f"CROWDB_PHYSICAL_HOST_ID={host}"), + environment=("CROWDB_DEPLOYMENT_MODE=production", "CROWDB_INITIAL_VOTERS=3", + "CROWDB_SSH_PASSWORD=crowdb", f"CROWDB_CLUSTER={self.run_id}", + f"CROWDB_PHYSICAL_HOST_ID={host}"), network_container=self.servers[0])) wait(lambda: HTTP.open(f"http://{self.addresses[0]}:9091/healthz", timeout=5).read(), lambda body: body == b"ok", "independent release Web", seconds=20) diff --git a/e2e-test/console_fixture.py b/e2e-test/console_fixture.py index 984d5f124..a4157108e 100644 --- a/e2e-test/console_fixture.py +++ b/e2e-test/console_fixture.py @@ -10,8 +10,7 @@ import time from control import Control -from managed_fixture import ManagedCluster, SOURCE -from allocation import start_node +from managed_fixture import ManagedCluster SERVICES = ("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web") @@ -21,20 +20,6 @@ def __init__(self, image, artifacts, bundle, **kwargs): super().__init__(image, artifacts, services=SERVICES, **kwargs) self.bundle = bundle - def profile(self): - # Exercise the supported managed Docker mode from the shipped web config. - return super().profile().replace( - ", \"--node-monitor\", \"http://127.0.0.1:9095\"", "") - - def launch(self, data, profile): - template = self.artifacts / "crowdb-web.toml" - template.write_text((SOURCE / "templates/crowdb-web.toml").read_text().replace( - "readonly = true", "readonly = false")) - return start_node(self, 0, data, (), address=self.address, memory="2048m", - environment=("CROWDB_STARTUP_MODE=single", "CROWDB_PROFILE_PATH=/opt/crowdb/etc/e2e-profile.toml"), - mounts=(f"type=bind,source={profile},target=/opt/crowdb/etc/e2e-profile.toml,readonly", - f"type=bind,source={template},target=/opt/crowdb/etc/templates/crowdb-web.toml,readonly")) - def observation(self): return json.loads(self.command("exec", self.servers[0], "cat", "/opt/crowdb/run/status/monitor.json")) diff --git a/e2e-test/console_native_fixture.py b/e2e-test/console_native_fixture.py index eecc6ea7c..c768da8b3 100644 --- a/e2e-test/console_native_fixture.py +++ b/e2e-test/console_native_fixture.py @@ -1,6 +1,6 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Normal manual release nodes with public SSH admission and fixed ports.""" +"""Production-policy release nodes with explicit admission and fixed ports.""" import json import urllib.request @@ -16,6 +16,7 @@ from console_native_candidates import prepare as prepare_candidates, verify as verify_candidates from profiles.setup_choices import diskless_setup from profiles.node_resources import capture as capture_resources +from console_native_policy import accept as accept_policy class NativeConsoleCluster(ShellBrowserCluster): @@ -65,7 +66,8 @@ def __init__(self, image, artifacts, bundle, *, disk_gib=(80, 8, 8), def api(self, path, body=None, method=None, status=200): payload = None if body is None else json.dumps(body).encode() - request = urllib.request.Request(f"http://{self.addresses[0]}:9090" + path, + origin = getattr(self, "management_address", self.addresses[0]) + request = urllib.request.Request(f"http://{origin}:9090" + path, data=payload, method=method, headers={"Content-Type": "application/json"}) try: with HTTP.open(request, timeout=20) as response: @@ -82,7 +84,8 @@ def start(self): if not isinstance(host, str) or not host: raise AssertionError("Docker physical host identity is absent") for index in range(self.node_count): - data = node_data(self, index) + data = node_data(self, index, prepare_logs=False) + environment, mounts = accept_policy(self, index, data, host) disks = data / "disks" disks.mkdir() disk = disks / f"disk-{index + 1}.img" @@ -103,17 +106,26 @@ def start(self): "-R", "crowdb:crowdb", "/opt/crowdb/data") self.addresses.append(start_node(self, index, data, (), memory="2048m", pids_limit=512, address=str(subnet.network_address + index + 2), - environment=("CROWDB_STARTUP_MODE=manual", f"CROWDB_PHYSICAL_HOST_ID={host}", + mounts=mounts, environment=(*environment, "CROWDB_DISK_INPUTS=" + json.dumps(self.candidate_inputs if index == 0 else [])))) self.ready() for address in self.addresses: with HTTP.open(f"http://{address}:9090/api/node/auto-join", timeout=5) as response: - assert json.load(response)["enabled"] is True + assert json.load(response)["enabled"] is False request = urllib.request.Request(f"http://{address}:9090/api/node/auto-join", data=json.dumps({"enabled": False}).encode(), headers={"Content-Type": "application/json"}) with HTTP.open(request, timeout=5) as response: assert json.load(response)["enabled"] is False + if self.auto_bootstrap: + discovered = wait(lambda: self.api("/api/node/candidates"), + lambda result: len(result["candidates"]) == self.node_count, + "complete automatic formation", seconds=10)["candidates"] + coordinator = min(discovered, key=lambda entry: entry["advertisement"]["discovery_id"]) + self.management_address = next( + urlparse(endpoint).hostname + for endpoint in coordinator["advertisement"]["monitor_endpoints"] + if urlparse(endpoint).hostname in self.addresses) for rack in sorted(set(self.rack_ids)): self.api("/api/racks", {"id": rack, "name": "native acceptance"}, status=201) if self.auto_bootstrap: @@ -220,9 +232,9 @@ def control(self, body): self.browser("native diagnostics: production split displays actual inherited and current journals", config="native.config.ts", extra_env={"CROWDB_NATIVE_TRANSITION_ACCEPTANCE": "1"}) return {"passed": True} - if self.case_name == "console::native_browser::large_journal_pagination_and_restart": + if self.case_name == "console::native_browser::large_journal_records_and_restart": self.browser_timeout = 180 - self.browser("native diagnostics: large Journal replaces 100 extent fences with its remainder", + self.browser("native diagnostics: large Journal exposes a bounded tail chunk record across restart", config="native.config.ts", extra_env={"CROWDB_NATIVE_JOURNAL_WINDOWS": "1"}) return {"passed": True} if self.case_name == "console::native_browser::auxiliary_process_menus": diff --git a/e2e-test/console_native_policy.py b/e2e-test/console_native_policy.py new file mode 100644 index 000000000..d0ed9e1ac --- /dev/null +++ b/e2e-test/console_native_policy.py @@ -0,0 +1,27 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Accept the owned diskless policy before attaching unadmitted fixture media.""" + +from allocation import LABEL + + +def accept(cluster, index, data, physical_host): + policy = cluster.artifacts / f"node-policy-{index}.toml" + kinds = ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server") + services = ", ".join(f'{{ kind = "{kind}", enabled = false }}' for kind in kinds) + policy.write_text(f'services = [{services}]\n[storage]\nmode = "diskless"\n') + environment = ("CROWDB_DEPLOYMENT_MODE=production", "CROWDB_INITIAL_VOTERS=3", + f"CROWDB_CLUSTER={cluster.run_id}", "CROWDB_SSH_PASSWORD=crowdb", + f"CROWDB_PHYSICAL_HOST_ID={physical_host}", + "CROWDB_NODE_POLICY_FILE=/run/native-policy.toml") + mount = f"type=bind,source={policy},target=/run/native-policy.toml,readonly" + name = f"{cluster.run_id}-policy{index}" + cluster.containers.append(name) + output = cluster.command("run", "--rm", "--name", name, + "--label", f"{LABEL}={cluster.run_id}", "--network", "none", + "--cpu-shares", "1024", "--memory", "128m", "--pids-limit", "32", + "--mount", f"type=bind,source={data},target=/opt/crowdb/data", "--mount", mount, + *(part for value in environment for part in ("--env", value)), + "--entrypoint", "/opt/crowdb/bin/crowdb-monitor", cluster.image, "accept-policy") + assert output.strip() == "production", output + return environment, (mount,) diff --git a/e2e-test/console_native_storage.py b/e2e-test/console_native_storage.py index 2dddab124..74bfdbe93 100644 --- a/e2e-test/console_native_storage.py +++ b/e2e-test/console_native_storage.py @@ -3,6 +3,7 @@ """Private monitor file preparation on owned standard release nodes.""" import json +import time import uuid from fixture import wait @@ -73,6 +74,16 @@ def file_operation(cluster, node, operation_id): def verify_file_admission(cluster): """Real queued creation, monitor restart, DiskIO read and protected membership.""" node = cluster.node_ids[0] + # New files remain Init until DiskIO verifies them; only existing media + # must be loaded by DiskDB before that consumer can start. + expected = {} + for member in cluster.node_ids: + for disk_group in cluster.api(f"/api/nodes/{member}/disk-groups"): + group_id = disk_group["id"] + count = len(cluster.api(f"/api/nodes/{member}/disk-groups/{group_id}/disks")) + if count: + expected[group_id] = count + group = 592 cluster.api(f"/api/nodes/{node}/disk-groups", { "id": group, "name": "managed-file-acceptance", "store_id": 1, "group_id": 1}, status=201) @@ -97,15 +108,28 @@ def verify_file_admission(cluster): raise AssertionError("legacy deletion bypassed retained disk lifecycle") cluster.command("restart", cluster.servers[0]) cluster.ready() + loaded_after_ms = time.time_ns() // 1_000_000 for request in requests: resumed = wait(lambda: file_operation(cluster, node, request["operation_id"]), lambda result: result["phase"] == "waiting-for-diskio", "monitor resumes original file admission", seconds=10) assert resumed["request"]["operation_id"] == request["operation_id"], resumed page = cluster.api(f"/api/nodes/{node}/disk-files?limit=32") assert {request["operation_id"] for request in requests} <= {row["request"]["operation_id"] for row in page["items"]}, page + + def loaded(instances): + for group_id, count in expected.items(): + owners = [instance for instance in instances if group_id in instance["owned_dg_ids"]] + if len(owners) != 1 or owners[0]["last_heartbeat_ms"] <= loaded_after_ms or not any(usage["disk_group_id"] == group_id + and usage["usage_known"] and usage["disk_count"] >= count + and usage["sampled_at_ms"] > loaded_after_ms + for usage in owners[0]["group_usages"]): + return False + return True + wait(lambda: cluster.api("/api/diskdb/instances"), loaded, + "DiskDB loads every registered file before DiskIO", seconds=10) deployment = cluster.api(f"/api/deployment-defaults?node_id={node}")["diskio"] deployment.update({"kind": "diskio", "instance_id": str(node), "rpc_port": 7200, - "disk_group_id": 0, "test_single_node": False}) + "disk_group_id": 0}) cluster.api(f"/api/nodes/{node}/services/deploy", deployment, status=201) completed = [wait(lambda request=request: file_operation(cluster, node, request["operation_id"]), lambda result: result["phase"] == "registered", "DiskIO-verified file admission", seconds=10) for request in requests] diff --git a/e2e-test/containerd_profile.py b/e2e-test/containerd_profile.py index 6f22f4800..b248e5945 100644 --- a/e2e-test/containerd_profile.py +++ b/e2e-test/containerd_profile.py @@ -12,7 +12,7 @@ import time import uuid from fixture import docker, redact -from client import bundle_client +from native_containerd_profile import client_bundle from image_identity import verify_server_sources ROOT = Path(__file__).resolve().parents[1] @@ -41,15 +41,13 @@ def cleanup(prefix): def run_profile(args, parser): - cluster = args.profile == "containerd-cluster" - needed_cpu = 2 if cluster else 1.5 - needed_memory = 5120 if cluster else 2048 + needed_cpu, needed_memory = 3.5, 8192 if args.layer != "all" or args.scope != "all" or args.case or args.require_isolation: parser.error("containerd cluster is an explicit deployment profile; domain selectors do not apply") - if args.node_cpus <= 0 or args.concurrency != 1 or args.cpus < needed_cpu or args.memory_mib < needed_memory: - parser.error(f"{args.profile} requires --concurrency 1, {needed_cpu:g} CPU and {needed_memory} MiB including Linux hosts") + if args.node_cpus != 1 or args.concurrency != 1 or args.cpus < needed_cpu or args.memory_mib < needed_memory: + parser.error(f"{args.profile} requires --node-cpus 1, --concurrency 1, {needed_cpu:g} CPU and {needed_memory} MiB including Linux hosts") if args.list: - print(json.dumps([{"scope":"deployment.containerd", "case":"three_host_quorum_and_recovery" if cluster else "manual_and_single_host_recovery", "nodes":3 if cluster else 1}])) + print(json.dumps([{"scope":"deployment.containerd", "case":"three_host_quorum_and_recovery", "nodes":3}])) return info = json.loads(docker("info", "--format", "{{json .}}")) if info["OSType"] != "linux" or info["NCPU"] < needed_cpu or info["MemTotal"] < needed_memory * 1048576: @@ -69,27 +67,16 @@ def run_profile(args, parser): (run_root / "identity.json").write_text(json.dumps({"profile":args.profile, "artifact":verified, "sources":source,"concurrency":1,"cpus":needed_cpu,"memory_mib":needed_memory, "node_cpus":args.node_cpus,"host_data":str(data)}, indent=2)) - client = None - if not cluster: - staged = ROOT / "target/container-runtime" - if (staged / "SOURCE_REVISION").read_text().strip() != verified["revision"]: - raise RuntimeError("storage SDK probe requires the exact OCI dependency stage") - client = bundle_client(run_root / "media-client") - for library in (ROOT / "target/container-runtime/lib").iterdir(): - destination = client / "lib" / library.name - if library.is_file() and not destination.exists(): - shutil.copy2(library, destination) - env = os.environ.copy() - if client: - env["CROWDB_CONTAINERD_MEDIA_CLIENT"] = str(client) - + client = client_bundle(run_root / 'host-client') + env = dict(os.environ, CROWDB_E2E_CLI_BUNDLE=str(client), + CROWDB_CONTAINERD_CLUSTER='oci-' + run_root.name[-20:]) env.update(CROWDB_CONTAINERD_DATA=str(data), CROWDB_CONTAINERD_PREFIX=prefix, CROWDB_CONTAINERD_ARCHIVE=str(archive), CROWDB_CONTAINERD_HOST_IMAGE=verified["base_image"], CROWDB_CONTAINERD_NODE_CPUS=str(args.node_cpus)) started = time.monotonic() status, code = "failed", None print(f"Containerd {args.profile} profile: {verified['image_digest']}; artifacts {run_root}", flush=True) - script = "containerd-cluster.py" if cluster else "containerd-single-host.py" + script = "containerd-cluster.py" child = subprocess.Popen([sys.executable,str(ROOT / "e2e-test/profiles" / script)], cwd=ROOT,env=env,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) try: diff --git a/e2e-test/deployment_anonymous_fixture.py b/e2e-test/deployment_anonymous_fixture.py index 5a72464ad..8e3a92a18 100644 --- a/e2e-test/deployment_anonymous_fixture.py +++ b/e2e-test/deployment_anonymous_fixture.py @@ -5,7 +5,7 @@ import json import time from allocation import create_network, start_node -from fixture import LABEL, HTTP +from fixture import LABEL, HTTP, request from managed_fixture import ManagedCluster @@ -14,7 +14,8 @@ def start(self): subnet = create_network(self) address = str(subnet.network_address + 2) self.addresses.append(start_node(self, 0, None, (), address=address, - memory="2048m", environment=("CROWDB_STARTUP_MODE=single",))) + memory="2048m", environment=("CROWDB_DEPLOYMENT_MODE=local", "CROWDB_INITIAL_VOTERS=1", + f"CROWDB_CLUSTER={self.run_id}"))) deadline = time.monotonic() + 240 while time.monotonic() < deadline: state = json.loads(self.command("inspect", self.servers[0]))[0]["State"] @@ -24,16 +25,19 @@ def start(self): self.command("exec", self.servers[0], "/opt/crowdb/bin/crowdb-monitor", "readiness") if HTTP.open(f"http://{address}:9090/healthz", timeout=5).read() != b"ok": raise RuntimeError("default release Web health response is invalid") + if request(f"http://{address}:9090", "/api/cluster/status")["phase"] != "ready": + raise RuntimeError("default node application is not ready") break except (RuntimeError, OSError): time.sleep(0.1) else: raise AssertionError("default anonymous-volume startup never became ready") info = json.loads(self.command("inspect", self.servers[0]))[0] - manifest = json.loads(self.command("exec", self.servers[0], "cat", - "/opt/crowdb/data/bootstrap/manifest.json")) + accepted = json.loads(self.command("exec", self.servers[0], "cat", + "/opt/crowdb/data/accepted-node.json")) + status = request(f"http://{address}:9090", "/api/cluster/status") (self.artifacts / "anonymous-runtime.json").write_text(json.dumps({ - "mounts": info["Mounts"], "manifest": manifest, + "mounts": info["Mounts"], "accepted": accepted, "status": status, "log": self.command("logs", self.servers[0])}, indent=2)) return self diff --git a/e2e-test/deployment_bootstrap_fixture.py b/e2e-test/deployment_bootstrap_fixture.py index 360286f07..c2c373d13 100644 --- a/e2e-test/deployment_bootstrap_fixture.py +++ b/e2e-test/deployment_bootstrap_fixture.py @@ -1,6 +1,6 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Interrupt the shipped monitor after its first durable completed boot step.""" +"""Interrupt the shipped monitor after its first durable completed setup step.""" import json import time @@ -17,17 +17,19 @@ def before_ready(self, data, profile): raise AssertionError("release container exited before bootstrap interruption") try: before = json.loads(self.command("exec", self.servers[0], "cat", - "/opt/crowdb/data/bootstrap/manifest.json")) + "/opt/crowdb/data/node-setup.json")) except RuntimeError as error: if "No such file" not in str(error): raise - if before and before["state"] == "initializing" and any( - step["complete"] for step in before["steps"]): + if before and before["status"]["phase"] == "running" and any( + step["phase"] == "ready" and step.get("operation_id") for step in before["status"]["services"]): + accepted = json.loads(self.command("exec", self.servers[0], "cat", + "/opt/crowdb/data/accepted-node.json")) break time.sleep(0.1) else: - raise AssertionError("bootstrap did not expose a completed step before readiness") - (self.artifacts / "interrupted-manifest.json").write_text(json.dumps(before, indent=2)) + raise AssertionError("setup did not expose a completed step before readiness") + (self.artifacts / "interrupted-setup.json").write_text(json.dumps({"setup": before, "accepted": accepted}, indent=2)) self.command("kill", "--signal", "KILL", self.servers[0]) exited = json.loads(self.command("inspect", self.servers[0]))[0]["State"] if exited["ExitCode"] == 0: @@ -43,5 +45,8 @@ def before_ready(self, data, profile): def ready(self): super().ready() body = self.command("exec", self.servers[0], "cat", - "/opt/crowdb/data/bootstrap/manifest.json") - (self.artifacts / "recovered-manifest.json").write_text(body) + "/opt/crowdb/data/node-setup.json") + accepted = self.command("exec", self.servers[0], "cat", + "/opt/crowdb/data/accepted-node.json") + (self.artifacts / "recovered-setup.json").write_text(json.dumps({ + "setup": json.loads(body), "accepted": json.loads(accepted)}, indent=2)) diff --git a/e2e-test/deployment_browser_fixture.py b/e2e-test/deployment_browser_fixture.py index 0b9ebfb37..30a0554c4 100644 --- a/e2e-test/deployment_browser_fixture.py +++ b/e2e-test/deployment_browser_fixture.py @@ -1,6 +1,6 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Pinned host Playwright checks of the normally configured readonly release UI.""" +"""Pinned host Playwright checks of the normally configured node UI.""" import json import os diff --git a/e2e-test/deployment_clients_fixture.py b/e2e-test/deployment_clients_fixture.py index ff3474ea7..5f97685a7 100644 --- a/e2e-test/deployment_clients_fixture.py +++ b/e2e-test/deployment_clients_fixture.py @@ -3,8 +3,7 @@ """Formal single deployment with an explicitly reachable ChunkDB inspector.""" import json -from allocation import start_node -from managed_fixture import ManagedCluster, SOURCE +from managed_fixture import ManagedCluster class DeploymentClientsCluster(ManagedCluster): @@ -18,7 +17,7 @@ def ready(self): status = self.monitor_status() modes = {file: self.command("exec", name, "stat", "-c", "%a", "/opt/crowdb/data/" + file).strip() - for file in ("secrets/server.env", "secrets/client.env", "crash")} + for file in ("secrets/server.env", "console/secrets/client.env", "crash")} cwd = {role: self.command("exec", "--user", "crowdb", name, "readlink", f"/proc/{pid}/cwd").strip() for role, pid in (("monitor", status["monitor_pid"]), @@ -26,12 +25,3 @@ def ready(self): info = json.loads(self.command("inspect", name))[0] (self.artifacts / "deployment-runtime.json").write_text(json.dumps({ "modes": modes, "cwd": cwd, "ports": info["HostConfig"]["PortBindings"]}, indent=2)) - - def launch(self, data, profile): - template = self.artifacts / "chunkdb-template.toml" - template.write_text((SOURCE / "templates/chunkdb.toml").read_text().replace( - 'rpc_listen_addr = "127.0.0.1:12200"', 'rpc_listen_addr = "0.0.0.0:12200"')) - return start_node(self, 0, data, (), address=self.address, memory="2048m", - environment=("CROWDB_STARTUP_MODE=single", "CROWDB_PROFILE_PATH=/opt/crowdb/etc/e2e-profile.toml"), - mounts=(f"type=bind,source={profile},target=/opt/crowdb/etc/e2e-profile.toml,readonly", - f"type=bind,source={template},target=/opt/crowdb/etc/templates/chunkdb.toml,readonly")) diff --git a/e2e-test/deployment_failure_fixture.py b/e2e-test/deployment_failure_fixture.py index 886473c14..697f195e4 100644 --- a/e2e-test/deployment_failure_fixture.py +++ b/e2e-test/deployment_failure_fixture.py @@ -22,18 +22,20 @@ def control(self, body): if info["Config"]["Labels"].get(LABEL) != self.run_id or not info["State"]["Running"]: raise ValueError("monitor container ownership or running state changed") if action == "prepare-lifecycle": - super().control({"action": "recover-service", "node": 0, - "service": "web", "signal": "STOP"}) + accepted = self.command("exec", name, "cat", "/opt/crowdb/data/accepted-node.json") self.command("stop", "--time", "15", name) self.command("rm", name) self.servers.clear() self.containers.remove(name) self.launch(self.data_paths[0], self.artifacts / "profile.toml") self.ready() + retained = self.command("exec", self.servers[0], "cat", "/opt/crowdb/data/accepted-node.json") + if json.loads(accepted) != json.loads(retained): + raise AssertionError("graceful monitor replacement changed accepted bootstrap") return {"phase": "ready"} secrets = [] - for file in ("server.env", "client.env"): - for line in self.command("exec", name, "cat", "/opt/crowdb/data/secrets/" + file).splitlines(): + for file in ("secrets/server.env", "console/secrets/client.env"): + for line in self.command("exec", name, "cat", "/opt/crowdb/data/" + file).splitlines(): key, _, value = line.partition("=") if value and any(word in key for word in ("TOKEN", "KEY")): secrets.append(value) @@ -42,18 +44,19 @@ def control(self, body): for _ in range(5): super().control({"action": "recover-service", "node": 0, "service": "web", "signal": "KILL"}) - events = self.command("exec", name, "cat", "/opt/crowdb/data/log/monitor/monitor.log") - before = sum(json.loads(line)["kind"] == "ready" for line in events.splitlines()) + before = self.monitor_status() if action == "reject-changed-credentials": self.command("exec", "--user", "root", name, "/bin/sh", "-ec", "printf 'invalid credentials\\n' > /opt/crowdb/data/secrets/server.env") - pid = self.monitor_status()["services"]["web"]["pid"] - if type(pid) is not int or pid <= 1: - raise ValueError("web PID must identify the observed monitor child") - executable = self.command("exec", name, "cat", f"/proc/{pid}/cmdline").split("\0")[0] - if Path(executable).name != "crowdb-web": - raise ValueError("observed web executable changed before monitor failure") - self.command("exec", name, "kill", "-KILL", str(pid)) + self.command("restart", name) + else: + pid = self.monitor_status()["services"]["web"]["pid"] + if type(pid) is not int or pid <= 1: + raise ValueError("web PID must identify the observed monitor child") + executable = self.command("exec", name, "cat", f"/proc/{pid}/cmdline").split("\0")[0] + if Path(executable).name != "crowdb-web": + raise ValueError("observed web executable changed before monitor failure") + self.command("exec", name, "kill", "-KILL", str(pid)) deadline = time.monotonic() + 40 while time.monotonic() < deadline: state = json.loads(self.command("inspect", name))[0]["State"] @@ -64,9 +67,10 @@ def control(self, body): def failure_result(self, action, state, before): log = self.command("logs", self.servers[0]) - events = self.data_paths[0] / "log/monitor/monitor.log" - after = sum(json.loads(line)["kind"] == "ready" for line in events.read_text().splitlines()) - kinds = set() + status_path = self.artifacts / "failed-monitor-status.json" + self.command("cp", f"{self.servers[0]}:/opt/crowdb/run/status/monitor.json", str(status_path)) + after = json.loads(status_path.read_text()) + credential_rejection = "server credentials are incomplete" in log assert self.private_values, "credential files contain no secret values to check" for path in (self.data_paths[0] / "log").rglob("*"): if not path.is_file(): @@ -74,9 +78,9 @@ def failure_result(self, action, state, before): body = gzip.decompress(path.read_bytes()) if path.suffix == ".gz" else path.read_bytes() if any(value.encode() in body for value in self.private_values): raise AssertionError("secret leaked into diagnostic logs") - if path.name.startswith("monitor.log"): - kinds.update(json.loads(line)["kind"] for line in body.splitlines()) + credential_rejection |= b"server credentials are incomplete" in body if any(value in log for value in self.private_values): raise AssertionError("secret leaked into container logs") - return {"action": action, "state": state, "ready_before": before, - "ready_after": after, "log": log, "events": sorted(kinds), "secret_free": True} + return {"action": action, "state": state, "before": before, + "after": after, "log": log, "credential_rejection": credential_rejection, + "secret_free": True} diff --git a/e2e-test/deployment_listener_fixture.py b/e2e-test/deployment_listener_fixture.py index ce734f948..50ff83d9e 100644 --- a/e2e-test/deployment_listener_fixture.py +++ b/e2e-test/deployment_listener_fixture.py @@ -3,6 +3,7 @@ """Verify fixed Access listeners recover through the monitor retry plan.""" import json +import tomllib from managed_fixture import ManagedCluster @@ -13,11 +14,12 @@ def __init__(self, image, artifacts, protocol, services, **kwargs): def start(self): super().start() - config = self.command("exec", self.servers[0], "cat", "/opt/crowdb/run/config/access.toml") - expected = {"s3": "0.0.0.0:9091", "iceberg": "0.0.0.0:9092"}[self.protocol] - if expected not in config: + config = self.command("exec", self.servers[0], "cat", "/opt/crowdb/data/services/access-1/service.toml") + port = {"s3": 9091, "iceberg": 9092}[self.protocol] + expected = f"{self.addresses[0]}:{port}" + if tomllib.loads(config)[self.protocol]["listen"] != expected: raise AssertionError(f"{self.protocol} listener was not kept on its fixed port {expected}") - before = self.monitor_status()["services"]["access"] + before = self.service_status("access") result = self.control({"action": "recover-service", "node": 0, "service": "access", "signal": "KILL"}) after = result["after"] @@ -25,10 +27,10 @@ def start(self): raise AssertionError("monitor did not schedule a new Access process") if json.loads(self.command("inspect", self.servers[0]))[0]["NetworkSettings"]["Networks"][self.run_id]["IPAddress"] != self.addresses[0]: raise AssertionError("Access recovery changed the owned node identity") - recovered_config = self.command("exec", self.servers[0], "cat", "/opt/crowdb/run/config/access.toml") - if expected not in recovered_config: + recovered_config = self.command("exec", self.servers[0], "cat", "/opt/crowdb/data/services/access-1/service.toml") + if tomllib.loads(recovered_config) != tomllib.loads(config): raise AssertionError(f"{self.protocol} listener changed after retry") (self.artifacts / "listener-recovery.json").write_text(json.dumps({ "protocol": self.protocol, "before": before, "after": after, - "fixed_listener": expected, "event": result["event"]}, indent=2)) + "fixed_listener": expected, "signal": result["signal"]}, indent=2)) return self diff --git a/e2e-test/deployment_validation_fixture.py b/e2e-test/deployment_validation_fixture.py index 1ce200810..2c690e81a 100644 --- a/e2e-test/deployment_validation_fixture.py +++ b/e2e-test/deployment_validation_fixture.py @@ -17,18 +17,19 @@ def profile(self): return "invalid = true\n" if self.kind == "profile" else super().profile() def launch(self, data, profile): - if self.kind == "manifest": - preparer = self.run_id + "-manifest-input" + if self.kind == "policy": + preparer = self.run_id + "-policy-input" self.containers.append(preparer) self.command("run", "--rm", "--name", preparer, "--label", f"org.crowdb.e2e.run={self.run_id}", "--network", "none", "--user", "root", "--cpu-shares", "1024", "--env", "TOKIO_WORKER_THREADS=1", "--memory", "128m", "--pids-limit", "32", "--mount", f"type=bind,source={data},target=/opt/crowdb/data", "--entrypoint", "/bin/sh", self.image, "-ec", - "mkdir /opt/crowdb/data/bootstrap; printf 'invalid manifest' > /opt/crowdb/data/bootstrap/manifest.json; chown -R crowdb:crowdb /opt/crowdb/data/bootstrap; chmod 0600 /opt/crowdb/data/bootstrap/manifest.json") + "printf 'invalid policy' > /opt/crowdb/data/node-policy.json; chown crowdb:crowdb /opt/crowdb/data/node-policy.json; chmod 0600 /opt/crowdb/data/node-policy.json") self.containers.remove(preparer) address = start_node(self, 0, data, (), address=self.address, running=False, - environment=("CROWDB_STARTUP_MODE=single", "CROWDB_PROFILE_PATH=/opt/crowdb/etc/e2e-profile.toml"), + environment=("CROWDB_DEPLOYMENT_MODE=local", "CROWDB_INITIAL_VOTERS=1", + f"CROWDB_CLUSTER={self.run_id}", "CROWDB_PROFILE_PATH=/opt/crowdb/etc/e2e-profile.toml"), mounts=(f"type=bind,source={profile},target=/opt/crowdb/etc/e2e-profile.toml,readonly",)) self.command("start", self.servers[0]) return address diff --git a/e2e-test/failure_logs.py b/e2e-test/failure_logs.py index c8d83c2b8..f08406817 100644 --- a/e2e-test/failure_logs.py +++ b/e2e-test/failure_logs.py @@ -7,7 +7,12 @@ import os import re -ERROR = re.compile(r"\b(?:error|critical|fatal)\b|panicked|segmentation fault|core dumped", re.I) +# Match a log level, rather than a field or message containing the word +# "error". A transient `error=...` field is useful context, but it is not by +# itself evidence that a service failed. +ERROR = re.compile(r"\b(?:ERROR|CRITICAL|FATAL)\b|(?i:panicked|segmentation fault|core dumped)") +EXPLICIT_LEVEL = re.compile(r"\b(?:ERROR|CRITICAL|FATAL)\b") +REMOVED_CONTAINER = re.compile(r"(?:no such container|no such object)", re.I) RECORD = re.compile(r"^\[?\d{4}(?:-\d{2}-\d{2}|\d{4}-)") @@ -43,9 +48,18 @@ def print_failure(artifacts, data_paths, case_name, error, redact): opener = gzip.open if path.name.endswith(".gz") else open with opener(path, "rt", errors="replace") as stream: text = stream.read() + # Teardown may remove a short-lived helper container before the + # diagnostics pass reaches its saved docker output. This is a + # cleanup race, not a second service failure. + if path.suffix == ".error" and REMOVED_CONTAINER.search(text): + continue # Raw child stdout/stderr and diagnostic failures may have no level. raw = path.name == "service.log" or path.suffix == ".error" records = text.splitlines() if raw else error_records(text) + code = re.search(r"\bexit_code=(\d+)\b", text) + if raw and path.name == "service.log" and not EXPLICIT_LEVEL.search(text): + if not code or int(code[1]) == 0: + continue if records: node, service = "unknown", "unknown" for data in data_paths: @@ -53,12 +67,16 @@ def print_failure(artifacts, data_paths, case_name, error, redact): node = data.parent.name if data.name == "data" else data.name service = path.relative_to(data / "log").parts[0] break - code = re.search(r"\bexit_code=(\d+)\b", text) first_error = next((line for line in records if ERROR.search(line)), records[0]) summary = {"case": case_name, "container_node": node, "service": service, "service_exit_code": int(code[1]) if code else None, "log": str(path), "first_error": first_error} print(redact("[FAILURE SUMMARY] " + json.dumps(summary, ensure_ascii=False)), flush=True) print(redact(f"[CONTAINER ERROR LOG] {path}\n" + "\n".join(records)), flush=True) + except PermissionError: + # Container-owned files may be unreadable from the host after a + # failed run. The publisher records the omission separately; do + # not turn it into another failure line here. + continue except (OSError, EOFError) as log_error: print(redact(f"[LOG READ FAILED] {path}: {log_error}"), flush=True) diff --git a/e2e-test/fixture.py b/e2e-test/fixture.py index 9d6577880..bdaae20b9 100644 --- a/e2e-test/fixture.py +++ b/e2e-test/fixture.py @@ -37,7 +37,7 @@ def request(origin, path, body=None): content = response.read() return json.loads(content) if content else None except urllib.error.HTTPError as error: - error.add_note(error.read().decode()) + error.add_note(f"{origin}{path}: {redact(error.read().decode())}") raise diff --git a/e2e-test/image_identity.py b/e2e-test/image_identity.py index 44adeebdc..779030908 100644 --- a/e2e-test/image_identity.py +++ b/e2e-test/image_identity.py @@ -3,8 +3,9 @@ """Allow client-only revisions to reuse an image with identical server sources.""" import re +import os import subprocess -import sys +import tempfile import tomllib from pathlib import Path @@ -28,6 +29,13 @@ # Host launch resource flags do not enter the release artifact (stage.sh). ":(exclude)container/oci-image/runtime.py", ":(exclude)container/oci-image/run-node.py", + # These host CLI/cleanup modules are embedded only in crowdb-cli. stage.sh + # builds the node services and never reads or packages these Python files. + ":(exclude)container/oci-image/host_inputs.py", + ":(exclude)container/oci-image/host_runtime.py", + ":(exclude)container/oci-image/host_lifecycle.py", + ":(exclude)container/oci-image/host_cleanup.py", + ":(exclude)container/oci-image/host-clean.py", ":(exclude)container/oci-image/prepare-ci-host.py", # Host SDK client delegates are not read by staging or the OCI builder. ":(exclude)tools/pixi-tasks/test.py", @@ -72,14 +80,24 @@ ":(exclude)tools/sdk/package.py", # This host-only RPC benchmark is not a dependency or staged server binary. ":(exclude)app/crowdb-cli/src/commands/bench/rpc.rs", + ":(exclude)app/crowdb-cli/src/commands/bench/result.rs", + ":(exclude)app/crowdb-cli/src/commands/bench/kv/read.rs", + ":(exclude)app/crowdb-cli/src/commands/bench/kv/scan.rs", + ":(exclude)app/crowdb-cli/src/commands/bench/kv/write.rs", # This workload driver has only CLI consumers; server consumer changes # remain guarded. Its test hook does not enter a release build. ":(exclude)lib/crowdb-console-shared/src/ops/s3_bench.rs", # Host CLI lifecycle is not built or packaged by release staging. ":(exclude)app/crowdb-cli/src/commands.rs", ":(exclude)app/crowdb-cli/src/commands/s3.rs", - ":(exclude)app/crowdb-cli/src/commands/container_cluster.rs", - ":(glob,exclude)app/crowdb-cli/src/commands/container_cluster/**", + ":(glob,exclude)app/crowdb-cli/src/commands/s3/**", + ":(exclude)app/crowdb-cli/src/commands/cluster.rs", + ":(glob,exclude)app/crowdb-cli/src/commands/cluster/**", + ":(exclude)app/crowdb-cli/src/main.rs", + ":(exclude)app/crowdb-cli/src/commands/kv.rs", + ":(exclude)app/crowdb-cli/src/commands/kv/server.rs", + ":(glob,exclude)app/crowdb-cli/src/commands/kv/server/**", + ":(exclude)app/crowdb-cli/src/commands/launch.rs", # Audited host-only crate: every consumer uses a dev-dependency. A new # runtime/build dependency changes its owning manifest and rejects reuse. ":(exclude)lib/crowdb-test-harness/Cargo.toml", @@ -135,6 +153,14 @@ def production_manifest(manifest): result = dict(manifest) result.pop("test", None) result.pop("dev-dependencies", None) + if "example" in result: + examples = [entry for entry in result["example"] if not ( + entry.get("path", "").startswith("tests/common/") + and entry.get("required-features") == ["test-util"])] + if examples: + result["example"] = examples + else: + result.pop("example") if result.get("package", {}).get("name") == "crowdb-console-shared": features = dict(result.get("features", {})) if features.get("test-util") == []: @@ -181,11 +207,19 @@ def packages(lock): def verify_server_sources(root, revision, source_tree=None): + with tempfile.TemporaryDirectory(prefix="crowdb-source-check-") as directory: + return _verify_with_index(root, revision, source_tree, Path(directory) / "index") + + +def _verify_with_index(root, revision, source_tree, index): if not isinstance(revision, str) or not re.fullmatch(r"[0-9a-f]{40}", revision): raise ValueError("release image needs a full source revision label") + environment = dict(os.environ, GIT_INDEX_FILE=str(index), GIT_OPTIONAL_LOCKS="0") + def git(*arguments): - return subprocess.check_output(["git", "-C", str(root), *arguments], text=True).strip() + return subprocess.check_output(["git", "-C", str(root), *arguments], + env=environment, text=True).strip() try: git("rev-parse", "--verify", revision + "^{commit}") @@ -196,15 +230,14 @@ def git(*arguments): raise ValueError("release image source tree label is invalid") if git("rev-parse", "--verify", source_tree + "^{tree}") != source_tree: raise ValueError("release image source tree label must identify a tree") - tool = Path(__file__).resolve().parents[1] / "container/oci-image/source_tree.py" - current_tree = subprocess.check_output([sys.executable, str(tool), str(root)], text=True).strip() source = source_tree - comparison = [source, current_tree] - changed = git("diff", "--name-only", *comparison, "--", *SERVER_PATHS) - untracked = "" - else: - changed = git("diff", "--name-only", source, "--", *SERVER_PATHS) - untracked = git("ls-files", "--others", "--exclude-standard", "--", *SERVER_PATHS) + # A private baseline index includes dirty inputs staged into the image + # tree, even if the operator never added them to their real index. + # Comparing its working files writes neither that index nor Git objects. + git("read-tree", source) + comparison = [source] + changed = git("diff", "--name-only", *comparison, "--", *SERVER_PATHS) + untracked = git("ls-files", "--others", "--exclude-standard", "--", *SERVER_PATHS) added = set(git("diff", "--name-only", "--diff-filter=A", *comparison, "--", *SERVER_PATHS).splitlines()) except subprocess.CalledProcessError as error: raise ValueError("image source revision cannot be verified locally; rebuild the shared OCI image") from error diff --git a/e2e-test/managed_fixture.py b/e2e-test/managed_fixture.py index 3d5f5f621..cc6e78694 100644 --- a/e2e-test/managed_fixture.py +++ b/e2e-test/managed_fixture.py @@ -1,6 +1,6 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Normal release monitor startup with a mounted service-selection profile.""" +"""Normal node startup with a sealed service-selection policy.""" import json import time @@ -8,18 +8,14 @@ from allocation import create_network, node_data, start_node from fixture import Cluster, LABEL, request, wait -SOURCE = Path(__file__).resolve().parents[1] / "container/single-node-container" +SOURCE = Path(__file__).resolve().parents[1] / "container/oci-image" def selected_profile(services): - parts = (SOURCE / "profile.toml").read_text().split("[[services]]") - selected = [part for part in parts[1:] if any( - part.splitlines()[1] == f'id = "{service}"' for service in services)] - if len(selected) != len(services): + known = {"kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web"} + if len(set(services)) != len(services) or not set(services) <= known: raise ValueError("unknown or duplicate release service selection") - profile = parts[0] + "".join("[[services]]" + part for part in selected) - return profile.replace('"--management-port", "10000"', - '"--management-addr", "0.0.0.0", "--management-port", "10000"') + return (SOURCE / "node-profile.toml").read_text() class ManagedCluster(Cluster): @@ -55,10 +51,32 @@ def profile(self): return selected_profile(self.services) def launch(self, data, profile): + listeners = { + "paxos-kv": ("kv", {"http_port": 10000, "rpc_port": 10100}), + "diskdb": ("diskdb", {"rpc_port": 7100}), + "diskio": ("diskio", {"rpc_port": 7200}), + "chunkdb": ("chunkdb", {"http_port": 7300, "rpc_port": 7301}), + "chunk-kv": ("chunk-kv", {"http_port": 7400, "rpc_port": 7401}), + "access-server": ("access", {"http_port": 9092, "s3_port": 9091, + "dataset_port": 9093, "health_port": 9094}), + } + policy = self.artifacts / "node-policy.toml" + choices = [] + for kind, (role, ports) in listeners.items(): + ports = ", ".join(f"{key} = {value}" for key, value in ports.items()) + enabled = str(role in self.services).lower() + choices.append(f'{{ kind = "{kind}", enabled = {enabled}, listeners = {{ {ports} }} }}') + choices = ", ".join(choices) + storage = 'mode = "automatic"\ncapacity_bytes = 17179869184\nreserve_bytes = 1073741824\nsparse = true' \ + if "diskio" in self.services else 'mode = "diskless"' + policy.write_text(f'services = [{choices}]\n[storage]\n{storage}\n') return start_node(self, 0, data, (), address=self.address, memory="2048m" if len(self.services) > 2 else "512m", - environment=("CROWDB_STARTUP_MODE=single", "CROWDB_PROFILE_PATH=/opt/crowdb/etc/e2e-profile.toml"), - mounts=(f"type=bind,source={profile},target=/opt/crowdb/etc/e2e-profile.toml,readonly",)) + environment=("CROWDB_DEPLOYMENT_MODE=local", "CROWDB_INITIAL_VOTERS=1", + f"CROWDB_CLUSTER={self.run_id}", "CROWDB_PROFILE_PATH=/opt/crowdb/etc/e2e-profile.toml", + "CROWDB_NODE_POLICY_FILE=/run/e2e-node-policy.toml"), + mounts=(f"type=bind,source={profile},target=/opt/crowdb/etc/e2e-profile.toml,readonly", + f"type=bind,source={policy},target=/run/e2e-node-policy.toml,readonly")) def ready(self): deadline = time.monotonic() + 60 @@ -71,33 +89,46 @@ def ready(self): if not state["Running"]: raise AssertionError("normal release monitor exited: " + self.command("logs", self.servers[0])) try: - last = json.loads(self.command("exec", self.servers[0], "cat", - "/opt/crowdb/run/status/monitor.json")) - if last["phase"] == "ready": - if set(last["services"]) != set(self.services): - raise AssertionError(f"unexpected running services: {last}") + report = request(f"http://{self.addresses[0]}:9090", "/api/cluster/status") + last = report + if report["phase"] == "failed": + raise AssertionError(f"normal selected release services failed: {report}") + if report["phase"] == "ready": + setup = json.loads(self.command("exec", self.servers[0], "cat", + "/opt/crowdb/data/node-setup.json")) + last = {"cluster": report, "setup": setup["status"]} + if setup["status"]["phase"] == "failed": + raise AssertionError(f"selected service setup failed: {last}") + if setup["status"]["phase"] != "ready": + time.sleep(0.1) + continue + status = self.monitor_status() + expected = {"web", "kv"} | {role + "-1" for role in self.services + if role not in ("web", "kv")} + if set(status["services"]) != expected: + raise AssertionError(f"unexpected running services: {status}") self.command("exec", self.servers[0], "/opt/crowdb/bin/crowdb-monitor", "readiness") - (self.artifacts / "managed-status.json").write_text(json.dumps(last, indent=2)) + (self.artifacts / "managed-status.json").write_text(json.dumps(status, indent=2)) self.client_configuration() wait(lambda: request(self.origins[0], "/health"), lambda _: True, "host management reachability") return - except RuntimeError as error: + except (RuntimeError, OSError) as error: last = str(error) time.sleep(0.1) raise AssertionError(f"normal selected release services never became ready: {last}") def client_configuration(self): - manifest = self.command("exec", self.servers[0], "cat", - "/opt/crowdb/data/bootstrap/manifest.json") - (self.artifacts / "managed-manifest.json").write_text(manifest) + accepted = self.command("exec", self.servers[0], "cat", + "/opt/crowdb/data/accepted-node.json") + (self.artifacts / "managed-node.json").write_text(accepted) config_root = self.artifacts / "runtime-config" / "config" config_root.mkdir(parents=True, exist_ok=True) - for service in ("chunkdb", "chunk-kv"): - if service in self.services: + for role in ("chunkdb", "chunk-kv"): + if role in self.services: body = self.command("exec", self.servers[0], "cat", - f"/opt/crowdb/run/config/{service}.toml") - (config_root / f"{service}.toml").write_text(body) + f"/opt/crowdb/data/services/{role}-1/service.toml") + (config_root / f"{role}.toml").write_text(body) if "access" in self.services: credentials = self.artifacts / "client.env" credentials.touch(mode=0o600, exist_ok=True) @@ -117,6 +148,10 @@ def crash_restart(self): self.ready() self.timings["restart"] = time.monotonic() - started + def service_status(self, service, status=None): + key = service if service in ("kv", "web") else service + "-1" + return (self.monitor_status() if status is None else status)["services"][key] + def control(self, body): if body.get("action") != "recover-service": return super().control(body) @@ -127,7 +162,7 @@ def control(self, body): info = json.loads(self.command("inspect", name))[0] if info["Config"]["Labels"].get(LABEL) != self.run_id or not info["State"]["Running"]: raise ValueError("service container ownership or running state changed") - before = self.monitor_status()["services"][service] + before = self.service_status(service) pid = before["pid"] if type(pid) is not int or pid <= 1: raise ValueError("service PID must identify a child of the release monitor") @@ -139,6 +174,16 @@ def control(self, body): if Path(executable).name != expected: raise ValueError("observed service PID executable changed") self.command("exec", name, "kill", "-" + signal, str(pid)) + hung = None + if signal == "STOP": + try: + hung = self.observe_hang(name, service, before) + finally: + if self.service_status(service).get("pid") == pid: + self.command("exec", name, "kill", "-CONT", str(pid)) + # Readiness observes a live process; explicit termination triggers + # the monitor's eligible-exit recovery policy. + self.command("exec", name, "kill", "-KILL", str(pid)) deadline = time.monotonic() + 90 while time.monotonic() < deadline: cancelled = getattr(self, "control_cancelled", None) @@ -147,20 +192,36 @@ def control(self, body): if not json.loads(self.command("inspect", name))[0]["State"]["Running"]: raise AssertionError("release container exited during service recovery") state = self.monitor_status() - after = state["services"][service] + after = self.service_status(service, state) if (state["phase"] == "ready" and after.get("pid") and after["pid"] != pid and after["generation"] > before["generation"]): - expected_event = "child_exited" if signal == "KILL" else "probe_failed" - events = [json.loads(line) for line in self.command( - "exec", name, "cat", "/opt/crowdb/data/log/monitor/monitor.log").splitlines()] - if not any(event.get("service") == service and event.get("kind") == expected_event - for event in events): - raise AssertionError("service recovery did not record its original fault event") + if service not in ("kv", "web"): + execution = json.loads(self.command("exec", name, "cat", + f"/opt/crowdb/data/services/{service}-1/execution.json")) + if (not execution["complete"] or execution["pid"] != after["pid"] + or execution["intent"]["operation_id"] != before["operation_id"]): + raise AssertionError("service recovery changed its accepted operation or durable process result") self.command("exec", name, "/opt/crowdb/bin/crowdb-monitor", "readiness") - return {"before": before, "after": after, "event": expected_event} + return {"before": before, "after": after, "signal": signal, "hung": hung, + "recovery_trigger": "explicit-termination" if hung else "exit"} time.sleep(0.1) raise AssertionError(f"{service} did not recover after {signal} within 90 seconds") + def observe_hang(self, name, service, before): + state = self.command("exec", name, "cat", f"/proc/{before['pid']}/status") + if "State:\tT" not in state: + raise AssertionError("owned process was not stopped by the injected fault") + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + current = self.service_status(service) + if (current.get("pid") != before["pid"] + or current["generation"] != before["generation"]): + raise AssertionError("health observation restarted a live process") + if not current["healthy"]: + return current + time.sleep(0.1) + raise AssertionError(f"{service} hang was not reported by the independent monitor probe") + def monitor_status(self): return json.loads(self.command("exec", self.servers[0], "cat", "/opt/crowdb/run/status/monitor.json")) diff --git a/e2e-test/native_containerd_profile.py b/e2e-test/native_containerd_profile.py new file mode 100644 index 000000000..7b4518d40 --- /dev/null +++ b/e2e-test/native_containerd_profile.py @@ -0,0 +1,97 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Registered native containerd acceptance, independent of Docker host provisioning.""" + +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import time +import uuid + +from client import bundle_cli +from fixture import redact +from image_identity import verify_server_sources +from oci_native_runtime import docker_free + +ROOT = Path(__file__).resolve().parents[1] +sys.path.append(str(ROOT / 'container/oci-image')) +from artifact import receipt + + +def client_bundle(destination): + destination.mkdir(parents=True) + (destination / 'lib').mkdir() + supplied = os.environ.get('CROWDB_E2E_CLI_BUNDLE') + if supplied: + source = Path(supplied).resolve() + shutil.copy2(source / 'crowdb-cli', destination / 'crowdb-cli') + for path in (source / 'lib').iterdir(): + if path.is_file(): + shutil.copy2(path, destination / 'lib' / path.name) + return destination + bundle_cli(destination) + result = subprocess.run(['ldd', '-r', str(destination / 'crowdb-cli')], text=True, + capture_output=True, check=True, timeout=30) + if 'not found' in result.stdout or 'undefined symbol' in result.stdout: + raise RuntimeError(result.stdout) + prefix = Path(os.environ['CONDA_PREFIX']).resolve() + for line in result.stdout.splitlines(): + fields = line.split() + if len(fields) >= 3 and fields[1] == '=>': + source = Path(fields[2]).resolve() + if source.is_relative_to(prefix): + shutil.copy2(source, destination / 'lib' / fields[0]) + return destination + + +def run_profile(args, parser): + if args.layer != 'all' or args.scope != 'all' or args.case or args.require_isolation: + parser.error('native containerd has one host lifecycle acceptance; domain selectors do not apply') + if args.concurrency != 1 or args.cpus < 1.5 or args.memory_mib < 2560: + parser.error('native containerd requires --concurrency 1, 1.5 CPU and 2560 MiB including its host client') + if args.list: + print(json.dumps([dict(scope='deployment.containerd.native', + case='local_protocols_and_production_metadata_recreation', nodes=1)])) + return + if sys.platform != 'linux' or os.geteuid() != 0: + parser.error('native containerd acceptance requires an explicitly privileged disposable Linux host') + docker_free() + print('Native containerd preflight: verifying artifact and read-only server sources', flush=True) + archive = args.archive.resolve() + verified = receipt(archive) + sources = verify_server_sources(ROOT, verified['revision'], verified.get('source_tree') or None) + root = args.artifacts / ('run-' + uuid.uuid4().hex) + root.mkdir(parents=True, mode=0o700) + bundle = client_bundle(root / 'client') + environment = dict(os.environ, CROWDB_E2E_CLI_BUNDLE=str(bundle), + CROWDB_CONTAINERD_DATA=str(root / 'native-host'), CROWDB_CONTAINERD_CLUSTER='oci-' + root.name[-20:]) + (root / 'identity.json').write_text(json.dumps(dict(profile=args.profile, artifact=verified, + sources=sources, execution='native Linux host', docker_client=False, + docker_socket=False, docker_daemon=False), indent=2)) + print(f"Native containerd {verified['image_digest']}; artifacts {root}", flush=True) + started = time.monotonic() + child = subprocess.Popen([sys.executable, str(ROOT / 'e2e-test/profiles/containerd-host.py'), str(archive)], + env=environment, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + stdout, stderr = child.communicate(timeout=900) + finally: + if child.poll() is None: + child.terminate() + try: + stdout, stderr = child.communicate(timeout=45) + except subprocess.TimeoutExpired: + child.kill() + stdout, stderr = child.communicate(timeout=5) + (root / 'client-containerd-interrupted.log').write_text(redact(stdout + stderr)) + code = child.returncode + output = redact(stdout + stderr) + (root / 'client-containerd.log').write_text(output) + print(output, end='', flush=True) + (root / 'results.json').write_text(json.dumps(dict(profile=args.profile, + status='pass' if code == 0 else 'failed', wall_seconds=time.monotonic()-started, + passed=int(code == 0), failed=int(code != 0), skipped=0), indent=2)) + if code: + raise RuntimeError('native containerd acceptance failed; retained diagnostics: ' + str(root)) diff --git a/e2e-test/oci_host_node.py b/e2e-test/oci_host_node.py new file mode 100644 index 000000000..1aa925bed --- /dev/null +++ b/e2e-test/oci_host_node.py @@ -0,0 +1,205 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""One ordinary production node driven by the common CLI and independent cleanup.""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +from fixture import request, redact, wait +from profiles.node import request as admission_request +import oci_reference_protocols as protocols + +ROOT = Path(__file__).resolve().parents[1] +sys.path.append(str(ROOT / 'container/oci-image')) +from host_cleanup import clean, selected, read_private +from host_runtime import HostRuntime +from storage import inputs +from types import SimpleNamespace + + +def interface_address(): + result = subprocess.run(['ip', '-json', 'route', 'get', '192.0.2.1'], + capture_output=True, text=True, check=True, timeout=10) + route = json.loads(result.stdout)[0] + return route['dev'], route.get('prefsrc') or route['src'] + + +class HostNode: + node_count = 1 + + def __init__(self, root, bundle, image, runtime, *, address=None, voters=1, metadata_only=False, media=None): + self.root, self.bundle = Path(root), Path(bundle) + self.root.mkdir(mode=0o700, parents=True, exist_ok=True) + self.image, self.kind, self.address = image, runtime, address + self.cluster = os.environ.get('CROWDB_CONTAINERD_CLUSTER', 'oci-' + self.root.parent.name[-20:]) + self.cli = self.bundle / 'crowdb-cli' + self.cli_digest = self.client_digest() + self.interface, self.ip = interface_address() + self.origin = f'http://{self.ip}:9090' + self.voters, self.metadata_only = voters, metadata_only + self.private = self.root / 'private' + self.private.mkdir(mode=0o700) + self.data = self.private / 'node' + self.data.mkdir(mode=0o755) + os.chown(self.data, 10001, 10001) + self.secret = self.private / 'password' + self.secret.write_text('oci-host-acceptance-password') + self.secret.chmod(0o600) + self.policy = self.private / 'node-policy.toml' + self.devices = ([] if metadata_only else + [Path(value) for value in json.loads(os.environ.get('CROWDB_E2E_HOST_DEVICES', '[]'))]) + self.disk_files = [] + if media: + self.devices.append(media.raw) + self.disk_files = [media.file, media.bad] + self.make_policy() + self.env = dict(os.environ, CROWDB_RUNTIME_ROOT=str(self.private / 'runtime'), + LD_LIBRARY_PATH=str(self.bundle / 'lib'), NO_PROXY='*', no_proxy='*') + for key in ('CROWDB_CLUSTER', 'CROWDB_SYSTEM_IP', 'CROWDB_SYSTEM_PORT'): + self.env.pop(key, None) + + def make_policy(self): + lines = ['version = 1', 'mode = "production"', f'cluster = "{self.cluster}"', + f'initial_voters = {self.voters}'] + if self.metadata_only: + lines += ['[storage]', 'mode = "diskless"'] + for kind in ('paxos-kv', 'diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server'): + lines += ['[[services]]', f'kind = "{kind}"', f'enabled = {str(kind == "paxos-kv").lower()}'] + if kind == 'paxos-kv': + lines += ['[services.listeners]', 'http_port = 10000', 'rpc_port = 10100'] + else: + lines += ['[public_access]', *[f'{kind} = "http://{self.ip}:{port}"' + for kind, port in [('s3',9091), ('iceberg',9092), ('dataset',9093)]], '[storage]'] + if self.devices: + for device in self.devices: + os.chown(device, 10001, 10001) + device.chmod(0o660) + lines += ['mode = "supplied"'] + for disk in inputs(SimpleNamespace(device=self.devices, disk_file=[])): + capacity = int(Path('/sys/class/block', Path(disk['source']).name, 'size').read_text()) * 512 + lines += ['[[storage.disks]]', *[f'{key} = {json.dumps(disk[key])}' + for key in ('backend','path','backing_identity','resource_identity')], + f'capacity_bytes = {capacity}', 'initialize = true'] + else: + lines += ['mode = "automatic"', 'capacity_bytes = 8589934592', + 'reserve_bytes = 1073741824', 'sparse = false'] + self.policy.write_text('\n'.join(lines) + '\n') + self.policy.chmod(0o644) + + def command(self, arguments, expected=(0,), endpoint=None): + assert self.client_digest() == self.cli_digest + prefix = [str(self.cli), '--cluster', self.cluster] + if endpoint: + prefix += ['--management-endpoint', endpoint] + result = subprocess.run([*prefix, *arguments], env=self.env, text=True, + capture_output=True, timeout=210) + if result.returncode not in expected: + print(redact(result.stdout + result.stderr), flush=True) + raise AssertionError(f'host CLI {arguments[:2]} exited {result.returncode}') + return result.stdout + + def client_digest(self): + with self.cli.open('rb') as source: + return hashlib.file_digest(source, 'sha256').hexdigest() + + def start(self, fresh=False): + args = ['cluster', 'start', '--wait', '0' if fresh or self.metadata_only else '180', '--json'] + if fresh: + identity = Path('/etc/machine-id') + physical = (identity.read_text().strip() if identity.is_file() else '') or os.uname().nodename + args += ['--runtime', self.kind, '--network', 'host', '--image', self.image, + '--nodes', str(self.voters), '--cpus', '1', '--memory-mib', '2048', + '--data-root', str(self.data), '--physical-host-id', physical, + '--interface', self.interface, '--management-address', self.ip, + '--node-policy-file', str(self.policy), '--password-file', str(self.secret)] + if self.kind == 'containerd': + args += ['--runtime-endpoint', self.address, '--snapshotter', 'native'] + for device in self.devices: + args += ['--device', str(device)] + for disk in self.disk_files: + args += ['--disk-file', str(disk)] + report = json.loads(self.command(args, (0, 1) if self.metadata_only else ((1,) if fresh else (0,)))) + print('Host lifecycle', self.kind, self.cluster, 'start' if fresh else 'resume', report['phase'], flush=True) + return report + + def record(self): + return selected(self.cluster, Path(self.env['CROWDB_RUNTIME_ROOT'])) + + def facts(self): + record = self.record() + runtime = HostRuntime(record['options'], self.cluster) + runtime.check(record) + current = runtime.inspect(record['name']) + runtime.verify(current['Id'], 1, 2048, True) + return dict(runtime=self.kind, daemon_id=record['runtime']['daemon'], + container_id=current['Id'], image=current['Image'], host_network=True, + cpu_max=runtime.invoke('exec', current['Id'], 'cat', '/sys/fs/cgroup/cpu.max').strip(), + memory_max=runtime.invoke('exec', current['Id'], 'cat', '/sys/fs/cgroup/memory.max').strip(), + discovery_id=(self.data / 'node-identity').read_text().strip()) + + def admit(self): + deadline = time.monotonic() + 180 + while not (self.data / 'node-identity').is_file(): + if time.monotonic() > deadline: + raise AssertionError('ordinary entrypoint did not create node identity') + time.sleep(.25) + discovery = (self.data / 'node-identity').read_text().strip() + wait(lambda: request(self.origin, '/api/node/candidates'), + lambda value: (value.get('local') or {}).get('advertisement', {}).get('discovery_id') == discovery, + 'ordinary host management and discovery', seconds=180) + request(self.origin, '/api/racks', {'id':1, 'name':'host-production'}) + admitted = admission_request(self.origin, '/api/node/admit', dict(discovery_id=discovery, rack_id=1, + ssh_user='crowdb', ssh_port=2222, ssh_password=self.secret.read_text())) + request(self.origin, '/api/cluster/init', {'nodes':[admitted['node_id']]}) + if self.metadata_only: + self.metadata_report() + return self.start() + + def metadata_report(self): + def authority_ready(report): + return any(item['id'] == 'cluster/authority' and item['state'] == 'ready' + for item in report['observation']['dependencies']) + return wait(lambda: request(self.origin, '/api/cluster/status'), authority_ready, + 'production metadata authority', seconds=180) + + def endpoint(self, index): + assert index == 0 + return self.origin + + def invoke(self, *arguments, endpoint=None): + return self.command(list(arguments), endpoint=endpoint or self.origin) + + def protocols(self, witness=None): + credentials = request(self.origin, '/api/access/credentials') + original = protocols.credentials + protocols.credentials = lambda _cluster, _index: request(self.origin, '/api/access/credentials') + try: + witness = protocols.write_witnesses(self) if witness is None else witness + protocols.read_witnesses(self, witness) + return witness, credentials + finally: + protocols.credentials = original + + def recreate(self): + self.command(['cluster','stop']) + # This cleanup remains usable when the common CLI is absent. + clean(self.record()) + self.start() + + def cleanup(self): + path = Path(self.env['CROWDB_RUNTIME_ROOT']) / 'persistent/oci-hosts' / (self.cluster + '.json') + if read_private(path) is None: + return + record = self.record() + runtime = HostRuntime(record['options'], self.cluster) + current = runtime.named(record['name']) + if current: + result = subprocess.run([*runtime.command, 'logs', current['Id']], capture_output=True, + text=True, timeout=60) + (self.root / 'node.log').write_text(redact(result.stdout + result.stderr)) + clean(record) diff --git a/e2e-test/oci_host_policy.py b/e2e-test/oci_host_policy.py new file mode 100644 index 000000000..c03bca218 --- /dev/null +++ b/e2e-test/oci_host_policy.py @@ -0,0 +1,53 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Accept a diskless test policy before attaching unadmitted candidate media.""" + +import json +import os +from pathlib import Path +import uuid + +from host_runtime import HostRuntime +from runtime import OWNER + + +def accept_candidates(node): + runtime = HostRuntime(dict(runtime=node.kind, snapshotter='overlayfs', runtime_endpoint=None), node.cluster) + name = 'crowdb-' + node.cluster + '-candidate-policy' + token = str(uuid.uuid4()) + identity = runtime.identity() + if runtime.named(name) is not None: + raise ValueError('candidate-policy scope already contains a container') + intent = node.private/'candidate-policy.json' + with intent.open('x') as stream: + intent.chmod(0o600) + json.dump(dict(name=name, owner=token, image=node.image, runtime=identity), stream) + stream.flush() + os.fsync(stream.fileno()) + machine = Path('/etc/machine-id') + physical = (machine.read_text().strip() if machine.is_file() else '') or os.uname().nodename + try: + assert runtime.identity() == identity + result = runtime.invoke('run', '--rm', '--name', name, '--network', 'none', '--read-only', + '--cpus', '.25', '--memory', '128m', '--user', '10001:10001', + '--label', f'{OWNER}={token}', '--label', f'org.crowdb.cluster={node.cluster}', + '--mount', f'type=bind,source={node.data},target=/opt/crowdb/data', + '--mount', f'type=bind,source={node.policy},target=/run/node-policy.toml,readonly', + '--env', 'CROWDB_NODE_POLICY_FILE=/run/node-policy.toml', + '--env', f'CROWDB_PHYSICAL_HOST_ID={physical}', + '--env', f'CROWDB_CLUSTER={node.cluster}', '--env', 'CROWDB_INITIAL_VOTERS=1', + '--entrypoint', '/opt/crowdb/bin/crowdb-monitor', node.image, 'accept-policy') + assert result.strip() == 'production', result + accepted = json.loads((node.data/'node-policy.json').read_text()) + assert accepted['cluster'] == node.cluster and accepted['setup']['storage']['mode'] == 'diskless' + assert not (node.data/'node-identity').exists() and not (node.data/'disks').exists() + finally: + current = runtime.named(name) + if current: + labels = current['Config'].get('Labels', {}) + if (runtime.identity() != identity or current['Image'] != node.image + or labels.get(OWNER) != token or labels.get('org.crowdb.cluster') != node.cluster): + raise ValueError('candidate-policy runtime ownership changed; preserving container') + if current['State']['Running']: + runtime.invoke('stop', current['Id']) + runtime.invoke('rm', current['Id']) diff --git a/e2e-test/oci_local_host_node.py b/e2e-test/oci_local_host_node.py new file mode 100644 index 000000000..30e229993 --- /dev/null +++ b/e2e-test/oci_local_host_node.py @@ -0,0 +1,171 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Explicit local one-node policy for full host-network runtime qualification.""" + +import json +import os +from pathlib import Path +import subprocess +import uuid + +from fixture import redact +from oci_host_node import HostNode +from host_runtime import HostRuntime +from runtime import OWNER +from host_cleanup import read_private + + +class LocalHostNode(HostNode): + def make_policy(self): + self.devices, self.disk_files = [], [] + lines = ['version = 1', 'mode = "local"', f'cluster = "{self.cluster}"', 'initial_voters = 1', + '[public_access]', *[f'{kind} = "http://{self.ip}:{port}"' + for kind, port in [('s3',9091), ('iceberg',9092), ('dataset',9093)]], + '[storage]', 'mode = "automatic"', 'capacity_bytes = 8589934592', + 'reserve_bytes = 1073741824', 'sparse = true'] + self.policy.write_text('\n'.join(lines) + '\n') + self.policy.chmod(0o644) + + def runtime(self): + return HostRuntime(dict(runtime=self.kind, snapshotter='native' if self.kind == 'containerd' else 'overlayfs', + runtime_endpoint=self.address), self.cluster) + + def record(self): + path = self.private / 'local-runtime.json' + record = read_private(path) + if record is None: + raise ValueError('local test ownership is absent') + return record + + def save(self, record): + path = self.private / 'local-runtime.json' + temporary = self.private / ('.local-runtime-' + uuid.uuid4().hex) + with temporary.open('x') as stream: + temporary.chmod(0o600) + json.dump(record,stream,indent=2) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary,path) + + def owned(self, runtime, record): + if runtime.identity() != record['runtime']: + raise ValueError('local test runtime identity changed') + current = runtime.named(record['name']) + if current is None: + return None + labels = current.get('Config', {}).get('Labels', {}) + if (current['Image'] != self.image or labels.get(OWNER) != record['owner'] + or labels.get('org.crowdb.cluster') != self.cluster): + raise ValueError('local test container ownership or image changed') + host = current['HostConfig'] + cpus = host.get('NanoCpus', 0) / 1_000_000_000 + if not cpus and host.get('CpuPeriod', 0): + cpus = host['CpuQuota'] / host['CpuPeriod'] + if host['NetworkMode'] != 'host' or abs(cpus - 1) > .00001 or host['Memory'] != 2048 * 1048576: + raise ValueError('local test networking or resource limits changed') + expected = {'/opt/crowdb/data':(str(self.data),True), + '/run/crowdb-node-policy.toml':(str(self.policy),False), + '/run/crowdb-ssh-password':(str(self.secret),False)} + observed = {mount['Destination']:(mount['Source'],mount['RW']) for mount in current.get('Mounts',[])} + if observed != expected: + raise ValueError('local test mounts changed') + environment = dict(value.split('=',1) for value in current['Config']['Env'] if '=' in value) + for key,value in dict(CROWDB_DEPLOYMENT_MODE='local',CROWDB_INITIAL_VOTERS='1', + CROWDB_CLUSTER=self.cluster,CROWDB_MANAGEMENT_INTERFACE=self.interface, + TOKIO_WORKER_THREADS='1').items(): + if environment.get(key) != value: + raise ValueError('local test startup policy changed') + return current + + def command(self, arguments, expected=(0,), endpoint=None): + assert self.client_digest() == self.cli_digest + result = subprocess.run([str(self.cli),'--management-endpoint',endpoint or self.origin,*arguments], + env=self.env,text=True,capture_output=True,timeout=210) + if result.returncode not in expected: + print(redact(result.stdout + result.stderr),flush=True) + raise AssertionError(f'local host client {arguments[:2]} exited {result.returncode}') + return result.stdout + + def start(self, fresh=False): + runtime = self.runtime() + path = self.private / 'local-runtime.json' + if not path.exists(): + if runtime.named('crowdb-' + self.cluster + '-evaluation'): + raise ValueError('local evaluation scope already contains a foreign container') + self.save(dict(version=1,owner=str(uuid.uuid4()),name='crowdb-' + self.cluster + '-evaluation', + runtime=runtime.identity(),policy='local',node_failure_tolerance=0, + options=dict(kind=self.kind,address=self.address,image=self.image, + cluster=self.cluster,interface=self.interface,ip=self.ip))) + record = self.record() + current = self.owned(runtime,record) + if current is None: + identity = Path('/etc/machine-id').read_text().strip() + runtime.invoke('create','--name',record['name'],'--label',f"{OWNER}={record['owner']}", + '--label',f'org.crowdb.cluster={self.cluster}','--network','host','--cpus','1','--memory','2048m', + '--mount',f'type=bind,source={self.data},target=/opt/crowdb/data', + '--mount',f'type=bind,source={self.policy},target=/run/crowdb-node-policy.toml,readonly', + '--mount',f'type=bind,source={self.secret},target=/run/crowdb-ssh-password,readonly', + '--env','CROWDB_DEPLOYMENT_MODE=local','--env','CROWDB_INITIAL_VOTERS=1', + '--env','TOKIO_WORKER_THREADS=1', + '--env',f'CROWDB_CLUSTER={self.cluster}','--env',f'CROWDB_PHYSICAL_HOST_ID={identity}', + '--env',f'CROWDB_MANAGEMENT_INTERFACE={self.interface}', + '--env','CROWDB_NODE_POLICY_FILE=/run/crowdb-node-policy.toml', + '--env','CROWDB_SSH_PASSWORD_FILE=/run/crowdb-ssh-password',self.image) + current = self.owned(runtime,record) + if not current['State']['Running']: + runtime.invoke('start',current['Id']) + current = self.owned(runtime,record) + runtime.verify(current['Id'],1,2048,True) + report = json.loads(self.command(['cluster','check','--wait','180','--json'])) + actual = report['observation']['cluster_id'] + assert record.get('cluster_id',actual) == actual + record['cluster_id'] = actual + self.save(record) + print('Local host evaluation ready',self.kind,actual,'zero node-failure tolerance',flush=True) + return report + + def facts(self): + runtime,record = self.runtime(),self.record() + current = self.owned(runtime,record) + runtime.verify(current['Id'],1,2048,True) + return dict(runtime=self.kind,daemon_id=record['runtime']['daemon'],container_id=current['Id'], + image=current['Image'],host_network=True,policy='local',node_failure_tolerance=0, + cpu_max=runtime.invoke('exec',current['Id'],'cat','/sys/fs/cgroup/cpu.max').strip(), + memory_max=runtime.invoke('exec',current['Id'],'cat','/sys/fs/cgroup/memory.max').strip(), + discovery_id=(self.data/'node-identity').read_text().strip()) + + def recreate(self): + self.cleanup() + self.start() + + def cleanup(self): + if not (self.private/'local-runtime.json').exists(): + return + runtime,record = self.runtime(),self.record() + current = self.owned(runtime,record) + if current is None: + return + output = subprocess.run([*runtime.command,'logs',current['Id']],capture_output=True,text=True,timeout=60) + (self.root/'node-logs.log').write_text(redact(output.stdout + output.stderr)) + if current['State']['Running']: + runtime.invoke('stop','--time','30',current['Id']) + current = self.owned(runtime,record) + assert not current['State']['Running'] + runtime.invoke('rm',current['Id']) + + +def cleanup_local_host(root): + root = Path(root) + record = read_private(root/'private/local-runtime.json') + if record is None: + return + if (record['version'] != 1 or record['policy'] != 'local' or record['node_failure_tolerance'] != 0 + or record['name'] != 'crowdb-' + record['options']['cluster'] + '-evaluation' + or str(uuid.UUID(record['owner'])) != record['owner']): + raise ValueError('invalid local host test ownership') + node = LocalHostNode.__new__(LocalHostNode) + node.root, node.private = root, root/'private' + for key,value in record['options'].items(): + setattr(node,key,value) + node.data, node.policy, node.secret = (node.private/'node',node.private/'node-policy.toml',node.private/'password') + node.cleanup() diff --git a/e2e-test/oci_native_runtime.py b/e2e-test/oci_native_runtime.py new file mode 100644 index 000000000..59eafb7bb --- /dev/null +++ b/e2e-test/oci_native_runtime.py @@ -0,0 +1,106 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""An owned rootful containerd daemon on a Docker-free Linux host.""" + +import json +import os +import re +from pathlib import Path +import shutil +import signal +import subprocess +import sys +import tempfile +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.append(str(ROOT / 'container/oci-image')) +from runtime import Runtime + + +def docker_free(*, allow_container_host=False): + if shutil.which('docker') or Path('/var/run/docker.sock').exists(): + raise ValueError('native containerd acceptance requires no Docker client or socket') + if not allow_container_host and (Path('/.dockerenv').exists() + or Path('/run/.containerenv').exists() or os.environ.get('container')): + raise ValueError('native host acceptance cannot substitute a container-provisioned Linux host') + for path in Path('/proc').glob('[0-9]*/comm'): + try: + if path.read_text().strip() == 'dockerd': + raise ValueError('native containerd acceptance requires no Docker daemon') + except FileNotFoundError: + continue + + +class NativeRuntime: + def __init__(self, root, cluster, *, docker_provisioned=False): + self.root = Path(root) + self.root.mkdir(mode=0o700) + self.cluster = cluster + self.docker_provisioned = docker_provisioned + self.prefix = Path(os.environ.get('CROWDB_OCI_RUNTIME_PREFIX', + ROOT / 'container/oci-image/.pixi/envs/runtime')).resolve() + for name in ('containerd', 'containerd-shim-runc-v2', 'nerdctl', 'runc'): + if not (self.prefix / 'bin' / name).is_file(): + raise ValueError('install the locked OCI runtime Pixi environment before host acceptance') + os.environ['CONDA_PREFIX'] = str(self.prefix) + os.environ['PATH'] = str(self.prefix / 'bin') + ':' + os.environ.get('PATH', '') + self.socket_root = Path(tempfile.mkdtemp(prefix='crowdb-ctd-')) + self.socket = str(self.socket_root / 'containerd.sock') + self.runtime = Runtime('containerd', self.socket, 'crowdb-' + cluster, 'native') + self.process, self.log = None, None + + def __enter__(self): + docker_free(allow_container_host=self.docker_provisioned) + if os.geteuid() != 0: + raise ValueError('native host acceptance needs an explicitly privileged disposable Linux host') + binary = str(self.prefix / 'bin/containerd') + config = subprocess.run([binary, 'config', 'default'], capture_output=True, + text=True, check=True, timeout=10).stdout + # Retain the pinned runtime's complete defaults, including its diff + # service, while isolating every daemon socket and configuration import. + config = re.sub(r'(?m)^imports\s*=.*$', 'imports = []', config) + disabled = ["io.containerd.cri.v1.images", "io.containerd.cri.v1.runtime", + "io.containerd.grpc.v1.cri", "io.containerd.podsandbox.controller.v1.podsandbox", + "io.containerd.nri.v1.nri"] + config = re.sub(r'(?m)^disabled_plugins\s*=.*$', 'disabled_plugins = ' + json.dumps(disabled), config) + if '/run/containerd/containerd.sock' not in config: + raise ValueError('pinned containerd default socket configuration is unsupported') + config = config.replace('/run/containerd/containerd.sock', self.socket) + config += '\n[[plugins."io.containerd.transfer.v1.local".unpack_config]]\nplatform="linux/amd64"\nsnapshotter="native"\ndiffer="walking"\n' + path = self.root / 'config.toml' + path.write_text(config) + self.log = (self.root / 'daemon.log').open('w') + self.process = subprocess.Popen([binary, '--config', str(path), '--root', str(self.root / 'content'), + '--state', str(self.socket_root / 'state'), '--address', self.socket], + stdout=self.log, stderr=subprocess.STDOUT, start_new_session=True) + try: + deadline = time.monotonic() + 30 + while True: + if self.process.poll() is not None: + raise RuntimeError('owned containerd exited; see daemon.log') + try: + self.info = json.loads(self.runtime.invoke('info', '--format', '{{json .}}')) + break + except RuntimeError: + if time.monotonic() > deadline: + raise + time.sleep(.1) + return self + except BaseException: + self.close() + raise + + def close(self): + if self.process and self.process.poll() is None: + os.killpg(self.process.pid, signal.SIGTERM) + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(self.process.pid, signal.SIGKILL) + self.process.wait(timeout=5) + if self.log: + self.log.close() + + def __exit__(self, *_failure): + self.close() diff --git a/e2e-test/oci_reference_browser.py b/e2e-test/oci_reference_browser.py new file mode 100644 index 000000000..15b0e0f19 --- /dev/null +++ b/e2e-test/oci_reference_browser.py @@ -0,0 +1,43 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Existing rack editor flow against an unchanged reference deployment.""" + +import json +import os +import re +import shutil +import subprocess +import time + +TITLE = "datacenter root wraps racks; Add Rack from datacenter context menu" + + +def verify(cluster): + project = cluster.artifacts / "browser-client" + shutil.copytree(cluster.bundle / "clients/console-ui", project, + ignore=shutil.ignore_patterns("node_modules")) + env = dict(os.environ, NO_PROXY="*", no_proxy="*", + CROWDB_WEB_E2E_BASE_URL=cluster.endpoint(0), + CROWDB_WEB_E2E_OUTPUT=str((cluster.artifacts / "browser").resolve()), + CROWDB_WEB_E2E_REPORT=str((cluster.artifacts / "playwright.json").resolve())) + execute(cluster, project, env, "browser-prepare", ["npm", "ci", "--ignore-scripts"]) + execute(cluster, project, env, "browser", ["npx", "--no-install", "playwright", "test", + "--config=playwright.config.ts", "e2e/flows/10-cluster-rack-node.spec.ts", "--grep", re.escape(TITLE)]) + report = json.loads((cluster.artifacts / "playwright.json").read_text()) + stats = report["stats"] + if stats["expected"] != 1 or any(stats[field] for field in ("unexpected", "skipped", "flaky")): + raise AssertionError(f"reference browser did not pass exactly one flow: {stats}") + + +def execute(cluster, project, env, phase, arguments): + started = time.monotonic() + result = subprocess.run(["pixi", "run", *arguments], cwd=project, env=env, + capture_output=True, text=True, timeout=240) + output = result.stdout + result.stderr + (cluster.artifacts / f"client-{phase}.log").write_text(output) + (cluster.artifacts / f"client-{phase}.json").write_text(json.dumps({ + "case": cluster.case_name, "phase": phase, "execution": "host", + "seconds": time.monotonic() - started, "exit_code": result.returncode}, indent=2)) + print(output, end="", flush=True) + if result.returncode: + raise AssertionError(f"reference {phase} failed") diff --git a/e2e-test/oci_reference_fixture.py b/e2e-test/oci_reference_fixture.py new file mode 100644 index 000000000..764f89ddf --- /dev/null +++ b/e2e-test/oci_reference_fixture.py @@ -0,0 +1,136 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Reference Docker/Compose startup with host-only endpoint clients.""" + +import json +import os +from pathlib import Path +import subprocess +import time + +from console_cli_client import CliClient +from fixture import Cluster, redact, request +import oci_reference_protocols as protocols +import oci_reference_runtime as runtime +import oci_reference_browser as browser +import oci_reference_workloads as workloads +import oci_reference_recovery as recovery +import oci_shortcuts + + +class OciReferenceCluster(CliClient, Cluster): + def __init__(self, image, artifacts, bundle, reference, editable_ui=False, cli_workload=False, address_recovery=False, shortcuts=False, **kwargs): + implementation, topology = reference.split("-") + if implementation not in ("docker", "compose") or topology not in ("one", "three"): + raise ValueError("unsupported OCI reference case") + super().__init__(image, artifacts, node_count=1 if topology == "one" else 3, + wire_replicas=False, **kwargs) + self.implementation = implementation + self.editable_ui = editable_ui + self.cli_workload = cli_workload + self.address_recovery = address_recovery + self.shortcuts = shortcuts + self.bundle = Path(bundle).resolve() + self.root = (self.data_root / "client-runtime").resolve() + self.phase_number = 0 + self.runtime_accepted = False + self.expected_names = [f"{self.run_id}-node{index + 1}" for index in range(self.node_count)] + if implementation == "compose": + services = ["node"] if self.node_count == 1 else ["node1", "node2", "node3"] + self.expected_names = [f"{self.run_id}-{service}-1" for service in services] + + def start(self): + self.root.mkdir(parents=True, mode=0o700) + port_count = 34 if self.address_recovery else 24 + result = subprocess.run([str(self.bundle / "crowdb-cli"), "port-alloc", "--service", "web", + "--count", str(port_count), "--owner-pid", str(os.getpid())], capture_output=True, + text=True, timeout=10, check=True) + allocated = [int(port) for port in result.stdout.split()] + if len(allocated) != port_count or allocated != list(range(allocated[0], allocated[0] + port_count)): + raise AssertionError("reference fixture needs its reserved contiguous host ports") + self.port_base = allocated[0] + self.image_id = json.loads(self.command("image", "inspect", self.image))[0]["Id"] + runtime.start(self) + runtime.refresh(self) + return self + + def endpoint(self, index): + return f"http://127.0.0.1:{self.port_base + index * 10}" + + def ready_report(self): + report = json.loads(self.invoke("cluster", "check", "--wait", "180", "--json", endpoint=self.endpoint(0))) + assert report["phase"] == "ready" + return report + + def verify_reference(self): + report = self.ready_report() + if self.cli_workload: + workloads.pending_free(self) + return + if self.editable_ui: + browser.verify(self) + for index in range(self.node_count): + racks = request(self.endpoint(index), "/api/racks") + assert any(rack["id"] == 61 and rack["name"] == "Rack Sixty-One" for rack in racks) + return + cluster_id = report["observation"]["cluster_id"] + before = runtime.refresh(self) + private = [protocols.credentials(self, index) for index in range(self.node_count)] + witness = protocols.write_witnesses(self) + protocols.read_witnesses(self, witness) + for index in range(self.node_count): + observed = json.loads(self.invoke("cluster", "check", "--json", endpoint=self.endpoint(index))) + assert observed["phase"] == "ready" and observed["observation"]["cluster_id"] == cluster_id + assert not (self.root / "persistent/oci-clusters").exists() + runtime.recreate(self) + after_report = self.ready_report() + after = runtime.refresh(self) + assert after_report["observation"]["cluster_id"] == cluster_id + assert not {item["Id"] for item in before} & {item["Id"] for item in after} + assert [protocols.credentials(self, index) for index in range(self.node_count)] == private + protocols.read_witnesses(self, witness) + (self.artifacts / "reference-proof.json").write_text(json.dumps({ + "cluster_id": cluster_id, "implementation": self.implementation, + "nodes": self.node_count, "image_id": self.image_id, + "container_ids": [item["Id"] for item in after], + "published_ui_origins": [self.endpoint(index) for index in range(self.node_count)], + "peer_addresses_supplied": False, "runtime_receipt": False, + "s3_retained_reads": self.node_count * 2, + "iceberg_table_reads": self.node_count * 2, + "dataset_snapshot_reads": self.node_count * 2, + "editable_management": True, "recreated_retained_identity": True, + "retained_credentials": True}, indent=2)) + print("PASS OCI reference", self.implementation, self.node_count, cluster_id, flush=True) + if self.address_recovery: + recovery.verify(self, cluster_id, witness, private) + if self.shortcuts: + for name in self.expected_names: + self.command("stop", "--time", "30", name) + oci_shortcuts.verify(self) + + def diagnostics(self): + if not self.runtime_accepted: + return + for index in range(self.node_count): + try: + value = request(self.endpoint(index), "/api/cluster/status") + (self.artifacts / f"cluster-status-{index}.json").write_text(redact(value)) + except (OSError, ValueError) as error: + (self.artifacts / f"cluster-status-{index}.error").write_text(redact(str(error))) + for item in runtime.inventory(self, "container"): + runtime.validate_container(self, item) + target = self.artifacts / "runtime-logs" / item["Id"][:12] + target.mkdir(parents=True, exist_ok=True) + self.command("cp", f"{item['Id']}:/opt/crowdb/data/log", str(target)) + super().diagnostics() + + def close(self): + if self.closed: + return + started = time.monotonic() + runtime.close(self) + self.closed = True + self.timings["stop"] = time.monotonic() - started + if self.started is not None: + self.timings["total"] = time.monotonic() - self.started + self.write_timings() diff --git a/e2e-test/oci_reference_protocols.py b/e2e-test/oci_reference_protocols.py new file mode 100644 index 000000000..e313fed07 --- /dev/null +++ b/e2e-test/oci_reference_protocols.py @@ -0,0 +1,94 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Host protocol operations through the reference deployment's published ports.""" + +import json +import uuid +import urllib.error +import urllib.request + +from fixture import HTTP, redact, request + + +def call(origin, path, *, method="GET", body=None, token=None, expected=(200,)): + headers = {"Content-Type": "application/json"} + if token is not None: + headers["Authorization"] = "Bearer " + token + data = None if body is None else json.dumps(body).encode() + try: + with HTTP.open(urllib.request.Request(origin + path, data=data, headers=headers, + method=method), timeout=30) as response: + status, content = response.status, response.read() + except urllib.error.HTTPError as error: + status, content = error.code, error.read() + if status not in expected: + raise AssertionError(f"{method} {origin}{path}: HTTP {status}: {redact(content.decode(errors='replace'))[:2048]}") + return json.loads(content) if content else None + + +def credentials(cluster, index): + value = request(cluster.endpoint(index), "/api/access/credentials") + base = cluster.port_base + index * 10 + assert value["s3"]["endpoint"] == f"http://127.0.0.1:{base + 1}" + assert value["iceberg"]["uri"] == f"http://127.0.0.1:{base + 2}" + assert value["dataset"]["uri"] == f"http://127.0.0.1:{base + 3}" + return value + + +def write_witnesses(cluster): + # Normal management writes must be allowed in both one- and three-node UIs. + endpoint = cluster.endpoint(0) + cluster.invoke("cluster", "rack", "add", "--id", "71", "--name", "reference-edit", endpoint=endpoint) + assert "reference-edit" in cluster.invoke("cluster", "rack", "list", endpoint=endpoint) + cluster.invoke("s3", "bucket", "put", "reference-witness", endpoint=endpoint) + cluster.invoke("s3", "object", "put", "reference-witness", "retained", "--text", + "published-port-witness", endpoint=endpoint) + private = credentials(cluster, 0) + iceberg = private["iceberg"] + call(iceberg["uri"], "/v1/config", expected=(401, 403)) + call(iceberg["uri"], "/v1/namespaces", method="POST", token=iceberg["token"], + body={"namespace": ["reference"], "properties": {"witness": "persisted"}}, expected=(200, 201)) + table = call(iceberg["uri"], "/v1/namespaces/reference/tables", method="POST", token=iceberg["token"], + body={"name": "events", "schema": {"type": "struct", "schema-id": 0, + "fields": [{"id": 1, "name": "id", "type": "long", "required": True}]}}, + expected=(200, 201)) + committed = call(iceberg["uri"], "/v1/namespaces/reference/tables/events", method="POST", + token=iceberg["token"], body={"requirements": [{"type": "assert-table-uuid", + "uuid": table["metadata"]["table-uuid"]}], "updates": [{"action": "set-properties", + "updates": {"reference": "committed"}}]}) + assert committed["metadata"]["properties"]["reference"] == "committed" + dataset = private["dataset"] + path = "/api/dataset/v2/namespaces/dataset-ns/datasets/reference" + call(dataset["uri"], "/api/dataset/v2/capabilities", expected=(401, 403)) + created = call(dataset["uri"], path, method="PUT", token=dataset["token"], expected=(200, 201)) + assert created["name"] == "reference" and created["id"] + publication = {"operation": uuid.uuid4().hex, "parent": None, "manifest": { + "schema": [{"name": "value", "required": True}], "samples": [{"id": "c2FtcGxl", + "fields": {"value": {"kind": "inline", "data": "aGVsbG8="}}}]}} + published = call(dataset["uri"], path.replace("/v2/", "/v1/") + "/snapshots/publish", + method="POST", token=dataset["token"], body=publication) + assert published["state"] == "published" and published["is_latest"] + return {"dataset_id": created["id"], "dataset_snapshot": published["id"], + "dataset_manifest": publication["manifest"]} + + +def read_witnesses(cluster, witness): + for index in range(cluster.node_count): + endpoint = cluster.endpoint(index) + private = credentials(cluster, index) + assert cluster.invoke("s3", "object", "get", "reference-witness", "retained", endpoint=endpoint) == "published-port-witness" + assert "reference-edit" in cluster.invoke("cluster", "rack", "list", endpoint=endpoint) + iceberg = private["iceberg"] + namespace = call(iceberg["uri"], "/v1/namespaces/reference", token=iceberg["token"]) + assert namespace["namespace"] == ["reference"] + assert namespace["properties"]["witness"] == "persisted" + table = call(iceberg["uri"], "/v1/namespaces/reference/tables/events", token=iceberg["token"]) + assert table["metadata"]["schemas"][0]["fields"][0]["name"] == "id" + assert table["metadata"]["properties"]["reference"] == "committed" + dataset = private["dataset"] + opened = call(dataset["uri"], "/api/dataset/v2/namespaces/dataset-ns/datasets/reference", token=dataset["token"]) + assert opened["id"] == witness["dataset_id"] and opened["name"] == "reference" + assert opened["latest"] == witness["dataset_snapshot"] + manifest = call(dataset["uri"], "/api/dataset/v1/namespaces/dataset-ns/datasets/reference/snapshots/" + + witness["dataset_snapshot"] + "/manifest", token=dataset["token"]) + assert manifest == witness["dataset_manifest"] diff --git a/e2e-test/oci_reference_recovery.py b/e2e-test/oci_reference_recovery.py new file mode 100644 index 000000000..604ad5a8d --- /dev/null +++ b/e2e-test/oci_reference_recovery.py @@ -0,0 +1,198 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Confirmed nonvoter cold recovery across new names, addresses and an empty old bridge.""" + +import concurrent.futures +import json +from pathlib import Path +import socket +import time +import uuid +from urllib.parse import urlparse + +from fixture import HTTP, LABEL, request, wait +from profiles.node import request as admission_request +import oci_reference_runtime as runtime +import oci_reference_protocols as protocols + + +def origin(cluster, index): + return f'http://127.0.0.1:{cluster.port_base + index * 10}' + + +def create_nonvoter(cluster): + name = cluster.expected_names[3] + volume = runtime.node_volume(cluster, 3) + policy = cluster.artifacts / 'later-member.toml' + lines = ['version = 1', 'mode = "local"', f'cluster = "{cluster.run_id}"', + 'initial_voters = 3', '[storage]', 'mode = "diskless"'] + for kind in ('paxos-kv', 'diskdb', 'diskio', 'chunkdb', 'chunk-kv', 'access-server'): + lines += ['[[services]]', f'kind = "{kind}"', 'enabled = false'] + content = '\n'.join(lines) + '\n' + if policy.exists(): + assert policy.read_text() == content + else: + policy.write_text(content) + policy.chmod(0o644) + cluster.command('volume', 'create', '--label', f'{LABEL}={cluster.run_id}', volume) + cluster.command('create', '--name', name, '--label', f'{LABEL}={cluster.run_id}', + '--network', getattr(cluster, 'reference_network', cluster.run_id), + '--cpus', '0.25', '--memory', '768m', '--env', 'TOKIO_WORKER_THREADS=1', + '--mount', f'type=volume,source={volume},target=/opt/crowdb/data', + '--mount', f'type=bind,source={policy.resolve()},target=/run/later-member.toml,readonly', + '--env', 'CROWDB_NODE_POLICY_FILE=/run/later-member.toml', + '--env', f'CROWDB_CLUSTER={cluster.run_id}', '--env', 'CROWDB_INITIAL_VOTERS=3', + '--publish', f'127.0.0.1:{cluster.port_base + 30}:9090', cluster.image) + cluster.containers.append(name) + + +def read_json(cluster, name, path): + return json.loads(cluster.command('exec', '--user', '10001:10001', name, + 'cat', '/opt/crowdb/data/' + path)) + + +def identity(cluster, name): + accepted = read_json(cluster, name, 'accepted-node.json') + manifest = read_json(cluster, name, 'prepared-bootstrap.json') + pins = read_json(cluster, name, 'confirmed-member-trust.json') + assert len(manifest['nodes']) == len(pins['initial']['members']) == 3 + assert sorted(member['node_id'] for member in pins['current']['members']) == [1, 2, 3, 4] + assert manifest['identity'] == accepted['bootstrap'] == pins['current']['bootstrap'] + paths = ['node-identity', 'node-policy.json', 'prepared-bootstrap.json', 'member-trust.json', + 'ssh/id_ed25519.pub', 'ssh/ssh_host_ed25519_key.pub'] + if accepted.get('admission'): + operation = uuid.UUID(accepted['admission']['operation_id']) + paths.append(f'admitted-member-trust-{operation}.json') + hashes = cluster.command('exec', name, 'sha256sum', *['/opt/crowdb/data/' + path for path in paths]) + return dict(node_id=accepted['node_id'], bootstrap=accepted['bootstrap'], hashes=hashes) + + +def cluster_report(cluster, index): + # Match the public CLI's unchanged observation budget. Aggregate status may + # spend six seconds collecting an absent member; health reads use five. + # Connection errors remain retryable only inside the bounded read-only wait. + with HTTP.open(origin(cluster, index) + '/api/cluster/status', timeout=10) as response: + return json.load(response) + + +def authority(report): + return next((value for value in report['observation']['dependencies'] + if value['id'] == 'cluster/authority'), None) + + +def no_old_transports(addresses): + def unreachable(endpoint): + try: + with socket.create_connection(endpoint, timeout=.25): + raise AssertionError(f'old management transport remains reachable: {endpoint}') + except OSError: + return endpoint + endpoints = [(address, port) for address in addresses for port in (2222, 9090, 9095, 10000, 10100)] + with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: + return list(executor.map(unreachable, endpoints)) + + +def replace_addresses(cluster, before): + items = runtime.inventory(cluster, 'container') + for item in items: + runtime.validate_container(cluster, item) + runtime.validate_networks(cluster, items) + retained = [runtime.node_volume(cluster, index) for index in range(4)] + assert {item['Name'] for item in runtime.inventory(cluster, 'volume')} == set(retained) + network = cluster.run_id + '-recovered' + names = [name + '-cold' for name in cluster.expected_names] + (cluster.artifacts / 'cold-runtime-intent.json').write_text(json.dumps(dict( + old_names=cluster.expected_names, names=names, volumes=retained, + old_network=cluster.run_id, network=network, image_id=cluster.image_id), indent=2)) + for item in items: + cluster.command('stop', '--time', '30', item['Id']) + for item in runtime.inventory(cluster, 'container'): + runtime.validate_container(cluster, item) + assert not item['State']['Running'] + cluster.command('rm', item['Id']) + # Keep the empty original bridge so Docker cannot allocate its old subnet. + cluster.reference_networks = {cluster.run_id, network} + cluster.command('network', 'create', '--label', f'{LABEL}={cluster.run_id}', network) + cluster.reference_network = network + cluster.reference_volumes = retained + cluster.expected_names = names + cluster.containers = [] + runtime.create_docker_nodes(cluster, reversed(range(3))) + create_nonvoter(cluster) + created = runtime.inventory(cluster, 'container') + for item in created: + runtime.validate_container(cluster, item) + current = {item['NetworkSettings']['Networks'][network]['IPAddress'] for item in created} + # Docker may populate endpoint addresses only when the container starts. + assert not (current - {''}) & set(before) + old = next(item for item in runtime.inventory(cluster, 'network') if item['Name'] == cluster.run_id) + assert not old.get('Containers') + + +def verify(cluster, cluster_id, witness, credentials): + assert cluster.implementation == 'docker' and cluster.node_count == 3 + cluster.expected_names.append(cluster.run_id + '-node4') + create_nonvoter(cluster) + cluster.command('start', cluster.expected_names[3]) + local = wait(lambda: request(origin(cluster, 3), '/api/node/candidates'), + lambda value: bool(value.get('local')), 'later member discovery', seconds=30) + discovery = local['local']['advertisement']['discovery_id'] + wait(lambda: request(origin(cluster, 0), '/api/node/candidates'), + lambda value: any(row['advertisement']['discovery_id'] == discovery for row in value['candidates']), + 'later member multicast discovery', seconds=30) + # Reuse the host deployment client's existing admission bound. The short + # observation client must not cancel mutual SSH preparation mid-request. + admitted = admission_request(origin(cluster, 0), '/api/node/admit', dict(discovery_id=discovery, + rack_id=1,ssh_user='crowdb',ssh_port=2222,ssh_password='crowdb')) + assert admitted['node_id'] == 4 + wait(lambda: request(origin(cluster, 3), '/api/node/status'), + lambda value: value['phase'] == 'active', 'confirmed later nonvoter', seconds=180) + cluster.ready_report() + before = [identity(cluster, name) for name in cluster.expected_names] + assert before[3]['node_id'] == 4 + items = runtime.refresh(cluster) + old_ips = [item['NetworkSettings']['Networks'][cluster.run_id]['IPAddress'] for item in items] + for name in cluster.expected_names: + binding = read_json(cluster, name, 'node-binding.json') + assert all(urlparse(seed).hostname in old_ips for seed in binding['management_seeds']) + replace_addresses(cluster, old_ips) + cluster.command('start', cluster.expected_names[3]) + cluster.command('start', cluster.expected_names[1]) + wait(lambda: cluster_report(cluster, 3), + lambda value: bool(authority(value)), 'quorum-free fourth member management', seconds=30) + negative = [] + for _ in range(8): + observed = cluster_report(cluster, 3) + assert observed['phase'] != 'ready' and authority(observed)['state'] != 'ready', observed + negative.append(observed['phase']) + time.sleep(.5) + for name in (cluster.expected_names[3], cluster.expected_names[1]): + current = json.loads(cluster.command('inspect', name))[0] + runtime.validate_container(cluster, current) + assert current['State']['Running'], current['State'] + cluster.command('start', cluster.expected_names[0]) + partial = wait(lambda: cluster_report(cluster, 3), + lambda value: authority(value) and authority(value)['state'] == 'ready', + 'two original voters restore authority', seconds=180) + assert partial['phase'] != 'ready', partial + cluster.command('start', cluster.expected_names[2]) + report = cluster.ready_report() + assert report['observation']['cluster_id'] == cluster_id + after = [identity(cluster, name) for name in cluster.expected_names] + assert before == after + replicas = request(cluster.endpoint(0), '/api/stores/0/groups/0/replicas') + assert sorted(row['node_id'] for row in replicas) == [1, 2, 3] + items = runtime.refresh(cluster) + current_ips = [item['NetworkSettings']['Networks'][cluster.reference_network]['IPAddress'] for item in items] + assert not set(old_ips) & set(current_ips) + unreachable = no_old_transports(old_ips) + assert [protocols.credentials(cluster, index) for index in range(3)] == credentials + protocols.read_witnesses(cluster, witness) + (cluster.artifacts / 'cold-recovery-proof.json').write_text(json.dumps(dict( + cluster_id=cluster_id, identities=after, initial_voters=3, confirmed_contacts=4, + original_voter_ids=[1,2,3], negative_phases=negative, partial_phase=partial['phase'], + admission_submissions=1, + old_addresses=old_ips, current_addresses=current_ips, old_transports_unreachable=unreachable, + stable_names=False, peer_list=False, retained_credentials=True, + s3_retained_bytes=True, iceberg_committed_properties=True, dataset_snapshot_manifest=True), indent=2)) + print('PASS confirmed nonvoter cold recovery with new names/IPs and unchanged original votes', flush=True) diff --git a/e2e-test/oci_reference_runtime.py b/e2e-test/oci_reference_runtime.py new file mode 100644 index 000000000..1c024232b --- /dev/null +++ b/e2e-test/oci_reference_runtime.py @@ -0,0 +1,213 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Direct Docker and unchanged Compose examples with exact test ownership.""" + +import json +import os +from pathlib import Path +import subprocess + +from fixture import LABEL, request, wait + +ROOT = Path(__file__).resolve().parents[1] +PROJECT = "com.docker.compose.project" + + +def ports(base): + return {name: base + offset for offset, name in enumerate(("UI", "S3", "ICEBERG", "DATASET"))} + + +def compose_environment(cluster): + values = dict(os.environ, CROWDB_IMAGE=cluster.image, CROWDB_CLUSTER=cluster.run_id) + for index in range(cluster.node_count): + for name, port in ports(cluster.port_base + index * 10).items(): + key = f"CROWDB_NODE{index + 1}_{name}_PORT" if cluster.node_count == 3 else f"CROWDB_{name}_PORT" + values[key] = str(port) + return values + + +def compose(cluster, *arguments): + filename = "compose-three.yaml" if cluster.node_count == 3 else "compose.yaml" + command = ["docker", "compose", "--project-name", cluster.run_id, + "--file", str(ROOT / "container/oci-image" / filename), *arguments] + result = subprocess.run(command, env=compose_environment(cluster), capture_output=True, + text=True, timeout=120) + if result.returncode: + raise RuntimeError(f"Compose {arguments[0]} failed: {result.stdout}\n{result.stderr}") + return result.stdout + + +def inventory(cluster, kind): + label = PROJECT if cluster.implementation == "compose" else LABEL + args = ("ps", "-aq") if kind == "container" else (kind, "ls", "-q") + identities = cluster.command(*args, "--filter", f"label={label}={cluster.run_id}").splitlines() + if not identities: + return [] + return json.loads(cluster.command(kind, "inspect", *identities)) + + +def validate_container(cluster, item): + label = PROJECT if cluster.implementation == "compose" else LABEL + labels = item.get("Config", {}).get("Labels", {}) + if labels.get(label) != cluster.run_id or item.get("Image") != cluster.image_id: + raise ValueError("reference container ownership or image changed") + if cluster.implementation == "compose": + expected = {"node"} if cluster.node_count == 1 else {"node1", "node2", "node3"} + if labels.get("com.docker.compose.service") not in expected: + raise ValueError("unrelated service shares the reference Compose project") + if item["Name"].removeprefix("/") not in cluster.expected_names: + raise ValueError("unexpected reference container name") + + +def start(cluster): + for kind in ("container", "network", "volume"): + if inventory(cluster, kind): + raise ValueError("reference scope already contains runtime resources") + (cluster.artifacts / "runtime-intent.json").write_text(json.dumps({ + "project": cluster.run_id, "implementation": cluster.implementation, + "image_id": cluster.image_id, "nodes": cluster.node_count, + "port_base": cluster.port_base}, indent=2)) + cluster.runtime_accepted = True + if cluster.implementation == "compose": + cluster.command("compose", "version", "--short") + compose(cluster, "up", "--detach", "--no-build", "--pull", "never") + return + cluster.network_created = True + cluster.command("network", "create", "--label", f"{LABEL}={cluster.run_id}", cluster.run_id) + create_docker_nodes(cluster, range(cluster.node_count)) + start_docker_nodes(cluster) + + +def create_docker_nodes(cluster, indexes): + for index in indexes: + name = cluster.expected_names[index] + volume = node_volume(cluster, index) + cluster.command("volume", "create", "--label", f"{LABEL}={cluster.run_id}", volume) + cluster.containers.append(name) + arguments = ["create", "--name", name, "--label", f"{LABEL}={cluster.run_id}", + "--network", getattr(cluster, 'reference_network', cluster.run_id), "--cpus", "1", "--memory", "1536m", + "--mount", f"type=volume,source={volume},target=/opt/crowdb/data", + "--env", f"CROWDB_CLUSTER={cluster.run_id}", + "--env", f"CROWDB_INITIAL_VOTERS={cluster.node_count}"] + for protocol, port in ports(cluster.port_base + index * 10).items(): + internal = 9090 + (port - cluster.port_base - index * 10) + arguments += ["--publish", f"127.0.0.1:{port}:{internal}"] + if protocol != "UI": + arguments += ["--env", f"CROWDB_{protocol}_PUBLIC_URI=http://127.0.0.1:{port}"] + cluster.command(*arguments, cluster.image) + + +def start_docker_nodes(cluster): + # A different start order must not require a peer-address or bootstrap helper. + order = list(reversed(cluster.expected_names)) if cluster.node_count == 3 else cluster.expected_names + if cluster.node_count == 3: + for name in order[:2]: + cluster.command("start", name) + origin = cluster.endpoint(2) + wait(lambda: request(origin, "/api/cluster/status"), + lambda report: report["phase"] == "waiting", "two-node formation waiting for third voter") + report = json.loads(cluster.invoke("cluster", "check", "--wait", "2", "--json", + endpoint=origin, expected_exit=1)) + assert report["phase"] == "waiting" and report["observation"]["initial_voters"] == 3 + assert request(origin, "/api/nodes") == [] + assert request(origin, "/api/racks") == [] + for name in order[:2]: + for path in ("accepted-node.json", "prepared-bootstrap.json"): + cluster.command("exec", name, "test", "!", "-e", "/opt/crowdb/data/" + path) + (cluster.artifacts / "missing-peer-proof.json").write_text(json.dumps(report, indent=2)) + order = order[2:] + for name in order: + cluster.command("start", name) + + +def recreate(cluster): + items = inventory(cluster, "container") + for item in items: + validate_container(cluster, item) + validate_networks(cluster, items) + volumes = inventory(cluster, "volume") + if {item["Name"] for item in volumes} != expected_volumes(cluster): + raise ValueError("reference retained volumes are missing or have foreign ownership") + if cluster.implementation == "compose": + compose(cluster, "up", "--detach", "--force-recreate", "--no-build", "--pull", "never") + else: + for item in items: + cluster.command("rm", "--force", item["Id"]) + create_docker_nodes(cluster, reversed(range(cluster.node_count))) + for name in reversed(cluster.expected_names): + cluster.command("start", name) + + +def expected_volumes(cluster): + if cluster.implementation == "compose": + services = ["node"] if cluster.node_count == 1 else ["node1", "node2", "node3"] + return {f"{cluster.run_id}_{service}-data" for service in services} + return {node_volume(cluster, index) for index in range(len(cluster.expected_names))} + + +def node_volume(cluster, index): + retained = getattr(cluster, 'reference_volumes', None) + return retained[index] if retained is not None else f"{cluster.expected_names[index]}-data" + + +def validate_networks(cluster, containers): + owned_ids = {item["Id"] for item in containers} + expected_name = cluster.run_id + ("_default" if cluster.implementation == "compose" else "") + for network in inventory(cluster, "network"): + if any(identity not in owned_ids for identity in network.get("Containers", {})): + raise ValueError("reference network has an unrelated attachment") + if network.get("Name") not in getattr(cluster, 'reference_networks', {expected_name}): + raise ValueError("unexpected reference network name") + + +def refresh(cluster): + items = inventory(cluster, "container") + for item in items: + validate_container(cluster, item) + items.sort(key=lambda item: item["Name"]) + cluster.servers = [item["Name"].removeprefix("/") for item in items] + cluster.containers = list(cluster.servers) + if len(items) != len(cluster.expected_names): + raise AssertionError("reference deployment did not create the intended node count") + for item in items: + if not item["State"]["Running"]: + raise AssertionError("reference node exited: " + cluster.command("logs", item["Id"])) + env = item["Config"]["Env"] + assert not any(value.startswith("CROWDB_DISCOVERY_SEEDS=") for value in env) + assert all(not network.get("IPAMConfig", {}).get("IPv4Address") + for network in item["NetworkSettings"]["Networks"].values() + if network.get("IPAMConfig") is not None) + return items + + +def close(cluster): + if not cluster.runtime_accepted: + return + containers = inventory(cluster, "container") + for item in containers: + validate_container(cluster, item) + validate_networks(cluster, containers) + volumes = inventory(cluster, "volume") + if any(item["Name"] not in expected_volumes(cluster) for item in volumes): + raise ValueError("unrelated volume shares the reference scope") + if cluster.implementation == "compose": + # Compose's volume deletion follows YAML names without checking our owner. + compose(cluster, "down", "--timeout", "30") + else: + for item in containers: + current = json.loads(cluster.command("inspect", item["Id"]))[0] + validate_container(cluster, current) + cluster.command("rm", "-f", current["Id"]) + for network in inventory(cluster, "network"): + cluster.command("network", "rm", network["Id"]) + if cluster.timings["status"] == "pass": + for volume in inventory(cluster, "volume"): + if volume["Name"] not in expected_volumes(cluster): + raise ValueError("unrelated volume shares the reference scope") + cluster.command("volume", "rm", volume["Name"]) + assert not inventory(cluster, "container") and not inventory(cluster, "network") + remaining = inventory(cluster, "volume") + if cluster.timings["status"] == "pass" and remaining: + raise AssertionError("successful reference case leaked owned volumes") + (cluster.artifacts / "runtime-cleanup.json").write_text(json.dumps({ + "ok": True, "project": cluster.run_id, "retained_volumes": [item["Name"] for item in remaining]}, indent=2)) diff --git a/e2e-test/oci_reference_workloads.py b/e2e-test/oci_reference_workloads.py new file mode 100644 index 000000000..89c37a97b --- /dev/null +++ b/e2e-test/oci_reference_workloads.py @@ -0,0 +1,62 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Opt-in CLI accounting regression, owned by a normal disposable node.""" + +import json +import os +import subprocess +import time + +from fixture import redact, request, wait + + +def quiesce_chunk_allocators(cluster): + # The benchmark compares whole-group busy bytes. Its accounting requires + # exclusive allocation activity, including background strip reservation. + if cluster.node_count != 1: + raise ValueError("pending-free fixture requires its owned single node") + origin = cluster.endpoint(0) + before = request(origin, "/api/servers") + producers = [] + for kind in ("access-server", "chunk-kv", "chunkdb"): + services = [item for item in before if item["service_type"] == kind] + if len(services) != 1: + raise AssertionError(f"expected one owned {kind} before allocator workload") + producers.append(services[0]["id"]) + stopped = [] + for service_id in producers: + request(origin, f"/api/services/{service_id}/stop", {}) + wait(lambda: request(origin, "/api/servers"), + lambda items: any(item["id"] == service_id and item.get("pid") is None for item in items), + f"confirmed stop of allocator producer {service_id}") + stopped.append(service_id) + after = request(origin, "/api/servers") + preserved = ("paxos-kv", "diskdb", "diskio") + for item in before: + if item["service_type"] in preserved: + current = next(row for row in after if row["id"] == item["id"]) + assert current["pid"] == item["pid"] and current["pid"], current + (cluster.artifacts / "allocator-workload-precondition.json").write_text(json.dumps({ + "stopped_producers": stopped, "preserved_services": [ + {"id": item["id"], "pid": item["pid"]} for item in after + if item["service_type"] in preserved]}, indent=2)) + + +def pending_free(cluster): + quiesce_chunk_allocators(cluster) + name = "mixed_disk_workload_normalizes_frees_predating_its_baseline" + env = cluster.environment() + env.update(CROWDB_TEST_CLUSTER_MANAGEMENT_ENDPOINT=cluster.endpoint(0), + CROWDB_E2E_CLI_BIN=str(cluster.bundle / "crowdb-cli"), + LD_LIBRARY_PATH=str(cluster.bundle / "lib") + ":" + os.environ.get("LD_LIBRARY_PATH", "")) + started = time.monotonic() + result = subprocess.run([str(cluster.bundle / "cli-workload-test"), "--ignored", "--exact", name, + "--nocapture"], env=env, capture_output=True, text=True, timeout=120) + output = redact(result.stdout + result.stderr) + (cluster.artifacts / "client-pending-free.log").write_text(output) + (cluster.artifacts / "client-pending-free.json").write_text(json.dumps({ + "case": cluster.case_name, "phase": "pending-free-accounting", "execution": "host", + "seconds": time.monotonic() - started, "exit_code": result.returncode}, indent=2)) + print(output, end="", flush=True) + if result.returncode or "test result: ok. 1 passed; 0 failed;" not in output: + raise AssertionError("pending-free regression did not execute exactly one passing test") diff --git a/e2e-test/oci_shortcuts.py b/e2e-test/oci_shortcuts.py new file mode 100644 index 000000000..163342562 --- /dev/null +++ b/e2e-test/oci_shortcuts.py @@ -0,0 +1,109 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Real Pixi shortcuts without a usable CLI, including independent named roots.""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess + +from fixture import HTTP, request, wait + +ROOT = Path(__file__).resolve().parents[1] + + +def status(port): + with HTTP.open(f'http://127.0.0.1:{port}/api/cluster/status', timeout=10) as response: + return json.load(response) + + +def named_clusters(cluster, invoke, ready, container_ids, credentials, one, other, port): + for name, offset in ((one,0), (other,10)): + invoke('oci-run-1c', '--cluster', name, '--image', cluster.image, + '--port-base', str(port+offset)) + first, second = ready(0), ready(10) + first_id, second_id = first['observation']['cluster_id'], second['observation']['cluster_id'] + assert first_id != second_id + assert first['observation']['initial_voters']==second['observation']['initial_voters']==1 + original, unrelated = container_ids(one), container_ids(other) + credentials_before = credentials(0) + request(f'http://127.0.0.1:{port}', '/api/racks', {'id':67,'name':'Pixi retained'}) + invoke('oci-stop', '--cluster', one) + invoke('oci-clean', '--cluster', one) + assert not container_ids(one) and container_ids(other)==unrelated + assert ready(10)['observation']['cluster_id']==second_id + invoke('oci-run', '--cluster', one) + assert ready(0)['observation']['cluster_id']==first_id + assert container_ids(one)!=original and credentials(0)==credentials_before + assert any(row['id']==67 for row in request(f'http://127.0.0.1:{port}', '/api/racks')) + assert all(row['id']!=67 for row in request(f'http://127.0.0.1:{port+10}', '/api/racks')) + for name in (one, other): + invoke('oci-stop', '--cluster', name) + invoke('oci-clean', '--cluster', name) + return first_id, second_id, unrelated + + +def verify(cluster): + root = cluster.artifacts/'shortcut-runtime' + root.mkdir(mode=0o700) + poison = root/'no-cli' + poison.mkdir() + binary = poison/'crowdb-cli' + binary.write_text('#!/bin/sh\nprintf unexpected > "$CROWDB_TEST_CLI_MARKER"\nexit 97\n') + binary.chmod(0o755) + marker = root/'unexpected-cli' + environment = dict(os.environ, CROWDB_RUNTIME_ROOT=str(root.resolve()), + CROWDB_TEST_CLI_MARKER=str(marker.resolve()), PATH=str(poison.resolve())+':'+os.environ['PATH']) + actions = [] + + def invoke(task, *arguments, code=0): + result = subprocess.run(['pixi', 'run', task, *arguments], cwd=ROOT, env=environment, + text=True, capture_output=True, timeout=210) + (cluster.artifacts/f'shortcut-{len(actions)}.log').write_text(result.stdout+result.stderr) + actions.append(dict(task=task, arguments=arguments, exit_code=result.returncode)) + assert not marker.exists(), 'shortcut invoked crowdb-cli' + if result.returncode != code: + print(result.stdout+result.stderr, flush=True) + raise AssertionError(f'{task} exited {result.returncode}; expected {code}') + return result.stdout + + selected = invoke('sh', '-c', 'command -v crowdb-cli').strip() + assert selected == str(binary.resolve()), selected + prefix = 'pixi-' + cluster.run_id[-12:] + one, other, three, missing = [prefix+'-'+suffix for suffix in ('a', 'b', 'c', 'missing')] + port = cluster.port_base + + def ready(offset): + report = wait(lambda: status(port+offset), lambda row: row['phase']=='ready', + 'ordinary Pixi node preparation', seconds=180) + assert report['observation']['policy']['mode']=='local', report + return report + + def container_ids(name): + return cluster.command('ps', '-aq', '--filter', f'label=org.crowdb.cluster={name}').splitlines() + + def credentials(offset): + value = request(f'http://127.0.0.1:{port+offset}', '/api/access/credentials') + return hashlib.sha256(json.dumps(value,sort_keys=True).encode()).hexdigest() + + success = False + try: + invoke('oci-run', '--cluster', missing, '--image', 'crowdb-node:missing-'+cluster.run_id, code=1) + assert not container_ids(missing) + first_id, second_id, unrelated = named_clusters( + cluster, invoke, ready, container_ids, credentials, one, other, port) + invoke('oci-run-3c', '--cluster', three, '--image', cluster.image, '--port-base', str(port)) + report = ready(0) + assert report['observation']['initial_voters']==3 + assert len(container_ids(three))==3 + success = True + proof = dict(image_id=cluster.image_id, independent_clusters=[first_id,second_id], + three_node_cluster=report['observation']['cluster_id'], cli_invocations=0, + retained_credentials=True, retained_management_write=True, + unrelated_container_ids=unrelated, actions=actions) + finally: + invoke('oci-clean', '--all', *(['--delete-data'] if success else [])) + assert all(not container_ids(name) for name in (one,other,three,missing)) + (cluster.artifacts/'shortcuts-proof.json').write_text(json.dumps(proof,indent=2)) + print('PASS real Pixi one/three-node shortcuts, retained data and independent scoped cleanup without CLI',flush=True) diff --git a/e2e-test/profiles/bridge_node.py b/e2e-test/profiles/bridge_node.py index f0c89bfd4..7d1a6f317 100644 --- a/e2e-test/profiles/bridge_node.py +++ b/e2e-test/profiles/bridge_node.py @@ -28,9 +28,16 @@ def origin_for(name): return f"http://{address}:9090" -def start_node(name): +def start_node(name, *, default_setup=False): data = DATA / name.removeprefix(PREFIX + "-") data.mkdir(mode=0o755) + policy = DATA / (name.removeprefix(PREFIX + "-") + ("-default-policy.toml" if default_setup else "-policy.toml")) + kinds = ("paxos-kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access-server") + services = ", ".join(f'{{ kind = "{kind}", enabled = false }}' for kind in kinds) + if default_setup: + policy.write_text('[storage]\nmode = "automatic"\ncapacity_bytes = 17179869184\nreserve_bytes = 1073741824\nsparse = false\n') + else: + policy.write_text(f'services = [{services}]\n[storage]\nmode = "diskless"\n') docker("run", "--rm", "--name", PREFIX + "-prepare", "--network", "none", "--label", f"org.crowdb.e2e.run={PREFIX}", "--user", "0", "--cpu-shares", "1024", "--env", "TOKIO_WORKER_THREADS=1", "--memory", "32m", "--entrypoint", "/bin/chown", @@ -39,9 +46,14 @@ def start_node(name): docker("run", "-d", "--name", name, "--network", PREFIX, "--label", f"org.crowdb.e2e.run={PREFIX}", "--cpu-shares", "1024", "--env", "TOKIO_WORKER_THREADS=1", "--memory", "1g", - "--ulimit", "core=-1", "--pids-limit", "256", + "--ulimit", "core=-1", "--pids-limit", "256", "--security-opt", "seccomp=unconfined", "--mount", f"type=bind,source={data},target=/opt/crowdb/data", - "-e", "CROWDB_STARTUP_MODE=manual", + "--mount", f"type=bind,source={policy},target=/run/bridge-policy.toml,readonly", + "-e", "CROWDB_DEPLOYMENT_MODE=production", + "-e", "CROWDB_INITIAL_VOTERS=3", + "-e", f"CROWDB_CLUSTER={PREFIX}", + "-e", "CROWDB_NODE_POLICY_FILE=/run/bridge-policy.toml", + "-e", "CROWDB_SSH_PASSWORD=crowdb", "-e", "CROWDB_PHYSICAL_HOST_ID=one-docker-host", IMAGE) return origin_for(name) diff --git a/e2e-test/profiles/containerd-cluster.py b/e2e-test/profiles/containerd-cluster.py index d9a317241..06d76f47f 100644 --- a/e2e-test/profiles/containerd-cluster.py +++ b/e2e-test/profiles/containerd-cluster.py @@ -11,7 +11,6 @@ import urllib.request import uuid from pathlib import Path -from setup_choices import diskless_setup ROOT = Path(__file__).resolve().parents[2] HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) @@ -48,7 +47,7 @@ def wait(action, predicate, description, seconds=180): def control(host, operation): return docker("exec", host, "/bin/bash", "-euc", 'pixi run --as-is --manifest-path container/oci-image/pixi.toml -e runtime ' - 'python container/oci-image/control.py --runtime containerd --snapshotter native --address "$(cat /tmp/crowdb-containerd-address)" "$1" crowdb-node', + 'python container/oci-image/control.py --runtime containerd --snapshotter native --namespace "crowdb-$CROWDB_CONTAINERD_CLUSTER" --address "$(cat /tmp/crowdb-containerd-address)" "$1" "$(cat /tmp/crowdb-containerd-name)"', "_", operation) @@ -68,21 +67,30 @@ def main(): data = data_root / f"host{index}" data.mkdir(parents=True) docker("run", "-d", "--label", f"org.crowdb.e2e.run={prefix}", - "--cpu-shares", "1024", "--env", "TOKIO_WORKER_THREADS=1", "--memory", "1536m", "--privileged", "--cgroupns", "host", "--network", prefix, "--name", host, + "--cpu-shares", "1024", "--env", "TOKIO_WORKER_THREADS=1", "--memory", "2560m", "--privileged", "--cgroupns", "host", "--network", prefix, "--name", host, "--mount", f"type=bind,source={ROOT},target={ROOT},readonly", "--mount", f"type=bind,source={data},target=/opt/crowdb-e2e/data", "--mount", f"type=bind,source={archive},target=/opt/crowdb-e2e/archive.tar,readonly", "--mount", f"type=bind,source={archive}.json,target=/opt/crowdb-e2e/archive.tar.json,readonly", "--env", "CROWDB_CONTAINERD_DATA=/opt/crowdb-e2e/data", + "--mount", f"type=bind,source={os.environ['CROWDB_E2E_CLI_BUNDLE']},target=/opt/crowdb-e2e/client,readonly", + "--env", "CROWDB_E2E_CLI_BUNDLE=/opt/crowdb-e2e/client", + "--env", "CROWDB_CONTAINERD_CLUSTER=" + os.environ['CROWDB_CONTAINERD_CLUSTER'], "--env", "CROWDB_CONTAINERD_NODE_CPUS=" + os.environ.get("CROWDB_CONTAINERD_NODE_CPUS", "2"), "--mount", f"type=bind,source={pixi},target=/usr/local/bin/pixi,readonly", "--mount", "type=bind,source=/etc/ssl/certs,target=/etc/ssl/certs,readonly", "--workdir", str(ROOT), os.environ["CROWDB_CONTAINERD_HOST_IMAGE"], "/bin/bash", "-euc", '! command -v docker; test ! -e /var/run/docker.sock; ' + 'export DEBIAN_FRONTEND=noninteractive; apt-get update; ' + 'apt-get install -y --no-install-recommends iproute2; ' 'pixi run --as-is --manifest-path container/oci-image/pixi.toml -e runtime ' 'python e2e-test/profiles/containerd-host.py /opt/crowdb-e2e/archive.tar --serve') address = json.loads(docker("inspect", host))[0]["NetworkSettings"]["Networks"][prefix]["IPAddress"] origins.append("http://" + address + ":9090") + # Host creation includes archive unpacking. Finish each owned host's + # ordinary management startup before importing into another daemon. + wait(lambda: request(origins[-1], '/api/node/candidates'), + lambda value: bool(value['local']), 'containerd host management startup') snapshots = [wait(lambda origin=origin: request(origin, "/api/node/candidates"), lambda value: bool(value["local"]), "containerd host discovery") for origin in origins] identities = {snapshot["local"]["advertisement"]["discovery_id"] for snapshot in snapshots} @@ -91,7 +99,7 @@ def main(): lambda value: len(value["candidates"]) == 3, "cross-host discovery") request(origins[0], "/api/racks", {"id": 1, "name": "containerd-cluster"}) members = [request(origins[0], "/api/node/admit", {"discovery_id": identity, "rack_id": 1, - "ssh_user": "crowdb", "ssh_port": 2222, "ssh_password": "containerd-test", "setup": diskless_setup()})["node_id"] + "ssh_user": "crowdb", "ssh_port": 2222, "ssh_password": "oci-host-acceptance-password"})["node_id"] for identity in sorted(identities)] request(origins[0], "/api/cluster/init", {"nodes": members}) states = [wait(lambda origin=origin: request(origin, "/api/node/status"), @@ -116,16 +124,22 @@ def main(): finally: errors = [] for host in hosts: - try: - print(control(host, "logs"), flush=True) - print(docker("logs", host), flush=True) - except Exception as error: - errors.append(str(error)) - finally: + for kind, action in ( + ('inspect', lambda: docker('inspect', host)), + ('node', lambda: control(host, 'logs')), + ('host', lambda: docker('logs', host)), + ): try: - docker("rm", "-f", host) + output = action() + (data_root / f'{host}-{kind}.log').write_text(output) + print(output, flush=True) except Exception as error: + print(f'Containerd {kind} diagnostics failed for {host}: {error}', flush=True) errors.append(str(error)) + try: + docker("rm", "-f", host) + except Exception as error: + errors.append(str(error)) try: docker("network", "rm", prefix) except Exception as error: diff --git a/e2e-test/profiles/containerd-host.py b/e2e-test/profiles/containerd-host.py index 217c2424f..eac4a03ca 100644 --- a/e2e-test/profiles/containerd-host.py +++ b/e2e-test/profiles/containerd-host.py @@ -1,190 +1,103 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Real containerd host-network startup, bootstrap and persistence acceptance.""" +"""Docker-free production metadata lifecycle and Local protocol recreation.""" import argparse import json import os +from pathlib import Path import signal -import subprocess import sys -import tempfile import time -import urllib.request -from pathlib import Path -from setup_choices import diskless_setup -from node_resources import verify_reaping, verify_stopped -from supplied_media import Media, provision as provision_media, remove_and_rejoin, wait_for_descriptors -OCI = Path(__file__).resolve().parents[2] / "container/oci-image" - -sys.path.insert(0, str(OCI)) +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'e2e-test')) +sys.path.append(str(ROOT / 'container/oci-image')) from artifact import receipt -from builder import tool -from runtime import Runtime - - -def request(path, body=None): - data = None if body is None else json.dumps(body).encode() - req = urllib.request.Request("http://127.0.0.1:9090" + path, data=data, headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(req, timeout=10) as response: - return json.load(response) - - -def wait(action, predicate, description, seconds=120): - deadline = time.monotonic() + seconds - last = None - while time.monotonic() < deadline: - try: - last = action() - if predicate(last): - return last - except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error: - last = str(error) - time.sleep(0.25) - raise AssertionError(f"{description} failed: {last}") +from oci_native_runtime import NativeRuntime, docker_free +from oci_host_node import HostNode +from oci_local_host_node import LocalHostNode +from fixture import redact +from failure_logs import print_failure -def scenario(runtime, image, data, secret, mode): - media = Media(data.parent / "supplied-media") if mode == "manual" else None - media_state = None - command = [sys.executable, str(OCI / "run-node.py"), - "--runtime", "containerd", "--address", runtime.command[2], "--snapshotter", "native", "--image", image, - "--name", "crowdb-node", "--network", "host", "--data-root", str(data), - "--physical-host-id", "containerd-acceptance-host", "--interface", "eth0", - "--cpu-shares", "1024", "--worker-threads", "1", "--memory-mib", "1024", "--password-file", str(secret), "--startup-mode", mode] - if media: - command.extend(media.arguments()) +def metadata_runtime(root,bundle,image,address): + node = HostNode(root,bundle,image,'containerd',address=address,metadata_only=True) try: - subprocess.run(command, check=True, timeout=90) - verify_reaping(runtime.invoke, "crowdb-node", wait) - if mode == "manual": - snapshot = wait(lambda: request("/api/node/candidates"), lambda s: bool(s["local"]), "local discovery") - owner = runtime.invoke("exec", "crowdb-node", "stat", "-c", "%u:%a", "/opt/crowdb/data/disks").strip() - assert owner.split(":")[0] == "10001", f"supplied-media parent owner/mode: {owner}" - runtime.invoke("exec", "--user", "10001:10001", "crowdb-node", "test", "-w", "/opt/crowdb/data/disks") - discovery_id = snapshot["local"]["advertisement"]["discovery_id"] - request("/api/racks", {"id": 1, "name": "containerd-rack"}) - node = request("/api/node/admit", {"discovery_id": discovery_id, "rack_id": 1, - "ssh_user": "crowdb", "ssh_port": 2222, "ssh_password": "containerd-test", "setup": diskless_setup()}) - request("/api/cluster/init", {"nodes": [node["node_id"]]}) - state = wait(lambda: request("/api/node/status"), lambda s: s["phase"] == "active", "Group 0 bootstrap") - cluster = state["cluster_id"] - media_state = provision_media(media, data, node["node_id"], cluster, request, wait) - remove_and_rejoin(media, media_state, runtime, request, wait) - wait(lambda: runtime.invoke("exec", "crowdb-node", "crowdb-monitor", "readiness"), lambda _: True, "manual readiness") - else: - wait(lambda: runtime.invoke("exec", "crowdb-node", "crowdb-monitor", "readiness"), lambda _: True, "automatic readiness", 240) - authority = request("/api/authority") - assert authority["available"] and authority["source"] == "group0", authority - try: - request("/api/racks", {"id": 9, "name": "forbidden"}) - except urllib.error.HTTPError as error: - assert error.code == 403, error - else: - raise AssertionError("single-node mode accepted a management mutation") - key = runtime.invoke("exec", "crowdb-node", "cat", "/opt/crowdb/data/ssh/id_ed25519.pub") - runtime.invoke("stop", "--time", "30", "crowdb-node") - verify_stopped(runtime.invoke, "crowdb-node") - runtime.invoke("rm", "crowdb-node") - subprocess.run(command, check=True, timeout=90) - if mode == "manual": - restored = wait(lambda: request("/api/node/status"), lambda s: s["phase"] == "active", "persisted cluster") - assert restored["cluster_id"] == cluster - assert request("/api/node/candidates")["local"]["advertisement"]["discovery_id"] == discovery_id - wait(lambda: request("/api/servers"), - lambda services: any(row["service_type"] == "diskio" and row["health"] == "up" for row in services), - "retained DiskIO process readiness", 10) - wait_for_descriptors(media, media_state, runtime, request, wait) - media.probe(media_state, "verify") - assert not media.exclusive_available() - media.verify_tails() - print("containerd supplied file/raw: fsync/read, Offline claim release, Remove/Rejoin and recreated-container persistence passed", flush=True) - else: - wait(lambda: runtime.invoke("exec", "crowdb-node", "crowdb-monitor", "readiness"), lambda _: True, "persisted single-node readiness", 240) - assert runtime.invoke("exec", "crowdb-node", "cat", "/opt/crowdb/data/ssh/id_ed25519.pub") == key - runtime.verify("crowdb-node", 1, 1024, True, cpu_shares=1024) - print(f"containerd {mode}: host network, resource limits, initialization and persisted identity passed", flush=True) + node.start(fresh=True) + before = node.admit() + node.recreate() + after = node.metadata_report() + assert before['observation']['cluster_id'] == after['observation']['cluster_id'] + (root/'common-cli-metadata-proof.json').write_text(json.dumps(dict( + cluster_id=after['observation']['cluster_id'],production=True,diskless=True, + retained_recreation=True,common_cli=True),indent=2)) + except BaseException as error: + print_failure(root,[node.data],'deployment.containerd.cli',error,redact) + raise finally: - try: - group = next(line.split(":", 2)[2] for line in Path("/proc/self/cgroup").read_text().splitlines() if line.startswith("0::")) - print("Owned host memory events: " + (Path("/sys/fs/cgroup") / group.lstrip("/") / "memory.events").read_text(), flush=True) - except (OSError, StopIteration) as error: - print(f"Owned host memory diagnostic unavailable: {error}", flush=True) - try: - print(runtime.invoke("logs", "crowdb-node"), flush=True) - runtime.invoke("rm", "-f", "crowdb-node") - except (RuntimeError, OSError, ValueError, subprocess.SubprocessError) as error: - print(f"containerd diagnostic/cleanup: {error}", flush=True) - finally: - if media: - media.close() + node.cleanup() def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("archive", type=Path) - parser.add_argument("--serve", action="store_true", help="serve one manual node for isolated-host cluster acceptance") + parser.add_argument('archive', type=Path) + parser.add_argument('--serve', action='store_true', help='serve a diskless voter for the separately labeled Docker-provisioned cluster fixture') args = parser.parse_args() if os.geteuid() != 0: - parser.error("this isolated host acceptance requires explicit root privileges") + parser.error('this acceptance needs an explicitly privileged disposable Linux host') + docker_free(allow_container_host=args.serve) verified = receipt(args.archive) - with tempfile.TemporaryDirectory(prefix="crowdb-containerd-") as directory: - root = Path(directory) - socket = str(root / "containerd.sock") - config = root / "config.toml" - config.write_text('version = 4\ndisabled_plugins = ["io.containerd.cri.v1.images", "io.containerd.cri.v1.runtime", "io.containerd.grpc.v1.cri", "io.containerd.podsandbox.controller.v1.podsandbox"]\n') - command = [tool("containerd"), "--config", str(config), "--root", str(root / "content"), - "--state", str(root / "state"), "--address", socket] - diagnostics = Path(os.environ["CROWDB_CONTAINERD_DATA"]) if "CROWDB_CONTAINERD_DATA" in os.environ else root - diagnostics.mkdir(parents=True, exist_ok=True) - with (diagnostics / "daemon.log").open("w+") as log: - daemon = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) - runtime = Runtime("containerd", socket, snapshotter="native") - try: - wait(lambda: runtime.invoke("info"), lambda _: True, "containerd readiness", 30) - subprocess.run([tool("ctr"), "--address", socket, "--namespace", "crowdb", "images", "import", - "--local", "--snapshotter", "native", "--platform", "linux/amd64", "--base-name", "docker.io/crowdb/crowdb-acceptance", "--digests", str(args.archive.resolve())], check=True) - image = "docker.io/crowdb/crowdb-acceptance@" + verified["image_digest"] - if args.serve: - data = diagnostics / "node" - data.mkdir() - os.chown(data, 10001, 10001) - secret = diagnostics / "password" - secret.write_text("containerd-test\n") - secret.chmod(0o600) - Path("/tmp/crowdb-containerd-address").write_text(socket) - subprocess.run([sys.executable, str(OCI / "run-node.py"), - "--runtime", "containerd", "--address", socket, "--snapshotter", "native", - "--image", image, "--name", "crowdb-node", "--network", "host", - "--data-root", str(data), "--physical-host-id", os.uname().nodename, - "--interface", "eth0", "--cpu-shares", "1024", "--worker-threads", "1", "--memory-mib", "1024", - "--password-file", str(secret)], check=True, timeout=90) - while daemon.poll() is None: - time.sleep(1) - raise RuntimeError("containerd exited during cluster acceptance") - for mode in ("manual", "single"): - data = diagnostics / mode - data.mkdir() - os.chown(data, 10001, 10001) - secret = diagnostics / "password" - secret.write_text("containerd-test\n") - secret.chmod(0o600) - scenario(runtime, image, data, secret, mode) - print("Verified OCI digest: " + verified["image_digest"], flush=True) - finally: - os.killpg(daemon.pid, signal.SIGTERM) - try: - daemon.wait(timeout=10) - except subprocess.TimeoutExpired: - os.killpg(daemon.pid, signal.SIGKILL) - daemon.wait(timeout=5) - log.seek(0) - print(log.read(), end="", flush=True) + root = Path(os.environ['CROWDB_CONTAINERD_DATA']) + root.mkdir(parents=True, exist_ok=True, mode=0o700) + bundle = Path(os.environ['CROWDB_E2E_CLI_BUNDLE']) + cluster = os.environ.get('CROWDB_CONTAINERD_CLUSTER', 'oci-' + root.parent.name[-20:]) + os.environ['CROWDB_CONTAINERD_CLUSTER'] = cluster + with NativeRuntime(root / 'daemon', cluster, docker_provisioned=args.serve) as daemon: + daemon.runtime.invoke('load', '--input', str(args.archive.resolve())) + if args.serve: + node = HostNode(root,bundle,verified['image_digest'],'containerd',address=daemon.socket, + voters=3,metadata_only=True) + else: + metadata_runtime(root/'metadata',bundle,verified['image_digest'],daemon.socket) + node = LocalHostNode(root/'evaluation',bundle,verified['image_digest'],'containerd',address=daemon.socket) + try: + report = node.start(fresh=True) + before = node.facts() + (root / 'host-facts-before.json').write_text(json.dumps(before, indent=2)) + if args.serve: + Path('/tmp/crowdb-containerd-address').write_text(daemon.socket) + Path('/tmp/crowdb-containerd-name').write_text(node.record()['name']) + while daemon.process.poll() is None: + time.sleep(1) + raise RuntimeError('owned containerd exited during fixture') + witness, credentials = node.protocols() + node.recreate() + restored = json.loads(node.command(['cluster', 'check', '--wait', '180', '--json'])) + assert report['observation']['cluster_id'] == restored['observation']['cluster_id'] + _, after_credentials = node.protocols(witness) + assert credentials == after_credentials + after = node.facts() + assert before['container_id'] != after['container_id'] + for field in ('daemon_id','image','discovery_id','cpu_max','memory_max','host_network'): + assert before[field] == after[field], field + docker_free() + proof = dict(runtime='native rootful containerd', docker_client=False, docker_socket=False, + docker_daemon=False, image_manifest=verified['image_digest'], + policy='local',node_failure_tolerance=0,production_cli_metadata_recreation=True, + cluster_id=report['observation']['cluster_id'], before=before, after=after, + s3_retained_bytes=True, iceberg_committed_properties=True, dataset_snapshot_manifest=True) + (root / 'protocol-recreation-proof.json').write_text(json.dumps(proof, indent=2)) + print('Native containerd Local protocols and production metadata recreation passed: ' + verified['image_digest'], flush=True) + except BaseException as error: + print_failure(root, [node.data], 'deployment.containerd.native', error, redact) + raise + finally: + node.cleanup() -if __name__ == "__main__": +if __name__ == '__main__': def interrupt(_signal, _frame): - raise KeyboardInterrupt("containerd host interrupted") + raise KeyboardInterrupt('owned native host interrupted') signal.signal(signal.SIGTERM, interrupt) main() diff --git a/e2e-test/profiles/containerd-single-host.py b/e2e-test/profiles/containerd-single-host.py deleted file mode 100644 index a8eae3ac9..000000000 --- a/e2e-test/profiles/containerd-single-host.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2026-present Gian -# Licensed under the Apache License, Version 2.0. -"""Manual and single-node acceptance inside one disposable privileged Linux host.""" - -import json -import os -from pathlib import Path -import signal -import subprocess - -ROOT = Path(__file__).resolve().parents[2] - - -def docker(*args): - return subprocess.check_output(["docker", *args],text=True,timeout=60).strip() - - -def main(): - prefix = os.environ["CROWDB_CONTAINERD_PREFIX"] - archive = Path(os.environ["CROWDB_CONTAINERD_ARCHIVE"]) - data = Path(os.environ["CROWDB_CONTAINERD_DATA"]) - pixi = Path(subprocess.check_output(["which","pixi"],text=True).strip()).resolve() - name = prefix + "-host" - try: - docker("run","-d","--name",name,"--label",f"org.crowdb.e2e.run={prefix}", - "--cpu-shares", "1024", "--env", "TOKIO_WORKER_THREADS=1","--memory","1536m","--privileged","--cgroupns","host", - "--mount",f"type=bind,source={ROOT},target={ROOT},readonly", - "--mount",f"type=bind,source={pixi},target=/usr/local/bin/pixi,readonly", - "--mount","type=bind,source=/etc/ssl/certs,target=/etc/ssl/certs,readonly", - "--mount",f"type=bind,source={data},target=/opt/crowdb-e2e/data", - "--mount",f"type=bind,source={archive},target=/opt/crowdb-e2e/archive.tar,readonly", - "--mount",f"type=bind,source={os.environ['CROWDB_CONTAINERD_MEDIA_CLIENT']},target=/opt/crowdb-e2e/client,readonly", - "--mount",f"type=bind,source={archive}.json,target=/opt/crowdb-e2e/archive.tar.json,readonly", - "--env","CROWDB_CONTAINERD_DATA=/opt/crowdb-e2e/data","--workdir",str(ROOT), - os.environ["CROWDB_CONTAINERD_HOST_IMAGE"],"/bin/bash","-euc", - '! command -v docker; test ! -e /var/run/docker.sock; ' - 'pixi run --as-is --manifest-path container/oci-image/pixi.toml -e runtime ' - 'python e2e-test/profiles/containerd-host.py /opt/crowdb-e2e/archive.tar') - # The host/client environment is separate from the formal node image. - # Docker wait returns the worker's actual status, including failures. - status = int(subprocess.check_output(["docker","wait",name],text=True)) - print(docker("logs",name),flush=True) - if status: - raise RuntimeError(f"isolated containerd host acceptance exited {status}") - finally: - docker("rm","-f",name) - - -if __name__ == "__main__": - def interrupt(_signal, _frame): - raise KeyboardInterrupt("owned containerd host interrupted") - signal.signal(signal.SIGTERM,interrupt) - main() diff --git a/e2e-test/profiles/docker_media.py b/e2e-test/profiles/docker_media.py deleted file mode 100644 index 38a96bf20..000000000 --- a/e2e-test/profiles/docker_media.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2026-present Gian -# Licensed under the Apache License, Version 2.0. -"""Own one loop mapping in an isolated helper without granting the node privileges.""" - -import json -import os -from pathlib import Path -import shutil - -from node import docker, wait -from supplied_media import Media - -ROOT = Path(__file__).resolve().parents[2] - - -class DockerMedia(Media): - def __init__(self, root, name, image): - self.root = root / "supplied-media" - self.backing = self.root / "loop-backing.img" - self.file = self.root / "supplied-file.img" - self.bad = self.root / "supplied-nonblank.img" - self.raw = self.root / "supplied-loop" - self.name = name + "-media" - self.image = image - self.owner = name - self.host_root = root - self.descriptor = None - pixi = Path(shutil.which("pixi")).resolve() - try: - docker("run", "-d", "--init", "--name", self.name, "--label", f"org.crowdb.e2e.run={name}", - "--network", "none", "--privileged", "--entrypoint", "/bin/bash", - "--mount", f"type=bind,source={root},target=/owned", - "--mount", f"type=bind,source={ROOT},target={ROOT},readonly", - "--mount", f"type=bind,source={pixi},target=/usr/local/bin/pixi,readonly", - "--env", "CROWDB_OWNED_MEDIA_ROOT=/owned", "--env", f"CROWDB_OWNED_MEDIA_GID={os.getgid()}", - "--workdir", str(ROOT), image, "-euc", - 'exec pixi run --as-is --manifest-path container/oci-image/pixi.toml -e runtime ' - 'python e2e-test/profiles/supplied_media.py') - ready = root / "media-ready.json" - wait(ready.exists, bool, "owned Docker loop helper readiness") - observed = json.loads(ready.read_text()) - self.identity = tuple(observed["identity"]) - self.permissions = observed["permissions"] - self.descriptor = os.open(self.raw, os.O_RDONLY | os.O_NOFOLLOW | os.O_CLOEXEC) - self.status() - except BaseException: - self.close() - raise - - def close(self): - if self.descriptor is not None: - os.close(self.descriptor) - self.descriptor = None - cleanup_owned_helper(self.name, self.host_root, self.image, self.owner) - - def binding(self, data): - assert data == self.host_root / "node" - return json.loads(docker("exec", "--user", "10001:10001", self.owner, "cat", "/opt/crowdb/data/node-binding.json")) - - -def cleanup_owned_helper(name, root, image, owner): - names = docker("container", "ls", "-a", "--filter", f"name={name}", "--format", "{{.Names}}") - if name not in names.splitlines(): - return - current = json.loads(docker("inspect", name))[0] - if (current["Image"] != image or current["Config"].get("Labels", {}).get("org.crowdb.e2e.run") != owner - or not any(mount.get("Source") == str(root) and mount.get("Destination") == "/owned" - for mount in current.get("Mounts", []))): - raise RuntimeError("media cleanup refuses changed image, owner or data mount") - try: - docker("stop", "--time", "10", name) - stopped = json.loads(docker("inspect", name))[0] - logs = docker("logs", name) - if logs: - print(logs, flush=True) - assert stopped["State"]["ExitCode"] == 0, stopped["State"] - ready = root / "media-ready.json" - if ready.exists(): - assert json.loads((root / "media-closed.json").read_text()) == json.loads(ready.read_text()) - finally: - docker("rm", "-f", name) - - -class NodeRuntime: - def __init__(self, name): - self.name = name - - def invoke(self, *args): - return docker(*(self.name if argument == "crowdb-node" else argument for argument in args)) diff --git a/e2e-test/profiles/host-node.py b/e2e-test/profiles/host-node.py index 6c665df2e..dfc98889e 100644 --- a/e2e-test/profiles/host-node.py +++ b/e2e-test/profiles/host-node.py @@ -1,121 +1,127 @@ #!/usr/bin/env python3 # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. +"""Common CLI Docker host lifecycle, protocols and supplied-media durability.""" -"""Same-image host networking, explicit credentials/resources and persistent recovery.""" import json import os from pathlib import Path -import subprocess -import shutil -import sys -import tempfile -import uuid import signal -from contextlib import nullcontext -from node import docker, request, wait, IMAGE -from setup_choices import diskless_setup +import sys + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'e2e-test')) +from fixture import wait, redact +from failure_logs import print_failure +from oci_host_node import HostNode +from oci_host_policy import accept_candidates +from oci_local_host_node import LocalHostNode +from node import request from node_resources import verify_reaping, verify_stopped -from docker_media import DockerMedia, NodeRuntime -from supplied_media import provision as provision_media, remove_and_rejoin, wait_for_descriptors +from supplied_media import Media, provision, remove_and_rejoin, wait_for_descriptors +from host_runtime import HostRuntime -def main(): - name = os.environ.get("CROWDB_E2E_PROFILE_NAME", "crowdb-host-test-" + uuid.uuid4().hex[:8]) - with (nullcontext(os.environ["CROWDB_E2E_PROFILE_DATA"]) if "CROWDB_E2E_PROFILE_DATA" in os.environ - else tempfile.TemporaryDirectory(prefix="crowdb-host-node-")) as directory: - root = Path(directory) - secret = root / "password" - secret.write_text("host-ssh-acceptance") - secret.chmod(0o600) - data = root / "node" - docker("run", "--rm", "--network", "none", "--entrypoint", "/bin/sh", "--mount", - f"type=bind,source={root},target=/test", IMAGE, "-c", - 'mkdir /test/node; chown "10001:$1" /test/node; chmod 0770 /test/node', "sh", str(os.getgid())) - image = json.loads(docker("image", "inspect", IMAGE))[0]["Id"] - command = [sys.executable, "container/single-node-container/run-node.py", "--image", image, - "--name", name, "--network", "host", "--data-root", str(data), "--physical-host-id", "docker-acceptance-host", - "--interface", "lo", "--cpu-shares", "1024", "--worker-threads", "1", "--memory-mib", "1024", "--password-file", str(secret)] - media = None - runtime = NodeRuntime(name) - api = lambda path, body=None: request("http://127.0.0.1:9090", path, body) - observe = lambda action, predicate, description, seconds=10: wait(action, predicate, description, budget=seconds) - def launch(): - # Only the owned fixture operator writes launcher receipts. SSH must - # subsequently see a parent directory without group write permission. - docker("exec", media.name, "chmod", "0770", "/owned/node") - try: - subprocess.run(command, check=True, timeout=60) - finally: - docker("exec", media.name, "chmod", "0755", "/owned/node") +class NodeRuntime: + def __init__(self, runtime, name): + self.runtime, self.name = runtime, name + + def invoke(self, *arguments): + return self.runtime.invoke(*(self.name if value == 'crowdb-node' else value for value in arguments)) + +def protocols(root, bundle, image): + node = LocalHostNode(root, bundle, image, 'docker') + try: + report = node.start(fresh=True) + before = node.facts() + runtime = node.runtime() + name = node.record()['name'] + verify_reaping(runtime.invoke, name, wait) + witness, credentials = node.protocols() + runtime.invoke('stop','--time','30',name) + verify_stopped(runtime.invoke, name) + node.recreate() + restored = json.loads(node.command(['cluster', 'check', '--wait', '180', '--json'])) + assert report['observation']['cluster_id'] == restored['observation']['cluster_id'] + _, current = node.protocols(witness) + assert current == credentials + after = node.facts() + assert before['container_id'] != after['container_id'] + for field in ('daemon_id', 'image', 'discovery_id', 'host_network', 'cpu_max', 'memory_max'): + assert before[field] == after[field], field + (root / 'protocol-recreation-proof.json').write_text(json.dumps(dict( + before=before, after=after, cluster_id=report['observation']['cluster_id'], + s3_retained_bytes=True, iceberg_committed_properties=True, dataset_snapshot_manifest=True), indent=2)) + except BaseException as error: + print_failure(root, [node.data], 'deployment.host.protocols', error, redact) + raise + finally: + node.cleanup() + + +def supplied(root, bundle, image): + root.mkdir(mode=0o700) + os.environ['CROWDB_OWNED_MEDIA_ROOT'] = str(root) + media = Media(root / 'supplied-media') + node = None + try: + node = HostNode(root, bundle, image, 'docker', metadata_only=True, media=media) + # This fixture tests manual candidate admission after a genuine accepted + # diskless policy. Fresh populated automatic/diskless roots must reject. + accept_candidates(node) + node.start(fresh=True) + record = node.record() + os.environ['CROWDB_MEDIA_HOST_ID'] = record['options']['physical_host_id'] + runtime = HostRuntime(record['options'], node.cluster) + api = lambda path, body=None: request(node.origin, path, body) + def observe(action, predicate, description, seconds=10): + return wait(action, predicate, description, seconds=seconds) + report = node.admit() + active = api('/api/node/status') + accepted = json.loads((node.data / 'accepted-node.json').read_text()) + state = provision(media, node.data, accepted['node_id'], active['cluster_id'], api, observe) + owned = NodeRuntime(runtime, record['name']) + remove_and_rejoin(media, state, owned, api, observe) + key = (node.data / 'ssh/id_ed25519.pub').read_bytes() + node.recreate() + restored = api('/api/node/status') + assert restored['cluster_id'] == active['cluster_id'] == report['observation']['cluster_id'] + assert (node.data / 'ssh/id_ed25519.pub').read_bytes() == key + wait_for_descriptors(media, state, owned, api, observe) + media.probe(state, 'verify') + assert not media.exclusive_available() + media.verify_tails() + cleanup = api('/api/node/cleanup', dict(operation_id=active['operation_id'], confirm_delete_system_store=True)) + assert cleanup['pending'] == [], cleanup + (root / 'supplied-media-proof.json').write_text(json.dumps(dict( + retained_bytes=True, offline_releases_raw=True, remove_rejoin_preserves_id=True, + original_file_and_raw_descriptors=True, physical_tails_unchanged=True, + cluster_id=active['cluster_id']), indent=2)) + except BaseException as error: + print_failure(root, [node.data] if node else [], 'deployment.host.supplied', error, redact) + raise + finally: try: - media = DockerMedia(root, name, image) - command.extend(media.arguments()) - oversized = command.copy() - oversized[oversized.index("--memory-mib") + 1] = str(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") // (1024 * 1024) + 1) - rejected = subprocess.run(oversized, text=True, capture_output=True, timeout=60) - assert rejected.returncode != 0 and "boundaries must fit the host" in rejected.stderr, rejected.stderr - launch() - origin = "http://127.0.0.1:9090" - snapshot = wait(lambda: request(origin, "/api/node/candidates"), lambda result: bool(result["local"]), "host-network discovery") - owner = docker("exec", name, "stat", "-c", "%u:%a", "/opt/crowdb/data/disks") - assert owner.split(":")[0] == "10001", f"supplied-media parent owner/mode: {owner}" - docker("exec", "--user", "10001:10001", name, "test", "-w", "/opt/crowdb/data/disks") - verify_reaping(docker, name, observe) - discovery_id = snapshot["local"]["advertisement"]["discovery_id"] - assert snapshot["local"]["hardware"]["memory_bytes"] <= 1024 * 1024 * 1024 - assert snapshot["local"]["hardware"]["logical_cpus"] > 0 - request(origin, "/api/racks", {"id":1,"name":"host-rack"}) - record = request(origin, "/api/node/admit", {"discovery_id":discovery_id,"rack_id":1, - "ssh_user":"crowdb","ssh_port":2222,"ssh_password":"host-ssh-acceptance", "setup":diskless_setup()}) - request(origin, "/api/cluster/init", {"nodes":[record["node_id"]]}) - active = wait(lambda: request(origin, "/api/node/status"), lambda state: state["phase"] == "active", "Group 0 bootstrap") - assert active["phase"] == "active", active - media_state = provision_media(media, data, record["node_id"], active["cluster_id"], api, observe) - remove_and_rejoin(media, media_state, runtime, api, observe) - key = docker("exec", name, "cat", "/opt/crowdb/data/ssh/id_ed25519.pub") - docker("stop", "--time", "30", name) - verify_stopped(docker, name) - docker("rm", name) - launch() - recovered = wait(lambda: request(origin, "/api/node/status"), lambda result: result["phase"] == "active", "host persistent recovery") - assert recovered["cluster_id"] == active["cluster_id"] - assert request(origin, "/api/node/candidates")["local"]["advertisement"]["discovery_id"] == discovery_id - assert docker("exec", name, "cat", "/opt/crowdb/data/ssh/id_ed25519.pub") == key - wait_for_descriptors(media, media_state, runtime, api, observe) - media.probe(media_state, "verify") - assert not media.exclusive_available() - media.verify_tails() - cleanup = request(origin, "/api/node/cleanup", {"operation_id":active["operation_id"],"confirm_delete_system_store":True}) - assert cleanup["pending"] == [], cleanup - print("Host networking, explicit credentials/resources and same-image persistent recovery passed", flush=True) - except Exception: - artifact = Path(".crowdb-runtime/artifacts") / name - artifact.mkdir(parents=True, exist_ok=True) - print(docker("logs", name), flush=True) - logs = data / "log" - if logs.exists(): - shutil.copytree(logs, artifact / "logs") - try: - topology = docker("exec", name, "curl", "-fsS", "http://127.0.0.1:10000/topology") - (artifact / "topology.json").write_text(topology) - print(topology, flush=True) - except RuntimeError as error: - print(f"Host topology diagnostic unavailable: {error}", flush=True) - print(f"Host failure logs retained at {artifact.resolve()}", flush=True) - raise + if node: + node.cleanup() finally: - docker("rm", "-f", name) - if media: - media.close() - if "CROWDB_E2E_PROFILE_DATA" not in os.environ: - docker("run", "--rm", "--network", "none", "--entrypoint", "/bin/chmod", "--mount", - f"type=bind,source={root},target=/test", IMAGE, "-R", "0777", "/test/node") + media.close() + + +def main(): + if os.geteuid() != 0: + raise RuntimeError('host acceptance requires an explicitly privileged disposable Linux host') + root = Path(os.environ['CROWDB_E2E_PROFILE_DATA']) + bundle = Path(os.environ['CROWDB_E2E_CLI_BUNDLE']) + image = os.environ['CROWDB_CONTAINER_IMAGE'] + protocols(root / 'protocols', bundle, image) + supplied(root / 'supplied', bundle, image) + print('Docker host Local protocols, production metadata recreation and supplied-media lifecycle passed', flush=True) -if __name__ == "__main__": +if __name__ == '__main__': def interrupt(_signal, _frame): - raise KeyboardInterrupt("owned host profile interrupted") + raise KeyboardInterrupt('owned host acceptance interrupted') signal.signal(signal.SIGTERM, interrupt) main() diff --git a/e2e-test/profiles/node-admission.py b/e2e-test/profiles/node-admission.py index 6063bdb90..5a1589976 100644 --- a/e2e-test/profiles/node-admission.py +++ b/e2e-test/profiles/node-admission.py @@ -4,6 +4,9 @@ """Interrupted SSH admission and cross-UI managed-service lifecycle.""" import json +import os +from pathlib import Path +import subprocess from bridge_node import start_node, origin_for, CLIENT, BROWSER_OUTPUT import urllib.error from setup_choices import diskless_setup @@ -30,6 +33,20 @@ def verify(docker, request, wait, IMAGE, PREFIX, names, origins, identities, rec if node["discovery_id"] == admission_body["discovery_id"]) binding = json.loads(docker("exec", names[1], "cat", "/opt/crowdb/data/node-binding.json")) grant = {"operation_id": pending["operation_id"], "management_seeds": binding["management_seeds"]} + # Reproduce the authenticated pin that precedes KV preparation. The owned + # fixture supplies this public key out of band before interrupting admission. + observed = request(origins[1], "/api/stores/0/groups/0/kv/get?key=%2Fdeployment%2Fnodes") + assert observed["found"], observed + registry = json.loads(observed["value_utf8"]) + member = next(node for node in registry["nodes"] if node["discovery_id"] == pending["discovery_id"]) + assert not member["confirmed"] and not member.get("host_key"), member + host_key = docker("exec", fourth, "cat", "/opt/crowdb/data/ssh/ssh_host_ed25519_key.pub") + member["host_key"] = " ".join(host_key.split()[:2]) + bundle = Path(os.environ["CROWDB_E2E_CLI_BUNDLE"]) + env = dict(os.environ, LD_LIBRARY_PATH=str(bundle / "lib") + ":" + os.environ.get("LD_LIBRARY_PATH", "")) + subprocess.run(["pixi", "run", str(bundle / "crowdb-cli"), "--management-endpoint", origins[1], + "kv", "data", "put", "--store", "0", "--group", "0", "--key", "/deployment/nodes", + "--value", json.dumps(registry)], env=env, check=True, timeout=30) # Interrupt after KV preparation but before hardware confirmation/binding. public_key = docker("exec", names[1], "cat", "/opt/crowdb/data/ssh/id_ed25519.pub") install = {"command": "install_key", "operation_id": pending["operation_id"], "public_key": public_key} @@ -60,8 +77,11 @@ def verify(docker, request, wait, IMAGE, PREFIX, names, origins, identities, rec def services(docker, request, wait, origins, fourth, joined): + diskdb = request(origins[0], f"/api/nodes/{joined['node_id']}/diskdb/deploy", + {"instance_id": 71, "rpc_port": 19070}) + assert diskdb["pid"], diskdb service = request(origins[0], f"/api/nodes/{joined['node_id']}/services/deploy", - {"kind":"diskio", "instance_id":71, "rpc_port":19071}) + {"kind":"diskio", "instance_id":71, "rpc_port":19072}) assert service["pid"] wait(lambda: json.loads(docker("exec", fourth, "cat", "/opt/crowdb/run/status/monitor.json")), lambda status: status["services"].get("diskio-71", {}).get("healthy") is True, "remote DiskIO application readiness") diff --git a/e2e-test/profiles/node-containers.py b/e2e-test/profiles/node-containers.py index 4344b63e4..51501d1f9 100644 --- a/e2e-test/profiles/node-containers.py +++ b/e2e-test/profiles/node-containers.py @@ -142,6 +142,16 @@ def main(): for origin in origins: wait(lambda: request(origin, "/api/node/status"), lambda result: result["phase"] == "unbound_draft", "cleanup releases UI binding") print("Shared rack mutations, minority gating and explicit cleanup passed", flush=True) + # Default setup is a separate policy acceptance. Preserve each retired + # root, then launch fresh owned roots with the full storage policy. + for index in range(3): + docker("stop", "--time", "10", names[index]) + docker("rm", names[index]) + (DATA / str(index)).rename(DATA / f"retired-{index}") + origins[index] = start_node(names[index], default_setup=True) + wait(lambda index=index: request(origins[index], "/api/node/candidates"), + lambda result: bool(result["local"]), "fresh default-policy node") + request(origins[index], "/api/node/auto-join", {"enabled": False}) scenario("node-default-setup.py").verify(docker, request, wait, names, origins) resources = capture_resources(docker, names, DATA / "node-resources.json") for name, state in resources.items(): diff --git a/e2e-test/profiles/node-default-setup.py b/e2e-test/profiles/node-default-setup.py index 0ee5171ae..30c3cfbdb 100644 --- a/e2e-test/profiles/node-default-setup.py +++ b/e2e-test/profiles/node-default-setup.py @@ -23,6 +23,11 @@ def verify(docker, request, wait, names, origins): steps[node] = wait(lambda node=node: request(selected[0], f"/api/nodes/{node}/setup"), lambda status: next(step for step in status["services"] if step["kind"] == kind)["phase"] == "ready", f"default node {node} {kind} ready without browser") + owner_key = "/api/stores/0/groups/0/kv/get?key=%2Fdeployment%2Fdefault-preparation" + ownership = json.loads(request(selected[0], owner_key)["value_utf8"]) + assert not ownership["retired"] and ownership["metadata"]["group_id"] == 1, ownership + assert sorted(replica["node_id"] for replica in ownership["metadata"]["replicas"]) == sorted( + record["node_id"] for record in records), ownership operations = {node: {step["kind"]: step["operation_id"] for step in status["services"] if step["operation_id"]} for node, status in steps.items()} files = {} @@ -54,6 +59,7 @@ def verify(docker, request, wait, names, origins): for step in status["services"]: if step["kind"] in operations[node]: assert step["operation_id"] == operations[node][step["kind"]], step + assert json.loads(request(selected[0], owner_key)["value_utf8"]) == ownership for node, file in files.items(): status = request(selected[0], f"/api/nodes/{node}/setup") assert status["phase"] == "ready" and status["storage_mode"] == "automatic", status diff --git a/e2e-test/profiles/node-recovery.py b/e2e-test/profiles/node-recovery.py index 15f903020..7344993f1 100644 --- a/e2e-test/profiles/node-recovery.py +++ b/e2e-test/profiles/node-recovery.py @@ -53,7 +53,8 @@ def verify(docker, request, wait, image, network, names, origins, cluster): node["advertisement"]["discovery_id"] == identity and node["state"] == "same_cluster" and changed in str(node["advertisement"]["monitor_endpoints"]) for node in result["candidates"]), "persistent node new endpoint") assert docker("exec", name, "cat", "/opt/crowdb/data/ssh/id_ed25519.pub") == key - browser(origins[0], "node update", "update", CROWDB_NODE_UI_UPDATE_UUID=identity) + browser(origins[0], "node update", "update", CROWDB_NODE_UI_UPDATE_UUID=identity, + CROWDB_NODE_UI_OLD_HOST=old, CROWDB_NODE_UI_NEW_HOST=changed) for peer in origins: updated = next(node for node in request(peer, "/api/node/admissions") if node["discovery_id"] == identity) assert updated["node_id"] == joined["node_id"] and updated["rack_id"] == 2 and updated["host"] == changed, updated diff --git a/e2e-test/profiles/supplied_media.py b/e2e-test/profiles/supplied_media.py index 7ffbdf12f..4eda0d679 100644 --- a/e2e-test/profiles/supplied_media.py +++ b/e2e-test/profiles/supplied_media.py @@ -164,14 +164,14 @@ def outcomes(): "supplied media waits for actual DiskIO and rejects nonblank bytes", seconds=10) assert "not blank" in waiting[2]["error"], waiting defaults = request(f"/api/deployment-defaults?node_id={node}")["diskio"] - defaults.update(kind="diskio", instance_id=str(node), disk_group_id=1, rpc_port=7200, test_single_node=False) + defaults.update(kind="diskio", instance_id=str(node), disk_group_id=1, rpc_port=7200, local_deployment=False) request(f"/api/nodes/{node}/services/deploy", defaults) completed = wait(outcomes, lambda rows: [row["phase"] for row in rows] == ["registered", "registered", "failed"], "real file/raw descriptor verification", seconds=10) assert not media.exclusive_available(), "DiskIO did not retain its exclusive raw-device claim" binding = media.binding(data) assert binding["bootstrap"]["cluster_id"] == cluster and binding["node_id"] == node - state = {"version": 1, "rack_id": 1, "node_id": node, "disk_group_id": 1, + state = {"version": 1, "cluster_id": cluster, "rack_id": 1, "node_id": node, "disk_group_id": 1, "physical_host_id": os.environ.get("CROWDB_MEDIA_HOST_ID", "containerd-acceptance-host"), "seeds": binding["management_seeds"], "disks": completed[:2]} media.probe(state, "write") diff --git a/e2e-test/release.py b/e2e-test/release.py index 24de94c39..cfae06763 100644 --- a/e2e-test/release.py +++ b/e2e-test/release.py @@ -89,15 +89,21 @@ def check(case, arguments, expected, mounts=(), environment=(), entrypoint=None) check("invalid-profile", [], "failed to decode deployment profile", mounts=( f"type=bind,source={profile.resolve()},target=/opt/crowdb/etc/selected.toml,readonly",), environment=("CROWDB_PROFILE_PATH=/opt/crowdb/etc/selected.toml",)) - source = Path(__file__).resolve().parents[1] / "container/single-node-container/profile.toml" + repository = Path(__file__).resolve().parents[1] + source = (repository / "container/oci-image/node-profile.toml").read_text() + catalog_error = "node profile requires the complete service catalog without preset topology or media" for case, body, expected in ( - ("unknown-selected-service", source.read_text().replace('id = "web"', 'id = "unknown"'), "unknown release service"), - ("missing-selected-dependency", source.read_text().replace('dependencies = ["kv"]', 'dependencies = []'), "release service dependency is absent")): + ("unknown-selected-service", source.replace('[[services]]\nid = "web"', + '[[services]]\nid = "unknown"', 1), catalog_error), + ("missing-selected-dependency", source.replace('dependencies = ["kv"]', + 'dependencies = ["missing"]', 1), "service diskdb has an invalid dependency"), + ("preset-selected-topology", (repository / "container/single-node-container/profile.toml").read_text(), + catalog_error)): profile = root / f"{case}.toml" profile.write_text(body) check(case, [], expected, mounts=( f"type=bind,source={profile.resolve()},target=/opt/crowdb/etc/selected.toml,readonly",), - environment=("CROWDB_PROFILE_PATH=/opt/crowdb/etc/selected.toml", "CROWDB_STARTUP_MODE=single")) + environment=("CROWDB_PROFILE_PATH=/opt/crowdb/etc/selected.toml",)) for path in ("node-identity", "ssh", "bootstrap", "kv"): if (root / case / path).exists(): raise AssertionError(f"{case}: invalid selection created {path}") diff --git a/e2e-test/report.py b/e2e-test/report.py index f16ecf493..da132ec0f 100644 --- a/e2e-test/report.py +++ b/e2e-test/report.py @@ -14,7 +14,7 @@ def write_report(root, timings): fixture_path = directory / "timing.json" fixture = json.loads(fixture_path.read_text()) if fixture_path.exists() else {} completed = fixture.get("status") == "pass" and bool(phases) and all( - phase.get("exit_code") == 0 for phase in phases) + phase.get("exit_code") == phase.get("expected_exit_code", 0) for phase in phases) actions = [json.loads(action.read_text()) for action in sorted( (directory / "control").glob("*.response.json"))] cases.append({**case, "actions": actions, "artifacts": str(directory), "phases": phases, diff --git a/e2e-test/resource_dispatch.py b/e2e-test/resource_dispatch.py new file mode 100644 index 000000000..b900b654b --- /dev/null +++ b/e2e-test/resource_dispatch.py @@ -0,0 +1,51 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +"""Admit fixtures in selection order against their individual resource costs.""" + +import concurrent.futures +from collections import deque + + +def run_with_resources(cases, slots, execute, resources, budget, stopped): + pending = deque((case, resources(case)) for case in cases) + if slots < 1 or any(any(cost > limit for cost, limit in zip(costs, budget)) + for _, costs in pending): + raise ValueError("fixture exceeds the dispatch resource budget") + errors, active = [], {} + used = [0, 0] + with concurrent.futures.ThreadPoolExecutor(max_workers=slots) as pool: + try: + while active or (pending and not stopped.is_set()): + while pending and len(active) < slots and not stopped.is_set(): + case, costs = pending[0] + if any(value + cost > limit for value, cost, limit in zip(used, costs, budget)): + break + pending.popleft() + used = [value + cost for value, cost in zip(used, costs)] + active[pool.submit(execute, case)] = costs + if not active: + break + completed, _ = concurrent.futures.wait(active, return_when=concurrent.futures.FIRST_COMPLETED) + for future in completed: + costs = active.pop(future) + used = [value - cost for value, cost in zip(used, costs)] + try: + future.result() + except Exception as error: + errors.append(error) + except BaseException: + stopped.set() + for future in active: + future.cancel() + raise + return errors + + +def batch_slots(cases, slots, resources, budget): + """Stable browser batches reserve every peer until the whole batch ends.""" + if not cases: + return slots + costs = [resources(case) for case in cases] + return min(slots, *(int(limit // max(cost[index] for cost in costs)) + for index, limit in enumerate(budget))) diff --git a/e2e-test/run.py b/e2e-test/run.py index df69fcc6d..4d361be3b 100644 --- a/e2e-test/run.py +++ b/e2e-test/run.py @@ -15,7 +15,7 @@ import uuid from pathlib import Path -from client import bundle_dataset_sdk, bundle_cli, bundle_client, bundle_iceberg_rust, bundle_iceberg_java, bundle_iceberg_rck, client_sources +from client import bundle_dataset_sdk, bundle_cli, bundle_cli_workload, bundle_client, bundle_iceberg_rust, bundle_iceberg_java, bundle_iceberg_rck, client_sources from fixture import Cluster, docker from release import check_startup from scenarios import case_nodes, case_resources, run_cases, select_cases @@ -88,7 +88,7 @@ def scenario(index): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--layer", choices=("kv", "diskdb", "diskio", "chunkdb", "chunk-stream", "chunk-client", "chunk-kv", "s3", "iceberg", "dataset", "console", "deployment", "all"), default="all") - parser.add_argument("--scope", choices=("all", "dataset.http", "dataset.sdk", "console.managed", "console.native", "console.native.binding", "console.native.s3", "console.native.recovery", "console.native.load", "console.native.inspection", "console.native.browser.s3", "console.native.browser.ownership", "console.native.browser.journal", "console.native.browser.journal-windows", "console.native.browser.scanner", "console.native.browser.storage-windows", "console.native.browser.lifecycle", "console.native.browser.defaults", "console.native.browser.plan", "console.native.browser.prerequisites", "console.native.browser.transition", "console.native.geometry", "console.native.browser.hardware", "console.native.browser.maintenance", "console.native.browser.disk", "console.native.browser.binding", "console.native.browser.diskdb", "console.native.browser.crud", "console.native.browser.shell", "console.native.browser.retry", "console.native.browser.topology", "console.native.browser.six-node", "console.native.browser.dataset", "console.browser", "console.browser.lifecycle", "console.browser.discovery", "console.browser.shell", "console.browser.registry", "console.browser.kv", "console.browser.canvas", "console.browser.inspector", "console.browser.logical", "console.browser.node", "console.discovery", "console.hardware", "console.logical", "console.bootstrap", "console.cleanup", "console.deployment", "console.cli", "console.cli.container", "console.operations", "console.web", "console.web.disposable", "console.web.recovery", "console.web.access", "console.web.authority", "console.web.slots", "console.web.catalog", "console.web.bootstrap", "console.web.standalone", "console.web.registry", "console.web.lifecycle", "console.web.wal", "console.clients", "iceberg.manual.tpc", "iceberg.benchmark", "iceberg.metadata", "iceberg.namespace", "iceberg.files", "iceberg.files.listing", "iceberg.catalog", "iceberg.gc", "iceberg.sdk", "iceberg.sdk.java", "iceberg.sdk.rust", "iceberg.sdk.rck", "s3.protocol", "s3.recovery", "s3.responses", "s3.benchmark", "s3.sdk", "s3.credentials", "s3.policy", "s3.nodes", "kv.management", "kv.membership", "kv.routing", "kv.recovery", "kv.isolation", "kv.group0", "kv.client", "kv.metadata", "diskdb.allocation", "diskdb.ownership", "diskdb.reuse", "diskdb.recovery", "chunkdb.allocation", "chunkdb.lifecycle", "chunkdb.recovery", "chunkdb.placement", "chunkdb.io", "chunkdb.conversion", "chunkdb.repair", "chunkdb.slots", "chunkdb.handoff", "chunk-kv.protocol", "chunk-kv.recovery", "chunk-client.reader", "chunk-client.large_repair", "chunk-client.small_flow", "chunk-client.benchmark", "chunk-client.large", "chunk-client.small", "chunk-stream.objects", "chunk-stream.recovery", "chunk-stream.errors", "deployment.selection", "deployment.supervision", "deployment.hardware", "deployment.bootstrap", "deployment.balance", "deployment.listeners", "deployment.browser", "deployment.anonymous", "deployment.failures", "deployment.clients", "deployment.storage", "deployment.validation", "deployment.kv", "diskio.durability", "diskio.direct", "diskio.io.mem", "diskio.io.null", "diskio.discovery", "diskio.errors", "diskio.faults", "diskio.semantic"), + parser.add_argument("--scope", choices=("all", "dataset.http", "dataset.sdk", "console.managed", "console.native", "console.native.binding", "console.native.s3", "console.native.recovery", "console.native.load", "console.native.inspection", "console.native.browser.s3", "console.native.browser.ownership", "console.native.browser.journal", "console.native.browser.journal-windows", "console.native.browser.scanner", "console.native.browser.storage-windows", "console.native.browser.lifecycle", "console.native.browser.defaults", "console.native.browser.plan", "console.native.browser.prerequisites", "console.native.browser.transition", "console.native.geometry", "console.native.browser.hardware", "console.native.browser.maintenance", "console.native.browser.disk", "console.native.browser.binding", "console.native.browser.diskdb", "console.native.browser.crud", "console.native.browser.shell", "console.native.browser.retry", "console.native.browser.topology", "console.native.browser.six-node", "console.native.browser.dataset", "console.browser", "console.browser.lifecycle", "console.browser.discovery", "console.browser.shell", "console.browser.registry", "console.browser.kv", "console.browser.canvas", "console.browser.inspector", "console.browser.logical", "console.browser.node", "console.discovery", "console.hardware", "console.logical", "console.bootstrap", "console.cleanup", "console.deployment", "console.cli", "console.cli.container", "console.operations", "console.web", "console.web.disposable", "console.web.recovery", "console.web.access", "console.web.authority", "console.web.slots", "console.web.catalog", "console.web.bootstrap", "console.web.standalone", "console.web.registry", "console.web.lifecycle", "console.web.wal", "console.clients", "iceberg.manual.tpc", "iceberg.benchmark", "iceberg.metadata", "iceberg.namespace", "iceberg.files", "iceberg.files.listing", "iceberg.catalog", "iceberg.gc", "iceberg.sdk", "iceberg.sdk.java", "iceberg.sdk.rust", "iceberg.sdk.rck", "s3.protocol", "s3.recovery", "s3.responses", "s3.benchmark", "s3.sdk", "s3.credentials", "s3.policy", "s3.nodes", "kv.management", "kv.membership", "kv.routing", "kv.recovery", "kv.isolation", "kv.group0", "kv.client", "kv.metadata", "diskdb.allocation", "diskdb.ownership", "diskdb.reuse", "diskdb.recovery", "chunkdb.allocation", "chunkdb.lifecycle", "chunkdb.recovery", "chunkdb.placement", "chunkdb.io", "chunkdb.conversion", "chunkdb.repair", "chunkdb.slots", "chunkdb.handoff", "chunk-kv.protocol", "chunk-kv.recovery", "chunk-client.reader", "chunk-client.large_repair", "chunk-client.small_flow", "chunk-client.benchmark", "chunk-client.large", "chunk-client.small", "chunk-stream.objects", "chunk-stream.recovery", "chunk-stream.errors", "deployment.reference", "deployment.selection", "deployment.supervision", "deployment.hardware", "deployment.bootstrap", "deployment.balance", "deployment.listeners", "deployment.browser", "deployment.anonymous", "deployment.failures", "deployment.clients", "deployment.storage", "deployment.validation", "deployment.kv", "diskio.durability", "diskio.direct", "diskio.io.mem", "diskio.io.null", "diskio.discovery", "diskio.errors", "diskio.faults", "diskio.semantic"), default="all") parser.add_argument("--profile", choices=("bridge", "bridge-admission", "host", "containerd-host", "containerd-cluster", "ecosystem", "ecosystem-python"), default="bridge") parser.add_argument("--concurrency", type=int, default=2) @@ -97,6 +97,8 @@ def main(): help="complete CI partition within Console or Iceberg") parser.add_argument("--list", action="store_true", help="list implemented named flows without starting servers") parser.add_argument("--require-isolation", action="store_true") + parser.add_argument("--fail-fast", action="store_true", + help="stop dispatching cases after the first failure; retain completed artifacts") parser.add_argument("--cpus", type=float, default=4) parser.add_argument("--node-cpus", type=float, default=os.environ.get("CROWDB_E2E_NODE_CPUS", "1"), help="Relative CPU weight per server (1 CPU = 1024 shares); no hard quota") parser.add_argument("--memory-mib", type=int, default=4096) @@ -127,7 +129,12 @@ def main(): from special_profiles import run_host run_host(args, parser) return - if args.profile in ("containerd-host", "containerd-cluster"): + if args.profile == "containerd-host": + signal.signal(signal.SIGTERM, interrupt) + from native_containerd_profile import run_profile + run_profile(args, parser) + return + if args.profile == "containerd-cluster": signal.signal(signal.SIGTERM, interrupt) from containerd_profile import run_profile run_profile(args, parser) @@ -162,7 +169,7 @@ def main(): available_memory = min(args.memory_mib, info["MemTotal"] // 1048576) if available_cpu < needed_cpu or available_memory < needed_memory: parser.error(f"selected cases require at least {needed_cpu:g} CPU / {needed_memory} MiB") - case_slots = min(args.concurrency, int(available_cpu // needed_cpu), available_memory // needed_memory) + case_slots = args.concurrency if (args.require_isolation or args.scope == "kv.isolation") and slots < 2: parser.error("concurrent isolation acceptance needs two fixture slots") identity = json.loads(docker("image", "inspect", args.image))[0] @@ -179,12 +186,16 @@ def main(): run_root.mkdir() (run_root / "identity.json").write_text(json.dumps({"image": args.image, "id": identity["Id"], "repo_digests": identity.get("RepoDigests", []), "labels": labels, "client_revision": revision, - "layer": args.layer, "scope": args.scope, "partition": args.partition, "slots": slots, "case_slots": case_slots, + "layer": args.layer, "scope": args.scope, "partition": args.partition, "fail_fast": args.fail_fast, + "slots": slots, "case_slots": case_slots, "largest_case_nodes": largest, "case_cpu_reservation": needed_cpu, "case_memory_reservation_mib": needed_memory, "selected_cases": [case.name for case in selected], + "resource_budget": {"cpus": available_cpu, "memory_mib": available_memory}, + "dispatch_policy": "per-case-ordered", "node_cpus": args.node_cpus, "cpu_policy": "shares", "tokio_worker_threads": 1, "node_cpu_shares": round(args.node_cpus * 1024), "source_identity": source_identity}, indent=2)) - print(f"KV isolation slots: {slots}; case slots: {case_slots}; exact image {identity['Id']}; artifacts {run_root}", flush=True) + print(f"KV isolation slots: {slots}; case slots: {case_slots}; per-case budget: " + f"{available_cpu:g} CPU / {available_memory} MiB; exact image {identity['Id']}; artifacts {run_root}", flush=True) started = time.monotonic() bundle = bundle_client(run_root / "client") timings = {"client_build": time.monotonic() - started} @@ -204,10 +215,14 @@ def main(): started = time.monotonic() bundle_iceberg_rck(bundle) timings["iceberg_rck_source"] = time.monotonic() - started - if any(case.scope.startswith("console.cli") for case in selected): + if any(case.scope.startswith("console.cli") or case.oci_reference for case in selected): started = time.monotonic() bundle_cli(bundle) timings["cli_build"] = time.monotonic() - started + if any(case.cli_workload for case in selected): + started = time.monotonic() + bundle_cli_workload(bundle) + timings["cli_workload_build"] = time.monotonic() - started with (bundle / "domain-test").open("rb") as executable: client_digest = hashlib.file_digest(executable, "sha256").hexdigest() client_source = hashlib.sha256() @@ -219,7 +234,7 @@ def main(): client_source.update(str(path.relative_to(bundle)).encode()) client_source.update(path.read_bytes()) executable_digests = {} - for name in ("domain-test", "crowdb-cli", "iceberg-rust-client"): + for name in ("domain-test", "crowdb-cli", "cli-workload-test", "iceberg-rust-client"): path = bundle / name if path.is_file(): with path.open("rb") as executable: @@ -234,7 +249,8 @@ def main(): if slots >= 2: run_mixed(identity["Id"], run_root, bundle, args.data_root, args.node_cpus) if selected: - run_cases(identity["Id"], run_root, bundle, case_slots, args.data_root, args.scope, args.case, args.layer, args.node_cpus, args.partition) + run_cases(identity["Id"], run_root, bundle, case_slots, args.data_root, args.scope, args.case, args.layer, args.node_cpus, args.partition, fail_fast=args.fail_fast, + resource_budget=(available_cpu, available_memory)) if args.layer in ("kv", "deployment", "all") and args.scope == "all" and not args.case: check_startup(identity["Id"], run_root) timings["status"] = "pass" diff --git a/e2e-test/s3_node_fixture.py b/e2e-test/s3_node_fixture.py index 14c6a13b0..27dd6fdc6 100644 --- a/e2e-test/s3_node_fixture.py +++ b/e2e-test/s3_node_fixture.py @@ -20,15 +20,24 @@ def __init__(self, image, artifacts, bundle, **kwargs): def start(self): super().start() self.client(self.bundle, "seed", "access_s3::node_failure::seed") + # Service indexes are logical identities, independent of dependency order. + base = ipaddress.ip_address(self.addresses[0]) + self.addresses = [str(base + index) for index in range(14)] for logical in range(3): self.diskdb(logical) - for logical in range(3): - self.chunkdb(logical) for logical in range(3): self.diskio(logical) self.client(self.bundle, "storage-ready", "access_s3::node_failure::seed") + for logical in range(3): + self.chunkdb(logical) self.chunk_kv() self.access() + mounted = dict(zip(self.servers, self.data_paths, strict=True)) + self.servers.sort(key=lambda name: int(name.rsplit("node", 1)[1])) + self.data_paths = [mounted[name] for name in self.servers] + (self.artifacts / "mounts.json").write_text(json.dumps({ + "container_data": "/opt/crowdb/data", + "host_data": [str(path) for path in self.data_paths]}, indent=2)) (self.artifacts / "s3-node-endpoints.json").write_text(json.dumps({ "endpoint": f"http://{self.addresses[13]}:7500", "nodes": [{"rack": n + 1, "node": n + 1, "services": [n, n + 3, n + 6, n + 9]} @@ -46,7 +55,7 @@ def launch_service(self, index, arguments, config=None, environment=(), preserve mounts=mounts, environment=environment, preserve_network=preserve_network) if observed != address: raise AssertionError("inspected service endpoint changed") - self.addresses.append(observed) + self.addresses[index] = observed return address def diskdb(self, logical): diff --git a/e2e-test/s3_recovery.py b/e2e-test/s3_recovery.py index 246c2aeda..d414c6377 100644 --- a/e2e-test/s3_recovery.py +++ b/e2e-test/s3_recovery.py @@ -31,7 +31,7 @@ def run_recovery(cluster, service): elif service == "chunkdb": cluster.ready_chunkdb() elif service == "chunk-kv": - cluster.ready_chunk_kv() + cluster.ready_chunk_kv(recovering=True) elif service == "diskdb": wait(lambda: request(f"http://{cluster.addresses[3]}:7101", "/ready"), lambda _: True, "recovered Diskdb readiness") diff --git a/e2e-test/scenarios.py b/e2e-test/scenarios.py index c7a95f1e9..490a67989 100644 --- a/e2e-test/scenarios.py +++ b/e2e-test/scenarios.py @@ -2,12 +2,14 @@ # Licensed under the Apache License, Version 2.0. """Named flow scopes, fresh server profiles and bounded host-client dispatch.""" -import concurrent.futures import json +import threading +import time from dataclasses import dataclass from contextlib import nullcontext from browser_batch import run_browser_batches +from resource_dispatch import batch_slots, run_with_resources from fixture import Cluster from diskdb_fixture import DiskdbCluster @@ -36,6 +38,8 @@ from console_shell_fixture import ShellBrowserCluster from console_discovery_browser_fixture import DiscoveryBrowserCluster from console_cli_fixture import CliContainerCluster +from console_cli_target import CliAttachedCluster +from oci_reference_fixture import OciReferenceCluster from console_native_fixture import NativeConsoleCluster from console_standalone_fixture import StandaloneWebCluster from console_registry_fixture import RegistryWebCluster @@ -99,6 +103,11 @@ class Case: shell_observations: bool = False console_browser: str | None = None cli_replay: str | None = None + oci_reference: str | None = None + oci_editable_ui: bool = False + oci_address_recovery: bool = False + oci_shortcuts: bool = False + cli_workload: bool = False client_timeout: int = 120 @@ -170,7 +179,13 @@ class Case: ) DEPLOYMENT_CASES = ( - Case("deployment.balance", "deployment::balance::single_node_disables_local_chunk_kv_split_planning", service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web")), + Case("deployment.reference", "deployment::reference::direct_docker_one_node", oci_reference="docker-one", oci_shortcuts=True), + Case("deployment.reference", "deployment::reference::editable_ui_one_node", oci_reference="docker-one", oci_editable_ui=True), + Case("deployment.reference", "deployment::reference::editable_ui_three_nodes", oci_reference="docker-three", oci_editable_ui=True), + Case("deployment.reference", "deployment::reference::direct_docker_three_nodes", oci_reference="docker-three", oci_address_recovery=True), + Case("deployment.reference", "deployment::reference::compose_one_node", oci_reference="compose-one"), + Case("deployment.reference", "deployment::reference::compose_three_nodes", oci_reference="compose-three"), + Case("deployment.balance", "deployment::balance::single_node_preserves_default_chunk_kv_balance_policy", service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web")), Case("deployment.listeners", "deployment::listeners::fixed_s3_listener_recovers_after_explicit_termination", service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web")), Case("deployment.listeners", "deployment::listeners::fixed_iceberg_listener_recovers_after_explicit_termination", service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web")), Case("deployment.browser", "deployment::browser::readonly_release_ui_preserves_domains_and_live_authority", service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web")), @@ -180,10 +195,10 @@ class Case: Case("deployment.clients", "deployment::clients::official_clients_and_protocol_layout_survive_restart", restart=True, service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web"), client_timeout=420), Case("deployment.kv", "deployment::kv_bootstrap::monitor_creates_real_kv_groups_then_validates_after_restart", restart=True, service_selection=("kv",)), Case("deployment.validation", "deployment::validation::invalid_profile_is_rejected", service_selection=("kv",)), - Case("deployment.validation", "deployment::validation::invalid_manifest_is_rejected", service_selection=("kv",)), + Case("deployment.validation", "deployment::validation::invalid_node_policy_is_rejected", service_selection=("kv",)), Case("deployment.storage", "deployment::storage::chunk_services_validate_live_registration_and_logical_topology", restart=True, service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web")), Case("deployment.storage", "deployment::storage::real_iceberg_catalog_and_listener_survive_restart", restart=True, service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web")), - Case("deployment.bootstrap", "deployment::bootstrap::interrupted_bootstrap_preserves_completed_steps", service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web")), + Case("deployment.bootstrap", "deployment::bootstrap::interrupted_setup_preserves_completed_steps", service_selection=("kv", "diskdb", "diskio", "chunkdb", "chunk-kv", "access", "web")), Case("deployment.hardware", "deployment::hardware_bootstrap::writes_and_validates_hardware_with_real_group_zero", (0, 0, 0), True, groups=(0,)), Case("deployment.hardware", "deployment::hardware_bootstrap::conflicting_group_zero_record_rejects_without_creating_hardware", (0, 0, 0), True, groups=(0,)), Case("deployment.selection", "deployment::selected_kv_services_recover_acknowledged_data", restart=True), @@ -231,10 +246,10 @@ class Case: Case("chunkdb.allocation", "chunkdb::allocation::allocation_connect_failure_reroutes_but_lost_reply_does_not"), ) CHUNK_STREAM_CASES = ( - Case("chunk-stream.objects", "chunk_stream::production_idle_writer_stops_after_ownership_takeover", chunkdb_storage=True), + Case("chunk-stream.objects", "chunk_stream::production_released_idle_writer_stops_after_owner_transfer", chunkdb_storage=True), Case("chunk-stream.recovery", "chunk_stream::production_stream_recovers_exact_bytes_after_service_restarts", chunkdb_storage=True, storage_backend="file", restart=True), Case("chunk-stream.errors", "chunk_stream::production_stream_write_returns_after_diskio_failure", chunkdb_storage=True, diskio_process_exit=True), - Case("chunk-stream.errors", "chunk_stream::production_stream_write_returns_after_live_diskio_errors", chunkdb_storage=True, diskio_fault_rate=1.0), + Case("chunk-stream.errors", "chunk_stream::production_stream_initialization_returns_after_live_diskio_errors", chunkdb_storage=True, diskio_fault_rate=1.0), ) CHUNK_CLIENT_CASES = ( Case("chunk-client.reader", "chunk_client::reader::data::ec_range_read_recovers_only_the_requested_bytes", chunkdb_storage=True, storage_backend="mem"), @@ -392,7 +407,7 @@ class Case: Case("console.native.browser.prerequisites", "console::native_browser::waiting_plan_prerequisites", native_client=True, client_timeout=360), Case("console.native.browser.storage-windows", "console::native_browser::storage_windows_and_access_recovery", native_client=True, native_disk_gib=(80, 256, 256), client_timeout=360), Case("console.native.browser.scanner", "console::native_browser::scanner_and_scoped_recalc", native_client=True, client_timeout=360), - Case("console.native.browser.journal-windows", "console::native_browser::large_journal_pagination_and_restart", native_client=True, client_timeout=480), + Case("console.native.browser.journal-windows", "console::native_browser::large_journal_records_and_restart", native_client=True, client_timeout=480), Case("console.native.recovery", "console::native_recovery::all_typed_restarts_preserve_objects_and_keep_one_live_process", native_client=True), Case("console.native.load", "console::native_load::production_load_keeps_heartbeats_and_durable_bytes_across_owner_restart", native_client=True), Case("console.native.inspection", "console::native_browser::page_layout_and_iceberg_inspection", native_client=True, native_disk_gib=(80, 256, 256), client_timeout=3840), @@ -408,9 +423,16 @@ class Case: Case("console.browser.logical", "console::browser_logical::delete", native_browser="deletes a replica and a group through the UI and verifies the real backend"), Case("console.browser.inspector", "console::browser_inspector::activity", shell_browser="records mutations and async operations, and clear empties the log"), Case("console.browser.inspector", "console::browser_inspector::selection", shell_browser="clears stale properties when the domain changes or the cluster resets"), + Case("console.cli.container", "console::cli_container::pending_free_accounting", oci_reference="docker-one", cli_workload=True), Case("console.cli.container", "console::cli_container::confirmed_group_zero_replay", cli_replay="bound"), Case("console.cli.container", "console::cli_container::confirmed_storage_replay", cli_replay="storage"), - Case("console.managed", "console::managed::authority_outage", console_browser="Managed native authority outage hides stale topology and retains monitor observation"), + Case("console.cli.container", "console::cli_container::terminal_chunk_writer_recovery", cli_replay="terminal-writer"), + Case("console.cli.container", "console::cli_container::plain_docker_target_attachment", cli_replay="attachment"), + Case("console.cli.container", "console::cli_container::observation_round_trip", cli_replay="observation"), + Case("console.cli.container", "console::cli_container::hardware_round_trip", cli_replay="hardware"), + Case("console.cli.container", "console::cli_container::logical_round_trip", cli_replay="logical"), + Case("console.cli.container", "console::cli_container::data_round_trip", cli_replay="data"), + Case("console.managed", "console::managed::authority_outage", console_browser="Node authority outage rejects mutations and marks retained observations stale"), Case("console.managed", "console::managed::hardware_and_user_flows", console_browser="Managed native KV, Iceberg, S3 and Chunk operations retain hardware boundaries"), Case("console.browser.kv", "console::browser_kv::pagination", shell_browser="Group selection opens data directly; pages replace rows and expose complete values"), Case("console.browser.kv", "console::browser_kv::binary_bytes", shell_browser="binary byte runs retain printable fields and use distinct uppercase hex without a prefix"), @@ -506,24 +528,11 @@ class Case: Case("console.deployment", "console::deployment::authenticated_node_update_preserves_ids_and_converges_from_another_console", (0, 0, 0), True, groups=(0,)), Case("console.bootstrap", "console::cluster::clean_requires_confirmed_group_replicas", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_direct::cluster_status_via_direct_group0", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_direct::cluster_rack_list_ignores_legacy_local_state", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_direct::cluster_node_list_ignores_legacy_local_state", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_direct::kv_server_controls_require_launch_registry", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_direct::incremental_local_deploy_reads_group_zero_instead_of_legacy_file", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_direct::destroy_reads_group_zero_and_requires_launch_registry", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_observation::cluster_status_topology_via_direct_group0", (0, 0, 0), True, groups=(0,)), Case("console.clients", "console::server_management::full_store_group_remote_cycle", stores_per_node=((),), groups=()), Case("console.clients", "console::server_data::put_get_delete_cycle", stores_per_node=((),), groups=()), - Case("console.cli", "console::cli_management::store_group_replica_round_trip", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_hardware::rack_node_lifecycle", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_data::kv_put_get_delete_round_trip", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_registry::registry_hardware_uses_group_zero_across_cli_invocations", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_registry::registry_bootstrap_uses_sealed_intent_without_legacy_topology_file", (0, 0, 0), True, groups=(0,)), - Case("console.cli", "console::cli_registry::cli_uses_launch_registry_and_runtime_identity_without_legacy_state", (0, 0, 0), True, groups=(0,)), Case("console.web.authority", "console::web_authority::bare_metal_hardware_routes_share_confirmed_group_zero_state", console_web=True), Case("console.web.authority", "console::web_authority::bare_metal_snapshot_requires_live_authority_without_a_docker_monitor", console_web=True), Case("console.web.authority", "console::web_authority::snapshot_validates_later_replica_hosts_as_well_as_original_store_hosts", console_web=True), @@ -555,6 +564,8 @@ def select_cases(layer="all", scope="all", case_name=None, partition=None): def case_nodes(case): + if case.oci_reference: + return 3 if case.oci_shortcuts else 4 if case.oci_address_recovery else 1 if case.oci_reference.endswith("-one") else 3 if case.scope == "console.browser.discovery": return 2 if case.native_browser or case.native_client: @@ -606,6 +617,9 @@ def case_nodes(case): return 2 if case.scope.startswith(("diskdb.", "chunkdb.")) else len(case.stores_per_node or case.stores) def case_resources(case, node_cpus): + if case.oci_reference: + count = 3 if case.oci_shortcuts else 1 if case.oci_reference.endswith("-one") else 3 + return count + 0.5 + (0.25 if case.oci_address_recovery else 0), count * 1536 + 512 + (768 if case.oci_address_recovery else 0) # CPU shares allow nodes to borrow idle capacity; memory remains a hard bound. if case.scope == "console.browser.discovery": return 1.5, 3072 @@ -629,6 +643,10 @@ def case_resources(case, node_cpus): def allocate_cluster(case, image, artifacts, bundle, data_root): + if case.oci_reference: + return OciReferenceCluster(image, artifacts, bundle, reference=case.oci_reference, + editable_ui=case.oci_editable_ui, cli_workload=case.cli_workload, + address_recovery=case.oci_address_recovery, shortcuts=case.oci_shortcuts, data_root=data_root) if case.scope == "console.browser.discovery": return DiscoveryBrowserCluster(image, artifacts, bundle, data_root=data_root) if case.native_browser or case.native_client: @@ -643,6 +661,8 @@ def allocate_cluster(case, image, artifacts, bundle, data_root): diskless_nodes=case.native_diskless_nodes, data_root=data_root) if case.cli_replay: + if case.cli_replay in ("attachment", "observation", "hardware", "logical", "data"): + return CliAttachedCluster(image, artifacts, bundle, data_root=data_root) return CliContainerCluster(image, artifacts, bundle, checkpoint=case.cli_replay, data_root=data_root) if case.shell_browser: return ShellBrowserCluster(image, artifacts, bundle, data_root=data_root) @@ -675,7 +695,7 @@ def allocate_cluster(case, image, artifacts, bundle, data_root): if case.scope == "deployment.clients": return DeploymentClientsCluster(image, artifacts, services=case.service_selection, data_root=data_root) if case.scope == "deployment.validation": - kind = "profile" if "invalid_profile" in case.name else "manifest" + kind = "profile" if "invalid_profile" in case.name else "policy" return RejectedBootstrapCluster(image, artifacts, kind, data_root=data_root) if case.scope == "deployment.bootstrap": return InterruptedBootstrapCluster(image, artifacts, services=case.service_selection, data_root=data_root) @@ -732,7 +752,7 @@ def allocate_cluster(case, image, artifacts, bundle, data_root): advertise=case.advertise) -def run_cases(image, artifacts, bundle, slots, data_root=None, scope="all", case_name=None, layer="all", node_cpus=1.0, partition=None): +def run_cases(image, artifacts, bundle, slots, data_root=None, scope="all", case_name=None, layer="all", node_cpus=1.0, partition=None, fail_fast=False, resource_budget=None): selected = select_cases(layer, scope, case_name, partition) if not selected: raise ValueError(f"no migrated cases for scope {scope}") @@ -745,13 +765,29 @@ def prepare(case): cluster.browser_timeout = case.client_timeout cluster.node_cpus = node_cpus (cluster.artifacts / "case.json").write_text(json.dumps({ - "scope": case.scope, "case": case.name}, indent=2)) + "scope": case.scope, "case": case.name, + "scheduled_at_unix_ms": time.time_ns() // 1_000_000, + "resources": dict(zip(("cpus", "memory_mib"), case_resources(case, node_cpus)))}, indent=2)) return cluster + stopped = threading.Event() + def run(case, owned=None): + if owned is None and stopped.is_set(): + return + try: + execute(case, owned) + except Exception: + if fail_fast: + stopped.set() + raise + + def execute(case, owned=None): cluster = owned if owned is not None else prepare(case) with nullcontext(cluster) if owned is not None else cluster: - if case.cli_replay: + if case.oci_reference: + cluster.verify_reference() + elif case.cli_replay: cluster.replay(case.cli_replay) elif case.native_browser: config = "native.config.ts" if case.scope.startswith("console.native.") else "shell.config.ts" @@ -793,24 +829,18 @@ def run(case, owned=None): cluster.client(bundle, case.name.rsplit("::", 1)[-1], case.name) def stable_network(case): - return case.cli_replay or case.native_client or case.native_browser or case.shell_browser or case.console_browser - - with concurrent.futures.ThreadPoolExecutor(max_workers=slots) as pool: - futures = [pool.submit(run, case) for case in selected if not stable_network(case)] - errors = [] - try: - for future in concurrent.futures.as_completed(futures): - try: - future.result() - except Exception as error: - errors.append(error) - except BaseException: - for future in futures: - future.cancel() - raise - errors.extend(run_browser_batches([case for case in selected if stable_network(case)], - slots, prepare, run)) - if errors: - for error in errors[1:]: - errors[0].add_note(f"another independent fixture failed: {error}") - raise errors[0] + return (case.cli_replay or case.native_client or case.native_browser or case.shell_browser + or case.console_browser or case.oci_editable_ui or case.scope == "deployment.browser") + + budget = resource_budget or (float("inf"), float("inf")) + resources = lambda case: case_resources(case, node_cpus) + errors = run_with_resources([case for case in selected if not stable_network(case)], + slots, run, resources, budget, stopped) + if not stopped.is_set(): + browsers = [case for case in selected if stable_network(case)] + browser_slots = batch_slots(browsers, slots, resources, budget) if resource_budget else slots + errors.extend(run_browser_batches(browsers, browser_slots, prepare, run, fail_fast=fail_fast)) + if errors: + for error in errors[1:]: + errors[0].add_note(f"another independent fixture failed: {error}") + raise errors[0] diff --git a/e2e-test/special_profiles.py b/e2e-test/special_profiles.py index 18c64406b..9067c094c 100644 --- a/e2e-test/special_profiles.py +++ b/e2e-test/special_profiles.py @@ -1,10 +1,11 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. -"""Explicit host deployment profile, outside the concurrent bridge domain matrix.""" +"""Explicit Docker host acceptance outside the concurrent bridge domain matrix.""" import json import os from pathlib import Path +import shutil import subprocess import sys import time @@ -12,101 +13,108 @@ from fixture import docker, redact from image_identity import verify_server_sources from client import bundle_client +from native_containerd_profile import client_bundle +from oci_host_node import clean, read_private, selected +from oci_local_host_node import cleanup_local_host ROOT = Path(__file__).resolve().parents[1] -def cleanup_owned_host(name, data, image): - names = docker("container", "ls", "-a", "--filter", f"name={name}", "--format", "{{.Names}}") - if name not in names.splitlines(): - return - node = json.loads(docker("inspect", name))[0] - mounts = node.get("Mounts", []) - if node["Image"] != image or not any(mount.get("Destination") == "/opt/crowdb/data" - and mount.get("Source") == str(data / "node") for mount in mounts): - raise ValueError("host cleanup refuses changed image or data ownership") - docker("rm", "-f", name) +def cleanup_owned_hosts(data): + errors = [] + for case in ('protocols', 'supplied'): + try: + cleanup_local_host(data/case) + except Exception as error: + errors.append(str(error)) + runtime_root = data / case / 'private/runtime' + directory = runtime_root / 'persistent/oci-hosts' + for path in directory.glob('*.json'): + try: + if read_private(path) is not None: + clean(selected(path.stem, runtime_root)) + except Exception as error: + errors.append(str(error)) + if errors: + raise RuntimeError('host cleanup failed: ' + '; '.join(errors)) -def cleanup_owned_media(name, data, image): - helper = name + "-media" - names = docker("container", "ls", "-a", "--filter", f"name={helper}", "--format", "{{.Names}}") - if helper not in names.splitlines(): - return - current = json.loads(docker("inspect", helper))[0] - if (current["Image"] != image or current["Config"].get("Labels", {}).get("org.crowdb.e2e.run") != name - or not any(mount.get("Destination") == "/owned" and mount.get("Source") == str(data) - for mount in current.get("Mounts", []))): - raise ValueError("media cleanup refuses changed image or data ownership") - docker("rm", "-f", helper) +def media_client(destination): + supplied = os.environ.get('CROWDB_E2E_DOMAIN_BUNDLE') + if not supplied: + return bundle_client(destination) + source = Path(supplied).resolve() + (destination / 'lib').mkdir(parents=True) + shutil.copy2(source / 'domain-test', destination / 'domain-test') + for path in (source / 'lib').iterdir(): + if path.is_file(): + shutil.copy2(path, destination / 'lib' / path.name) + return destination def run_host(args, parser): - if args.layer != "all" or args.scope != "all" or args.case or args.require_isolation: - parser.error("host profile has one deployment acceptance; domain selectors do not apply") + if args.layer != 'all' or args.scope != 'all' or args.case or args.require_isolation: + parser.error('host profile has one deployment acceptance; domain selectors do not apply') if args.concurrency != 1: - parser.error("host fixed ports require --concurrency 1 and an exclusive host") - if args.cpus < 1.5 or args.memory_mib < 1536: - parser.error("host profile requires a shared 1.5-CPU scheduling allowance and 1536 MiB including the node and host client") + parser.error('host fixed ports require --concurrency 1 and an exclusive host') + if args.cpus < 1.5 or args.memory_mib < 2560: + parser.error('host profile requires 1.5 CPU and 2560 MiB including the node and host client') if args.list: - print(json.dumps([{"scope":"deployment.host", "case":"host_network_admission_and_recovery", "nodes":1}])) + print(json.dumps([dict(scope='deployment.host', case='host_network_admission_and_recovery', nodes=1)])) return - if sys.platform != "linux": - parser.error("host deployment acceptance requires Linux") - info = json.loads(docker("info", "--format", "{{json .}}")) - if info["OSType"] != "linux" or info["NCPU"] < 1.5 or info["MemTotal"] < 1536 * 1048576: - parser.error("host deployment requires a Linux daemon with 1.5 CPU / 1536 MiB capacity") - identity = json.loads(docker("image", "inspect", args.image))[0] - labels = identity["Config"].get("Labels") or {} + if sys.platform != 'linux' or os.geteuid() != 0: + parser.error('host acceptance requires an explicitly privileged disposable Linux host') + info = json.loads(docker('info', '--format', '{{json .}}')) + if info['OSType'] != 'linux' or info['NCPU'] < 1.5 or info['MemTotal'] < 2560 * 1048576: + parser.error('host deployment requires a Linux daemon with 1.5 CPU / 2560 MiB capacity') + identity = json.loads(docker('image', 'inspect', args.image))[0] + labels = identity['Config'].get('Labels') or {} try: - source = verify_server_sources(ROOT, labels.get("org.opencontainers.image.revision"), - labels.get("org.crowdb.source.tree") or None) + source = verify_server_sources(ROOT, labels.get('org.opencontainers.image.revision'), + labels.get('org.crowdb.source.tree') or None) except ValueError as error: parser.error(str(error)) - run_root = args.artifacts / ("run-" + uuid.uuid4().hex) - run_root.mkdir(parents=True) - data = (args.data_root / run_root.name / "host" if args.data_root else ROOT / ".crowdb-runtime/test-data/container-e2e" / run_root.name / "host").resolve() + run_root = args.artifacts / ('run-' + uuid.uuid4().hex) + run_root.mkdir(parents=True, mode=0o700) + data = (args.data_root / run_root.name / 'host' if args.data_root else + ROOT / '.crowdb-runtime/test-data/container-e2e' / run_root.name / 'host').resolve() data.mkdir(parents=True, mode=0o700) - (run_root / "identity.json").write_text(json.dumps({"profile":"host", "id":identity["Id"], - "labels":labels, "sources":source, "concurrency":1, "cpus":1.5, "memory_mib":1536, - "host_data":str(data), "container_data":"/opt/crowdb/data", - "core_policy":Path("/proc/sys/kernel/core_pattern").read_text().strip()}, indent=2)) - env = os.environ.copy() - client = bundle_client(run_root / "media-client") - name = "crowdb-e2e-host-" + run_root.name.removeprefix("run-") - env.update(CROWDB_CONTAINER_IMAGE=identity["Id"], CROWDB_E2E_PROFILE_DATA=str(data), - CROWDB_E2E_PROFILE_NAME=name, CROWDB_MEDIA_CLIENT=str(client), - CROWDB_MEDIA_HOST_ID="docker-acceptance-host") - started = time.monotonic() - status, code = "failed", None - print(f"Host deployment profile: exact image {identity['Id']}; artifacts {run_root}", flush=True) - child = subprocess.Popen([sys.executable, str(ROOT / "e2e-test/profiles/host-node.py")], + (run_root / 'identity.json').write_text(json.dumps(dict(profile='host', id=identity['Id'], + labels=labels, sources=source, concurrency=1, cpus=1.5, memory_mib=2560, + host_data=str(data), core_policy=Path('/proc/sys/kernel/core_pattern').read_text().strip()), indent=2)) + cli = client_bundle(run_root / 'cli') + client = media_client(run_root / 'media-client') + env = dict(os.environ, CROWDB_CONTAINER_IMAGE=identity['Id'], CROWDB_E2E_PROFILE_DATA=str(data), + CROWDB_E2E_CLI_BUNDLE=str(cli), CROWDB_MEDIA_CLIENT=str(client), + CROWDB_CONTAINERD_CLUSTER='oci-' + run_root.name[-20:]) + started, status, code = time.monotonic(), 'failed', None + print(f"Docker host deployment: exact image {identity['Id']}; artifacts {run_root}", flush=True) + child = subprocess.Popen([sys.executable, str(ROOT / 'e2e-test/profiles/host-node.py')], cwd=ROOT, env=env, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) try: - stdout, _ = child.communicate() + stdout, _ = child.communicate(timeout=900) code = child.returncode - output = redact(stdout) - (run_root / "client-host.log").write_text(output) - print(output, end="", flush=True) + (run_root / 'client-host.log').write_text(redact(stdout)) + print(redact(stdout), end='', flush=True) if code: - raise RuntimeError("host deployment acceptance failed; inspect retained client log and node data") - status = "pass" + raise RuntimeError('host acceptance failed; inspect retained client log and node data') + status = 'pass' finally: if child.poll() is None: child.terminate() try: - stdout, _ = child.communicate(timeout=10) + stdout, _ = child.communicate(timeout=45) except subprocess.TimeoutExpired: child.kill() stdout, _ = child.communicate(timeout=5) - (run_root / "client-host.log").write_text(redact(stdout)) + (run_root / 'client-host.log').write_text(redact(stdout)) + print(redact(stdout), end='', flush=True) try: - cleanup_owned_host(name, data, identity["Id"]) - cleanup_owned_media(name, data, identity["Id"]) + cleanup_owned_hosts(data) except Exception: - status = "failed" + status = 'failed' raise finally: - (run_root / "results.json").write_text(json.dumps({"profile":"host", "status":status, - "exit_code":code, "wall_seconds":time.monotonic()-started, "passed":int(status=="pass"), - "failed":int(status!="pass"), "skipped":0}, indent=2)) + (run_root / 'results.json').write_text(json.dumps(dict(profile='host', status=status, + exit_code=code, wall_seconds=time.monotonic()-started, passed=int(status=='pass'), + failed=int(status!='pass'), skipped=0), indent=2)) diff --git a/e2e-test/tests/access_iceberg/file_http/profiles.rs b/e2e-test/tests/access_iceberg/file_http/profiles.rs index 5824a144d..d321b106d 100644 --- a/e2e-test/tests/access_iceberg/file_http/profiles.rs +++ b/e2e-test/tests/access_iceberg/file_http/profiles.rs @@ -7,6 +7,7 @@ use crowdb_access_iceberg::{ catalog::{CatalogRepository, ClearBounds}, file::{FileRepository, TableLocation}, }; +use crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES; use futures::StreamExt; use reqwest::Method; use std::time::Instant; @@ -14,7 +15,12 @@ use std::time::Instant; #[ignore = "requires the owned release storage fixture"] async fn small_file_profiles() { let (_stack, _process, client, table) = setup().await; - for (label, size) in [("1KiB", 1024), ("512KiB", 512 * 1024), ("aligned64KiB", 65502)] { + for (label, size) in [ + ("1KiB", 1024), + ("512KiB", 512 * 1024), + ("aligned64KiB", MAX_FRAME_V2_PAYLOAD_BYTES), + ("over64KiB", MAX_FRAME_V2_PAYLOAD_BYTES + 1), + ] { assert!(size < fixture_config().iceberg_small_write().threshold_exclusive()); let bytes = vec![0x5a; size]; let warmup = path(table, &format!("data/small-{label}-warmup.bin")); @@ -65,7 +71,10 @@ async fn small_file_profiles() { ); let receive_after = receive_counts(&client).await; let frames = receive_after.0 - receive_before.0; - assert_eq!(frames, (requests * size.div_ceil(65502)) as u64); + assert_eq!( + frames, + (requests * size.div_ceil(MAX_FRAME_V2_PAYLOAD_BYTES)) as u64 + ); assert_eq!(receive_after.1 - receive_before.1, requests as u64); println!("iceberg small native {label} concurrency={concurrency} prepared_frames={frames} receive_allocations={} direct_bytes={} prefix_copy_bytes={}", receive_after.1 - receive_before.1, receive_after.2 - receive_before.2, @@ -73,7 +82,7 @@ async fn small_file_profiles() { results.sort_by_key(|(_, latency)| *latency); println!("iceberg small {label} concurrency={concurrency} requests={requests} small_completions={} large_strips={} writer_feeds={} frames={} elapsed_us={} p50_us={} p95_us={} p99_us={} {}", route_after.0 - route_before.0, route_after.1 - route_before.1, route_after.2 - route_before.2, - size.div_ceil(65502), elapsed.as_micros(), results[(requests - 1) / 2].1, + size.div_ceil(MAX_FRAME_V2_PAYLOAD_BYTES), elapsed.as_micros(), results[(requests - 1) / 2].1, results[(requests * 95).div_ceil(100) - 1].1, results[(requests * 99).div_ceil(100) - 1].1, catalog_delta(before, catalog_counts(&client).await)); diff --git a/e2e-test/tests/access_s3/node_failure/recovery.rs b/e2e-test/tests/access_s3/node_failure/recovery.rs index d623ebd48..fc2b78e91 100644 --- a/e2e-test/tests/access_s3/node_failure/recovery.rs +++ b/e2e-test/tests/access_s3/node_failure/recovery.rs @@ -5,7 +5,7 @@ use crowdb_chunkdb_client::{ChunkdbClient, ChunkdbRpcTransport}; use crowdb_kv_client::{HardwareClient, RangeBindingClient, ServiceRegistryClient}; use crowdb_protocol::chunkdb::rpc::{ChunkType, ListChunksRequest, QueryChunkRequest, Strip, StripType}; use crowdb_protocol::common::{ChunkId, HwStatus}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; @@ -144,19 +144,32 @@ async fn diskdb_missing( .list_disk_groups() .await .map_err(|error| format!("hardware: {error}"))?; - let mut owned = HashSet::new(); - for (_, instance) in instances { + let owners = hardware + .list_owners() + .await + .map_err(|error| format!("ownership: {error}"))?; + let mut advertised = HashMap::>::new(); + for (id, instance) in instances { if instance.rpc_endpoint.is_empty() { continue; } if let Some(diskdb) = instance.extra.and_then(|extra| extra.diskdb) { - owned.extend(diskdb.owned_dg_ids); + for group in diskdb.owned_dg_ids { + advertised.entry(group).or_default().insert(id); + } } } Ok(groups .iter() + .filter(|group| { + !owners.iter().any(|owner| { + owner.dg_id == group.dg_id + && advertised.get(&group.dg_id).is_some_and(|instances| { + instances.len() == 1 && instances.contains(&owner.instance_id) + }) + }) + }) .map(|group| group.dg_id) - .filter(|group| !owned.contains(group)) .collect()) } diff --git a/e2e-test/tests/admission_profile_test.py b/e2e-test/tests/admission_profile_test.py index efee03d4f..7967587f9 100644 --- a/e2e-test/tests/admission_profile_test.py +++ b/e2e-test/tests/admission_profile_test.py @@ -65,3 +65,30 @@ def test_changed_mount_or_image_and_foreign_names_are_preserved(self): with self.assertRaisesRegex(RuntimeError,"foreign labeled"): cleanup("owned") self.assertEqual(docker.call_count,2) + + def test_each_start_owns_a_policy_file_that_other_nodes_cannot_rewrite(self): + path = Path(__file__).resolve().parents[1] / "profiles/bridge_node.py" + with tempfile.TemporaryDirectory() as directory, patch.dict(os.environ, { + "CROWDB_CONTAINER_IMAGE": "exact", "CROWDB_E2E_PROFILE_NAME": "owned", + "CROWDB_E2E_PROFILE_DATA": directory}): + spec = importlib.util.spec_from_file_location("test_bridge_policy", path) + bridge = importlib.util.module_from_spec(spec) + spec.loader.exec_module(bridge) + def reply(*arguments): + if arguments[0] == "inspect": + return json.dumps([{"NetworkSettings": {"Networks": {"owned": {"IPAddress": "172.30.0.2"}}}}]) + return "" + with patch.object(bridge, "docker", side_effect=reply) as docker: + bridge.start_node("owned-0") + bridge.start_node("owned-1") + policies = [] + for call in docker.call_args_list: + for argument in call.args: + if isinstance(argument, str) and "target=/run/bridge-policy.toml" in argument: + source = next(field.removeprefix("source=") for field in argument.split(",") + if field.startswith("source=")) + policies.append(Path(source)) + self.assertEqual(len(policies), 2) + self.assertNotEqual(policies[0], policies[1]) + self.assertTrue(all(policy.parent == Path(directory) for policy in policies)) + self.assertEqual(policies[0].read_text(), policies[1].read_text()) diff --git a/e2e-test/tests/browser_batch_test.py b/e2e-test/tests/browser_batch_test.py index ce12f9107..344571a3a 100644 --- a/e2e-test/tests/browser_batch_test.py +++ b/e2e-test/tests/browser_batch_test.py @@ -62,3 +62,13 @@ def test_partial_startup_closes_started_peer_without_dispatching_clients(self): self.assertEqual([str(error) for error in errors], ['startup rejected']) self.assertEqual(self.events, [('start', 0), ('start', 1), ('close', 0)]) self.assertEqual(self.active, set()) + + def test_fail_fast_closes_current_batch_before_skipping_later_fixtures(self): + def execute(case, fixture): + if case == 0: + raise ValueError('browser failed') + + errors = run_browser_batches([0, 1], 1, self.fixture, execute, fail_fast=True) + self.assertEqual([str(error) for error in errors], ['browser failed']) + self.assertEqual(self.events, [('start', 0), ('close', 0)]) + self.assertEqual(self.active, set()) diff --git a/e2e-test/tests/chunk_client/large.rs b/e2e-test/tests/chunk_client/large.rs index 6f39d21a5..203db156f 100644 --- a/e2e-test/tests/chunk_client/large.rs +++ b/e2e-test/tests/chunk_client/large.rs @@ -8,7 +8,7 @@ use crowdb_chunk_client::{ }; use crowdb_common::ec::{encode_parity_from_shards, EcScheme}; use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, EcState, Location, Strip}; -use crowdb_protocol::frame::{ChunkLocation, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_protocol::frame::{ChunkLocation, MAX_FRAME_V2_PAYLOAD_BYTES}; use std::sync::Arc; const MIB: usize = 1024 * 1024; fn ec_4_1() -> EcScheme { @@ -187,7 +187,7 @@ async fn large_raw_buffers_submit_complete_mirror_strips_concurrently() { let mut configured = policy(16 * MIB as u64); Arc::get_mut(&mut configured.client).unwrap().large_mirror_copies = Some(2); let mut writer = stack.client.prepare_large_write(None, configured); - let block = Bytes::from(vec![0x5a; 16 * MAX_FRAME_PAYLOAD_BYTES]); + let block = Bytes::from(vec![0x5a; 16 * MAX_FRAME_V2_PAYLOAD_BYTES]); for _ in 0..8 { assert_eq!( writer.on_data(block.clone()).await.unwrap(), @@ -296,7 +296,7 @@ async fn large_write_rotates_chunks_without_losing_data() { let mut streamed = Vec::new(); while let Some(part) = stream.next_chunk().await { let part = part.unwrap(); - assert!(part.len() <= MAX_FRAME_PAYLOAD_BYTES); + assert!(part.len() <= MAX_FRAME_V2_PAYLOAD_BYTES); streamed.extend_from_slice(&part); } assert_eq!(streamed, data); diff --git a/e2e-test/tests/chunk_client/large_repair.rs b/e2e-test/tests/chunk_client/large_repair.rs index 87d3782ca..aa24e9d65 100644 --- a/e2e-test/tests/chunk_client/large_repair.rs +++ b/e2e-test/tests/chunk_client/large_repair.rs @@ -20,8 +20,8 @@ use crowdb_kv_client::{ClientConfig, CrowdbKvClient, HardwareClient, ServiceRegi use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, Location, Strip}; use crowdb_protocol::diskdb::rpc::Segment; use crowdb_protocol::frame::{ - encode_frame_regions, FrameError, FrameMagic, FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, - MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES, + encode_frame_regions, FrameError, FrameMagic, FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_BYTES, + MAX_FRAME_V2_PAYLOAD_BYTES, }; use super::stack::TestStack as E2eStack; @@ -37,8 +37,8 @@ impl FullFramedOwner { fn new(frames: usize) -> Self { let mut bytes = vec![0; frames * MAX_FRAME_BYTES]; for index in 0..frames { - let start = index * MAX_FRAME_BYTES + FRAME_HEADER_PREFIX_BYTES; - bytes[start..start + MAX_FRAME_PAYLOAD_BYTES].fill(0x5a); + let start = index * MAX_FRAME_BYTES + FRAME_V2_HEADER_BYTES; + bytes[start..start + MAX_FRAME_V2_PAYLOAD_BYTES].fill(0x5a); } Self { bytes, frames } } @@ -46,7 +46,7 @@ impl FullFramedOwner { impl FramedWriteBuffer for FullFramedOwner { fn logical_len(&self) -> u64 { - (self.frames * MAX_FRAME_PAYLOAD_BYTES) as u64 + (self.frames * MAX_FRAME_V2_PAYLOAD_BYTES) as u64 } fn frame_count(&self) -> usize { @@ -54,7 +54,7 @@ impl FramedWriteBuffer for FullFramedOwner { } fn frame_payload_len(&self, index: usize) -> Option { - (index < self.frames).then_some(MAX_FRAME_PAYLOAD_BYTES) + (index < self.frames).then_some(MAX_FRAME_V2_PAYLOAD_BYTES) } fn finalize_frame( @@ -67,8 +67,8 @@ impl FramedWriteBuffer for FullFramedOwner { let start = index * MAX_FRAME_BYTES; let end = start + MAX_FRAME_BYTES; let frame = &mut self.bytes[start..end]; - let (header, remainder) = frame.split_at_mut(FRAME_HEADER_PREFIX_BYTES); - let (payload, footer) = remainder.split_at_mut(MAX_FRAME_PAYLOAD_BYTES); + let (header, remainder) = frame.split_at_mut(FRAME_V2_HEADER_BYTES); + let (payload, footer) = remainder.split_at_mut(MAX_FRAME_V2_PAYLOAD_BYTES); debug_assert_eq!(footer.len(), FRAME_FOOTER_BYTES); encode_frame_regions(magic, chunk_id, payload, write_time_ms, header, footer)?; Ok(start..end) diff --git a/e2e-test/tests/chunk_client/large_repair/mirror.rs b/e2e-test/tests/chunk_client/large_repair/mirror.rs index 0cf0e4a46..1e5421f95 100644 --- a/e2e-test/tests/chunk_client/large_repair/mirror.rs +++ b/e2e-test/tests/chunk_client/large_repair/mirror.rs @@ -62,7 +62,7 @@ async fn large_mirror_rotates_failed_partial_final_strip() { ChunkIoClient::from_parts_with_small_policy(allocator, fault.clone(), small_policy()).unwrap(); let mut configured = policy(16 * MIB as u64); Arc::get_mut(&mut configured.client).unwrap().large_mirror_copies = Some(2); - let expected = vec![0x5a; MAX_FRAME_PAYLOAD_BYTES]; + let expected = vec![0x5a; MAX_FRAME_V2_PAYLOAD_BYTES]; let mut writer = client.prepare_large_write(Some(expected.len() as u64), configured); writer .on_framed_data(Box::new(FullFramedOwner::new(1))) @@ -94,7 +94,7 @@ async fn large_mirror_abort_drains_delayed_submitted_write_before_delete() { ChunkIoClient::from_parts_with_small_policy(allocator, fault.clone(), small_policy()).unwrap(); let mut configured = policy(16 * MIB as u64); Arc::get_mut(&mut configured.client).unwrap().large_mirror_copies = Some(2); - let mut writer = client.prepare_large_write(Some((16 * MAX_FRAME_PAYLOAD_BYTES) as u64), configured); + let mut writer = client.prepare_large_write(Some((16 * MAX_FRAME_V2_PAYLOAD_BYTES) as u64), configured); writer .on_framed_data(Box::new(FullFramedOwner::new(16))) .await diff --git a/e2e-test/tests/chunk_client/large_repair/mirror_repair.rs b/e2e-test/tests/chunk_client/large_repair/mirror_repair.rs index 68d32aca0..09821d2c5 100644 --- a/e2e-test/tests/chunk_client/large_repair/mirror_repair.rs +++ b/e2e-test/tests/chunk_client/large_repair/mirror_repair.rs @@ -24,7 +24,7 @@ async fn large_mirror_retains_later_framed_buffers_until_failed_first_strip_is_r let mut configured = policy(16 * MIB as u64); Arc::get_mut(&mut configured.client).unwrap().large_mirror_copies = Some(2); let frames = 65; - let expected = vec![0x5a; frames * MAX_FRAME_PAYLOAD_BYTES]; + let expected = vec![0x5a; frames * MAX_FRAME_V2_PAYLOAD_BYTES]; let mut writer = client.prepare_large_write(Some(expected.len() as u64), configured); writer .on_framed_data(Box::new(FullFramedOwner::new(frames))) @@ -52,7 +52,7 @@ async fn large_mirror_rotates_after_repair_exhaustion_and_replays_ordered_data() let mut configured = policy(16 * MIB as u64); Arc::get_mut(&mut configured.client).unwrap().large_mirror_copies = Some(2); let frames = 225; - let expected = vec![0x5a; frames * MAX_FRAME_PAYLOAD_BYTES]; + let expected = vec![0x5a; frames * MAX_FRAME_V2_PAYLOAD_BYTES]; let mut writer = client.prepare_large_write(Some(expected.len() as u64), configured); writer .on_framed_data(Box::new(FullFramedOwner::new(frames))) diff --git a/e2e-test/tests/chunk_client/reader.rs b/e2e-test/tests/chunk_client/reader.rs index 56ebd96b1..45b8312fa 100644 --- a/e2e-test/tests/chunk_client/reader.rs +++ b/e2e-test/tests/chunk_client/reader.rs @@ -21,7 +21,7 @@ use crowdb_protocol::chunkdb::rpc::{ }; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::Segment; -use crowdb_protocol::frame::{parse_frame, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_protocol::frame::{parse_frame, MAX_FRAME_V2_PAYLOAD_BYTES}; use super::stack::TestStack as E2eStack; diff --git a/e2e-test/tests/chunk_client/reader/data_loss.rs b/e2e-test/tests/chunk_client/reader/data_loss.rs index d4564cced..f924074f8 100644 --- a/e2e-test/tests/chunk_client/reader/data_loss.rs +++ b/e2e-test/tests/chunk_client/reader/data_loss.rs @@ -60,7 +60,7 @@ async fn partial_read_preserves_healthy_ec_shard_ranges_around_data_loss() { panic!("large write did not produce EC"); }; let reader = reader_with_segment_failures(&stack, ec.segments[1..3].to_vec()).await; - let payload_per_shard = 16 * MAX_FRAME_PAYLOAD_BYTES as u64; + let payload_per_shard = 16 * MAX_FRAME_V2_PAYLOAD_BYTES as u64; let partial = reader .read_range_partial(&result.locations, 0, 4 * MIB as u64) .await @@ -105,7 +105,7 @@ async fn partial_read_preserves_healthy_ec_shard_ranges_around_data_loss() { assert!(matches!( stream.next_chunk().await.unwrap(), Err(ReadError::FailedRange { start, end, .. }) - if start == payload_per_shard && end == payload_per_shard + MAX_FRAME_PAYLOAD_BYTES as u64 + if start == payload_per_shard && end == payload_per_shard + MAX_FRAME_V2_PAYLOAD_BYTES as u64 )); assert!(stream.next_chunk().await.is_none()); } diff --git a/e2e-test/tests/chunk_client/small.rs b/e2e-test/tests/chunk_client/small.rs index 46261eacd..fd9954785 100644 --- a/e2e-test/tests/chunk_client/small.rs +++ b/e2e-test/tests/chunk_client/small.rs @@ -5,7 +5,7 @@ use super::stack::TestStack as E2eStack; use bytes::Bytes; use crowdb_chunk_client::{ChunkIoClient, ChunkIoWriter, SmallWritePolicy}; use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkType, Location, Strip}; -use crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES; +use crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES; use crowdb_protocol::frame::{parse_frame, FrameMagic}; use std::time::Duration; const KIB: usize = 1024; @@ -23,7 +23,7 @@ fn policy() -> SmallWritePolicy { control_interval: Duration::from_millis(1), cooldown: Duration::from_millis(1), chunk_capacity: 2 * MIB as u64, - object_limit: MAX_FRAME_PAYLOAD_BYTES, + object_limit: MAX_FRAME_V2_PAYLOAD_BYTES, mirror_copies: 2, ..SmallWritePolicy::new(crowdb_protocol::chunkdb::rpc::ChunkType::S3) } @@ -152,7 +152,7 @@ async fn small_write_rotates_strips_and_chunks_without_splitting_objects() { async fn write_full_small_strip(client: &ChunkIoClient, value: u8) -> Vec<(Bytes, Location)> { let mut frames = Vec::with_capacity(FRAMES_PER_MIB_STRIP); for _ in 0..FRAMES_PER_MIB_STRIP { - let payload = Bytes::from(vec![value; MAX_FRAME_PAYLOAD_BYTES]); + let payload = Bytes::from(vec![value; MAX_FRAME_V2_PAYLOAD_BYTES]); let location = write_object(client, payload.clone()).await; frames.push((payload, location)); } @@ -186,7 +186,7 @@ async fn assert_mirror_data(stack: &E2eStack, chunk: &Chunk, location: &Location .await; let frame = parse_frame(&actual, location.chunk_id.expect("location chunk")) .expect("durable RepoSmall frame"); - assert_eq!(frame.header.magic, FrameMagic::RepoSmallV1); + assert_eq!(frame.header.magic, FrameMagic::RepoSmallV2); assert_eq!(frame.payload, expected); } } diff --git a/e2e-test/tests/chunk_client/small_flow.rs b/e2e-test/tests/chunk_client/small_flow.rs index 55aa5cbb5..0cd58d485 100644 --- a/e2e-test/tests/chunk_client/small_flow.rs +++ b/e2e-test/tests/chunk_client/small_flow.rs @@ -22,7 +22,7 @@ use crowdb_protocol::chunkdb::rpc::{ }; use crowdb_protocol::common::DiskId; use crowdb_protocol::diskdb::rpc::Segment; -use crowdb_protocol::frame::{parse_frame, FrameMagic, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_protocol::frame::{parse_frame, FrameMagic, MAX_FRAME_V2_PAYLOAD_BYTES}; use super::stack::TestStack as E2eStack; @@ -138,7 +138,7 @@ fn policy() -> SmallWritePolicy { control_interval: Duration::from_millis(1), cooldown: Duration::from_millis(1), chunk_capacity: 2 * MIB as u64, - object_limit: MAX_FRAME_PAYLOAD_BYTES, + object_limit: MAX_FRAME_V2_PAYLOAD_BYTES, mirror_copies: 2, ..SmallWritePolicy::new(crowdb_protocol::chunkdb::rpc::ChunkType::S3) } @@ -153,7 +153,7 @@ async fn write_object(client: &ChunkIoClient, data: Bytes) -> Location { async fn write_full_small_strip(client: &ChunkIoClient, value: u8) -> Vec<(Bytes, Location)> { let mut frames = Vec::with_capacity(FRAMES_PER_MIB_STRIP); for _ in 0..FRAMES_PER_MIB_STRIP { - let payload = Bytes::from(vec![value; MAX_FRAME_PAYLOAD_BYTES]); + let payload = Bytes::from(vec![value; MAX_FRAME_V2_PAYLOAD_BYTES]); let location = write_object(client, payload.clone()).await; frames.push((payload, location)); } @@ -237,7 +237,7 @@ async fn assert_mirror_data(stack: &E2eStack, chunk: &Chunk, location: &Location .await; let frame = parse_frame(&actual, location.chunk_id.expect("location chunk")) .expect("durable RepoSmall frame"); - assert_eq!(frame.header.magic, FrameMagic::RepoSmallV1); + assert_eq!(frame.header.magic, FrameMagic::RepoSmallV2); assert_eq!(frame.payload, expected); } } diff --git a/e2e-test/tests/chunk_client/small_flow/batching.rs b/e2e-test/tests/chunk_client/small_flow/batching.rs index 84ff64d0f..1113a06f4 100644 --- a/e2e-test/tests/chunk_client/small_flow/batching.rs +++ b/e2e-test/tests/chunk_client/small_flow/batching.rs @@ -125,7 +125,7 @@ async fn small_write_scales_out_on_queued_bytes_then_scales_in_when_empty() { configured.scale_out_queue_bytes = 64 * KIB; configured.scale_out_queue_objects = configured.queue_capacity; let stack = E2eStack::start(configured).await; - let completed = concurrent_writes(&stack, 64, MAX_FRAME_PAYLOAD_BYTES).await; + let completed = concurrent_writes(&stack, 64, MAX_FRAME_V2_PAYLOAD_BYTES).await; wait_for_metrics(&stack.client, || stack.client.small_write_metrics().scale_out > 0).await; assert_eq!(completed.len(), 64); diff --git a/e2e-test/tests/chunk_client/small_flow/conversion.rs b/e2e-test/tests/chunk_client/small_flow/conversion.rs index e6b337603..a9d989520 100644 --- a/e2e-test/tests/chunk_client/small_flow/conversion.rs +++ b/e2e-test/tests/chunk_client/small_flow/conversion.rs @@ -188,7 +188,7 @@ async fn manual_chunkdb_trigger_converts_closed_active_range_end_to_end() { .await; assert_eq!(actual, expected); } - let appended_data = Bytes::from(vec![0xd3; MAX_FRAME_PAYLOAD_BYTES]); + let appended_data = Bytes::from(vec![0xd3; MAX_FRAME_V2_PAYLOAD_BYTES]); let appended = write_object(&stack.client, appended_data.clone()).await; assert_eq!(appended.chunk_id, location.chunk_id); assert_eq!(appended.offset, 8 * MIB as u64); diff --git a/e2e-test/tests/chunk_client/small_flow/repair_exhaustion.rs b/e2e-test/tests/chunk_client/small_flow/repair_exhaustion.rs index 598ce218a..01c8a683a 100644 --- a/e2e-test/tests/chunk_client/small_flow/repair_exhaustion.rs +++ b/e2e-test/tests/chunk_client/small_flow/repair_exhaustion.rs @@ -15,8 +15,8 @@ async fn small_write_repair_preserves_acknowledged_prefix_in_open_block() { failed_disk: Mutex::new(None), }); let client = ChunkIoClient::from_parts_with_small_policy(allocator, fault, policy()).unwrap(); - let prefix_data = Bytes::from(vec![0x31; MAX_FRAME_PAYLOAD_BYTES - KIB]); - let patch_data = Bytes::from(vec![0x72; MAX_FRAME_PAYLOAD_BYTES]); + let prefix_data = Bytes::from(vec![0x31; MAX_FRAME_V2_PAYLOAD_BYTES - KIB]); + let patch_data = Bytes::from(vec![0x72; MAX_FRAME_V2_PAYLOAD_BYTES]); let prefix = write_object(&client, prefix_data.clone()).await; let patch = write_object(&client, patch_data.clone()).await; assert_eq!(prefix.chunk_id, patch.chunk_id); @@ -43,9 +43,12 @@ async fn small_write_new_chunk_repair_exhaustion_preserves_sealed_predecessor() let client = ChunkIoClient::from_parts_with_small_policy(allocator, fault, configured).unwrap(); let predecessor_group = write_full_small_strip(&client, 0x29).await; let predecessor = predecessor_group[0].1.clone(); - let mut failed = client.prepare_small_write(MAX_FRAME_PAYLOAD_BYTES).await.unwrap(); + let mut failed = client + .prepare_small_write(MAX_FRAME_V2_PAYLOAD_BYTES) + .await + .unwrap(); failed - .on_data(Bytes::from(vec![0x81; MAX_FRAME_PAYLOAD_BYTES])) + .on_data(Bytes::from(vec![0x81; MAX_FRAME_V2_PAYLOAD_BYTES])) .await .unwrap(); assert!(matches!(failed.on_finish().await, Err(IoError::WriteFailed(_)))); diff --git a/e2e-test/tests/chunk_client/small_flow/replica_repair.rs b/e2e-test/tests/chunk_client/small_flow/replica_repair.rs index 2a39216c6..46d887735 100644 --- a/e2e-test/tests/chunk_client/small_flow/replica_repair.rs +++ b/e2e-test/tests/chunk_client/small_flow/replica_repair.rs @@ -15,7 +15,7 @@ async fn small_write_repairs_failed_replica_through_real_chunkdb_and_diskio() { failed_disk: Mutex::new(None), }); let client = ChunkIoClient::from_parts_with_small_policy(allocator, fault.clone(), policy()).unwrap(); - let data = Bytes::from(vec![0x5a; MAX_FRAME_PAYLOAD_BYTES]); + let data = Bytes::from(vec![0x5a; MAX_FRAME_V2_PAYLOAD_BYTES]); let location = write_object(&client, data.clone()).await; let failed_disk = fault.failed_disk.lock().unwrap().expect("injected disk identity"); client.shutdown_small_writes().await.unwrap(); @@ -46,7 +46,7 @@ async fn small_write_repairs_failed_replica_through_real_chunkdb_and_diskio() { async fn protected_small_write_reports_real_diskio_write_failure() { let stack = E2eStack::start_with_diskio_fault_rate(policy(), 1.0).await; let result = tokio::time::timeout(Duration::from_secs(15), async { - let data = Bytes::from(vec![0x5a; MAX_FRAME_PAYLOAD_BYTES]); + let data = Bytes::from(vec![0x5a; MAX_FRAME_V2_PAYLOAD_BYTES]); let mut writer = stack.client.prepare_small_write(data.len()).await?; writer.on_data(data).await?; writer.on_finish().await.map(|_| ()) diff --git a/e2e-test/tests/chunk_stream.rs b/e2e-test/tests/chunk_stream.rs index 7bfe9b552..bf6e5a4fd 100644 --- a/e2e-test/tests/chunk_stream.rs +++ b/e2e-test/tests/chunk_stream.rs @@ -3,10 +3,10 @@ use bytes::Bytes; use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, ChunkReadPolicy, SmallWritePolicy}; -use crowdb_chunk_stream::StreamRegistry; use crowdb_chunk_stream::{ - ProductionStreamRuntime, StreamBinding, StreamBindingState, StreamConfig, StreamError, StreamName, + ProductionStreamRuntime, StreamConfig, StreamError, StreamName, StreamRecord, StreamRecordState, }; +use crowdb_chunk_stream::{StreamMetadataStore, StreamRegistry}; use std::time::{Duration, Instant}; fn witness() -> std::path::PathBuf { @@ -39,7 +39,7 @@ async fn connect_chunk_io() -> ChunkIoClient { #[tokio::test] #[ignore = "requires the owned release container fixture"] -async fn production_idle_writer_stops_after_ownership_takeover() { +async fn production_released_idle_writer_stops_after_owner_transfer() { let io = connect_chunk_io().await; let config = StreamConfig { liveness_interval: Duration::from_millis(50), @@ -56,13 +56,15 @@ async fn production_idle_writer_stops_after_ownership_takeover() { let name = StreamName { high: 42, low: 142 }; old_runtime .registry() - .create(StreamBinding { + .create(StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, stream_name: name, metadata_group_id: 1, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("idle-authority-e2e".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("idle-authority-e2e".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }) .await .unwrap(); @@ -76,8 +78,12 @@ async fn production_idle_writer_stops_after_ownership_takeover() { StreamConfig::default(), ) .unwrap(); - let new = new_runtime.open(name, 0, 2).await.unwrap(); - let head = new.observe_metadata(None, 0).unwrap(); + old.release_writer().await.unwrap(); + let new = tokio::time::timeout(Duration::from_secs(10), new_runtime.open(name, 0, 2)) + .await + .expect("drained release must not wait for the live predecessor process to expire") + .unwrap(); + let published = new.observe_metadata(None, 0).unwrap(); // Observe three real idle periods without invoking an old-writer mutation. tokio::time::sleep(Duration::from_millis(150)).await; assert_eq!(old.metrics().rollovers, 0); @@ -87,8 +93,8 @@ async fn production_idle_writer_stops_after_ownership_takeover() { ); let reader = new_runtime.open_read_only(name, 0, 2).await.unwrap(); let observed = reader.observe_metadata(None, 0).unwrap(); - assert_eq!(observed.generation, head.generation); - assert_eq!(observed.active, head.active); + assert_eq!(observed.generation, published.generation); + assert_eq!(observed.chunk, published.chunk); let after = new.append(&[Bytes::from_static(b"new")]).await.unwrap(); assert_ne!(before.chunk_id, after.chunk_id); assert_eq!(new.read_at(0, 6).await.unwrap(), Bytes::from_static(b"oldnew")); @@ -110,16 +116,18 @@ async fn production_stream_recovers_exact_bytes_after_service_restarts() { if std::env::var("CROWDB_E2E_PHASE").as_deref() == Ok("write") { first_runtime .registry() - .create(StreamBinding { + .create(StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, stream_name, metadata_group_id: 1, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("restart-e2e".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("restart-e2e".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }) .await - .expect("publish stream binding"); + .expect("publish Stream record"); let first_stream = first_runtime .create_registered(stream_name, 0, 1) @@ -154,7 +162,7 @@ async fn production_stream_recovers_exact_bytes_after_service_restarts() { let second_stream = second_runtime .open(stream_name, 0, 2) .await - .expect("recover authoritative stream head"); + .expect("recover authoritative Stream tail"); let second = second_stream .append(&[Bytes::from_static(b"after-restart")]) .await @@ -186,16 +194,18 @@ async fn production_stream_write_returns_after_diskio_failure() { let stream_name = StreamName { high: 42, low: 142 }; runtime .registry() - .create(StreamBinding { + .create(StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, stream_name, metadata_group_id: 1, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("diskio-failure-e2e".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("diskio-failure-e2e".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }) .await - .expect("publish stream binding"); + .expect("publish Stream record"); let stream = runtime .create_registered(stream_name, 0, 1) .await @@ -204,6 +214,7 @@ async fn production_stream_write_returns_after_diskio_failure() { .append(&[Bytes::from_static(b"durable|")]) .await .expect("append before DiskIO failure"); + let published = stream.observe_metadata(None, 0).unwrap(); for index in 5..11 { crate::common::control::node("kill", index).await; } @@ -216,16 +227,28 @@ async fn production_stream_write_returns_after_diskio_failure() { .expect("journal append exceeded the 15-second fault budget"); assert!(result.is_err(), "journal append succeeded without DiskIO"); assert_eq!(stream.tail(), 8, "failed append advanced the journal tail"); + // All segments are offline, so the old chunk cannot drain and seal. A + // successor must not become authoritative before that boundary completes. assert_eq!( stream.metrics().rollovers, - 1, - "failed mirror write did not rotate once" + 0, + "offline segments allowed an unsealed chunk handoff" ); + let observed = stream.observe_metadata(None, 0).unwrap(); + assert_eq!(observed.generation, published.generation); + assert_eq!(observed.chunk, published.chunk); + assert_eq!(stream.metrics().completed, 1); + assert_eq!(stream.metrics().failed, 1); + assert_eq!( + stream.append(&[Bytes::from_static(b"still-unavailable")]).await, + Err(StreamError::WriteStalled) + ); + assert_eq!(stream.tail(), 8); } #[tokio::test] #[ignore = "requires the owned release container fixture"] -async fn production_stream_write_returns_after_live_diskio_errors() { +async fn production_stream_initialization_returns_after_live_diskio_errors() { let chunk_io = connect_chunk_io().await; let runtime = ProductionStreamRuntime::new( crate::common::hardware::kv_client().into(), @@ -238,32 +261,38 @@ async fn production_stream_write_returns_after_live_diskio_errors() { let stream_name = StreamName { high: 42, low: 143 }; runtime .registry() - .create(StreamBinding { + .create(StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, stream_name, metadata_group_id: 1, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("live-diskio-error-e2e".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("live-diskio-error-e2e".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }) .await - .expect("publish stream binding"); - let stream = runtime - .create_registered(stream_name, 0, 1) - .await - .expect("create stream"); + .expect("publish Stream record"); let result = tokio::time::timeout( Duration::from_secs(15), - stream.append(&[Bytes::from_static(b"record")]), + runtime.create_registered(stream_name, 0, 1), ) .await - .expect("journal append exceeded the 15-second fault budget"); + .expect("strip initialization exceeded the 15-second fault budget"); assert!( result.is_err(), - "journal append succeeded with all DiskIO writes failing" + "uninitialized strips became writable with all DiskIO writes failing" ); - assert_eq!(stream.tail(), 0); - assert_eq!(stream.metrics().rollovers, 1); + let metadata = + crowdb_chunk_stream::KvStreamMetadataStore::new(crate::common::hardware::kv_client().into(), 0, 1) + .unwrap(); + assert!(metadata + .record_store(1) + .unwrap() + .load_tail(stream_name) + .await + .unwrap() + .is_none()); for index in 5..11 { let state = crate::common::control::node("status", index).await; assert_eq!(state["Running"], true, "faulted Diskio exited"); diff --git a/e2e-test/tests/chunkdb/conversion.rs b/e2e-test/tests/chunkdb/conversion.rs index ed8c130bf..cd175bc27 100644 --- a/e2e-test/tests/chunkdb/conversion.rs +++ b/e2e-test/tests/chunkdb/conversion.rs @@ -29,7 +29,6 @@ pub(super) async fn converted_partial() -> (ChunkdbClient, ConversionDiskIo, Chu .seal_chunk(SealChunkRequest { chunk_id: Some(id), seal_bytes: length, - ..Default::default() }) .await .unwrap() diff --git a/e2e-test/tests/chunkdb/conversion/foreground.rs b/e2e-test/tests/chunkdb/conversion/foreground.rs index d69c3df18..d8df434db 100644 --- a/e2e-test/tests/chunkdb/conversion/foreground.rs +++ b/e2e-test/tests/chunkdb/conversion/foreground.rs @@ -16,7 +16,6 @@ async fn expired_client_claim_preserves_dirty_destination_tail_and_completes_onc .seal_chunk(SealChunkRequest { chunk_id: Some(id), seal_bytes: length, - ..Default::default() }) .await .unwrap() diff --git a/e2e-test/tests/chunkdb/lifecycle.rs b/e2e-test/tests/chunkdb/lifecycle.rs index 81c291a5f..66a214ca6 100644 --- a/e2e-test/tests/chunkdb/lifecycle.rs +++ b/e2e-test/tests/chunkdb/lifecycle.rs @@ -31,7 +31,6 @@ async fn allocate_seal_delete_preserves_physical_placement_and_free_facts() { .seal_chunk(SealChunkRequest { chunk_id: Some(id), seal_bytes: 100 * 1024 - 17, - ..Default::default() }) .await .unwrap() diff --git a/e2e-test/tests/chunkdb/recovery.rs b/e2e-test/tests/chunkdb/recovery.rs index cc28b82cb..26ea24c38 100644 --- a/e2e-test/tests/chunkdb/recovery.rs +++ b/e2e-test/tests/chunkdb/recovery.rs @@ -51,7 +51,6 @@ async fn crash_recovery_preserves_exact_chunk_records_and_placement() { .seal_chunk(SealChunkRequest { chunk_id: Some(id), seal_bytes: 100 * 1024 - 17, - ..Default::default() }) .await .unwrap() diff --git a/e2e-test/tests/chunkdb/repair.rs b/e2e-test/tests/chunkdb/repair.rs index 3d2166565..7104ac65f 100644 --- a/e2e-test/tests/chunkdb/repair.rs +++ b/e2e-test/tests/chunkdb/repair.rs @@ -3,7 +3,8 @@ mod topology; -use crowdb_protocol::chunkdb::rpc::{QueryChunkRequest, Strip, UpdateChunkStripRequest}; +use crowdb_protocol::chunkdb::rpc::{QueryChunkRequest, ReplaceChunkStripRangeRequest, Strip}; +use crowdb_protocol::common::ChunkId; use std::time::{Duration, Instant}; #[tokio::test] @@ -19,10 +20,16 @@ async fn four_missing_shards_repair_partial_data_and_parity() { let missing: Vec<_> = [0, 7, 8, 9].map(|index| before.segments[index]).to_vec(); damaged.unavailable_segments = missing.clone(); let marked = client - .update_chunk_strip(UpdateChunkStripRequest { + .replace_chunk_strip_range(ReplaceChunkStripRangeRequest { chunk_id: Some(id), - strip_index: 0, - strip: Some(damaged), + expected_modify_ts: converted.modify_ts, + start_index: 0, + old_strips: vec![converted.strips[0].clone()], + replacement_strips: vec![damaged], + operation_id: Some(ChunkId { + high: id.high ^ converted.modify_ts, + low: id.low, + }), }) .await .unwrap() diff --git a/e2e-test/tests/console.rs b/e2e-test/tests/console.rs index eeacd563d..8640cf481 100644 --- a/e2e-test/tests/console.rs +++ b/e2e-test/tests/console.rs @@ -14,13 +14,6 @@ mod rpc_response_proxy; mod bootstrap_authority; mod bootstrap_intent; mod bootstrap_publication; -mod cli_data; -mod cli_direct; -mod cli_fixture; -mod cli_hardware; -mod cli_management; -mod cli_observation; -mod cli_registry; mod cluster; mod deployment; mod logical_delete; diff --git a/e2e-test/tests/console/cli_data.rs b/e2e-test/tests/console/cli_data.rs deleted file mode 100644 index b79c70d9a..000000000 --- a/e2e-test/tests/console/cli_data.rs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2026-present Gian - -//! CLI e2e for the KV data-plane: `kv data put / get / delete / scan` -//! through a system-group endpoint against a real -//! `crowdb-kv-server` with group 0 initialized. - -use std::time::Duration; - -use super::cli_fixture::{crowdb_cli_bin, run, spawn_group0}; - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -#[allow(clippy::too_many_lines)] -async fn kv_put_get_delete_round_trip() { - let g0 = spawn_group0().await; - let cli = crowdb_cli_bin(); - std::fs::write(&g0.config_path, "invalid local topology").unwrap(); - - // put on group 0 (system store). - let (code, stdout, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &[ - "kv", "data", "put", "-s", "0", "-g", "0", "-k", "color", "-v", "indigo", - ], - ); - assert_eq!(code, 0, "put stdout={stdout}\nstderr={stderr}"); - - // get — should find the value. - let (code, stdout, _) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["kv", "data", "get", "-s", "0", "-g", "0", "-k", "color"], - ); - assert_eq!(code, 0); - assert!(stdout.contains("indigo"), "stdout={stdout}"); - - // delete. - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["kv", "data", "delete", "-s", "0", "-g", "0", "-k", "color"], - ); - assert_eq!(code, 0, "stderr={stderr}"); - - // get → not found returns exit code 0 with "not found" message. - let (code, stdout, _) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["kv", "data", "get", "-s", "0", "-g", "0", "-k", "color"], - ); - assert_eq!(code, 0, "stdout={stdout}"); - assert!(stdout.contains("not found"), "stdout={stdout}"); - - // scan: seed two keys with a prefix, then scan. - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &[ - "kv", "data", "put", "-s", "0", "-g", "0", "-k", "scan/a", "-v", "1", - ], - ); - assert_eq!(code, 0, "stderr={stderr}"); - - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &[ - "kv", "data", "put", "-s", "0", "-g", "0", "-k", "scan/b", "-v", "2", - ], - ); - assert_eq!(code, 0, "stderr={stderr}"); - - let (code, stdout, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["kv", "data", "scan", "-s", "0", "-g", "0", "-P", "scan/"], - ); - assert_eq!(code, 0, "stderr={stderr}"); - assert!(stdout.contains("scan/a"), "stdout={stdout}"); - assert!(stdout.contains("scan/b"), "stdout={stdout}"); - - tokio::time::sleep(Duration::from_millis(50)).await; -} diff --git a/e2e-test/tests/console/cli_direct.rs b/e2e-test/tests/console/cli_direct.rs deleted file mode 100644 index f025b12cc..000000000 --- a/e2e-test/tests/console/cli_direct.rs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! CLI direct Group-0 acceptance: `cluster status` and `cluster rack -//! list` route directly through `--system-ip` / `--system-port` against a -//! real `crowdb-kv-server` with group 0 initialized — no `crowdb-web` -//! intermediary. - -use super::cli_fixture::{crowdb_cli_bin, run, spawn_group0}; - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -async fn cluster_status_via_direct_group0() { - let g0 = spawn_group0().await; - let cli = crowdb_cli_bin(); - - // `cluster status` should list store 0 (the system store). - let (code, stdout, stderr) = run(&cli, g0.mgmt_port, &g0.config_path, &["cluster", "status"]); - assert_eq!(code, 0, "cluster status stderr={stderr}"); - // Store 0 should be present (group 0 was initialized). - assert!( - stdout.contains('0') || stdout.contains("(no stores)"), - "cluster status stdout={stdout}" - ); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -async fn cluster_rack_list_ignores_legacy_local_state() { - let g0 = spawn_group0().await; - let cli = crowdb_cli_bin(); - - std::fs::write(&g0.config_path, "invalid local topology").unwrap(); - // `cluster rack list` reads rack 1 from Group 0 despite the old file. - let (code, stdout, stderr) = run(&cli, g0.mgmt_port, &g0.config_path, &["cluster", "rack", "list"]); - assert_eq!(code, 0, "cluster rack list stderr={stderr}"); - assert!(stdout.contains('1'), "cluster rack list stdout={stdout}"); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -async fn cluster_node_list_ignores_legacy_local_state() { - let g0 = spawn_group0().await; - let cli = crowdb_cli_bin(); - - std::fs::write(&g0.config_path, "invalid local topology").unwrap(); - // `cluster node list` reads node 1 from Group 0 despite the old file. - let (code, stdout, stderr) = run(&cli, g0.mgmt_port, &g0.config_path, &["cluster", "node", "list"]); - assert_eq!(code, 0, "cluster node list stderr={stderr}"); - assert!(stdout.contains('1'), "cluster node list stdout={stdout}"); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -async fn kv_server_controls_require_launch_registry() { - let g0 = spawn_group0().await; - let cli = crowdb_cli_bin(); - - // A legacy topology file cannot supply process launch policy. - let (code, stdout, stderr) = run(&cli, g0.mgmt_port, &g0.config_path, &["kv", "server", "list"]); - assert_eq!(code, 2, "stdout={stdout} stderr={stderr}"); - assert!(stderr.contains("--registry"), "stderr={stderr}"); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -async fn incremental_local_deploy_reads_group_zero_instead_of_legacy_file() { - let g0 = spawn_group0().await; - std::fs::write(&g0.config_path, "invalid local topology").unwrap(); - let (code, _, stderr) = run( - &crowdb_cli_bin(), - g0.mgmt_port, - &g0.config_path, - &["cluster", "local-deploy", "-t", "diskdb", "--data-groups", "99"], - ); - assert_eq!(code, 2, "stderr={stderr}"); - assert!(stderr.contains("99"), "stderr={stderr}"); - assert!(!stderr.contains("load config"), "stderr={stderr}"); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -async fn destroy_reads_group_zero_and_requires_launch_registry() { - let g0 = spawn_group0().await; - std::fs::write(&g0.config_path, "invalid local topology").unwrap(); - let (code, _, stderr) = run( - &crowdb_cli_bin(), - g0.mgmt_port, - &g0.config_path, - &["cluster", "destroy"], - ); - assert_eq!(code, 2, "stderr={stderr}"); - assert!(stderr.contains("--registry"), "stderr={stderr}"); - - let context = crowdb_console_shared::ops::OpContext::new( - g0.rpc_url.trim_start_matches("http://").to_string(), - vec![g0.mgmt_url.clone()], - g0.bootstrap_config.clone(), - ); - crowdb_console_shared::ops::cluster::init(&context, &[1]) - .await - .expect("publish confirmed bootstrap metadata"); - - let registry_path = g0.config_path.with_file_name("launches.toml"); - crowdb_console_shared::config::web::LaunchRegistry { - version: 1, - launches: Vec::new(), - } - .save(®istry_path) - .unwrap(); - let output = std::process::Command::new(crowdb_cli_bin()) - .args([ - "--registry", - registry_path.to_str().unwrap(), - "--system-ip", - &g0.system_ip, - "--system-port", - ]) - .arg(g0.mgmt_port.to_string()) - .args(["cluster", "destroy"]) - .env("CROWDB_CLI_STATE", &g0.config_path) - .output() - .unwrap(); - assert!( - output.status.success(), - "stdout={} stderr={}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); -} diff --git a/e2e-test/tests/console/cli_fixture.rs b/e2e-test/tests/console/cli_fixture.rs deleted file mode 100644 index 06712e585..000000000 --- a/e2e-test/tests/console/cli_fixture.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use super::{bootstrap_authority, fixture}; -use crowdb_console_shared::ConsoleConfig; -use std::path::PathBuf; - -pub(super) struct TestGroup0 { - pub system_ip: String, - pub mgmt_url: String, - pub rpc_url: String, - pub mgmt_port: u16, - pub config_path: PathBuf, - pub bootstrap_config: ConsoleConfig, -} - -pub(super) fn crowdb_cli_bin() -> PathBuf { - let path = PathBuf::from(std::env::var("CROWDB_E2E_CLI_BIN").expect("pinned host CLI")); - assert!(path.is_file(), "missing CLI {}", path.display()); - path -} - -pub(super) async fn spawn_group0() -> TestGroup0 { - let cluster = fixture::TestKvCluster::start().await; - let context = bootstrap_authority::context(&cluster).await; - context.config_mut().racks[0].name = "rack-1".into(); - crowdb_console_shared::ops::hardware::add_rack_to_group0(&context, 1, "rack-1") - .await - .unwrap(); - let node = context.config().nodes[0].clone(); - crowdb_console_shared::ops::hardware::add_node_to_group0(&context, node) - .await - .unwrap(); - let bootstrap_config = context.config().clone(); - TestGroup0 { - system_ip: cluster.mgmt_endpoints[0] - .trim_start_matches("http://") - .trim_end_matches(":7000") - .into(), - mgmt_url: cluster.mgmt_endpoints[0].clone(), - rpc_url: cluster.group0_leader_endpoint, - mgmt_port: 7000, - config_path: fixture::client_data_dir("cli").join("legacy-console.toml"), - bootstrap_config, - } -} - -pub(super) fn run(cli: &PathBuf, port: u16, state: &PathBuf, args: &[&str]) -> (i32, String, String) { - let addresses = crate::common::management::addresses(3); - let output = std::process::Command::new(cli) - .current_dir(state.parent().unwrap()) - .args(["--system-ip", &addresses[0], "--system-port", &port.to_string()]) - .env("CROWDB_CLI_STATE", state) - .args(args) - .output() - .expect("run pinned host CLI"); - ( - output.status.code().unwrap_or(-1), - String::from_utf8_lossy(&output.stdout).into_owned(), - String::from_utf8_lossy(&output.stderr).into_owned(), - ) -} diff --git a/e2e-test/tests/console/cli_hardware.rs b/e2e-test/tests/console/cli_hardware.rs deleted file mode 100644 index 2735ebed9..000000000 --- a/e2e-test/tests/console/cli_hardware.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright 2026-present Gian - -//! CLI e2e for `cluster rack/node` round-trips through a system-group endpoint against a real -//! `crowdb-kv-server` with the system group -//! initialized — no `crowdb-web` intermediary. - -use super::cli_fixture::{crowdb_cli_bin, run, spawn_group0}; - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -#[allow(clippy::too_many_lines)] -async fn rack_node_lifecycle() { - let g0 = spawn_group0().await; - let cli = crowdb_cli_bin(); - - // rack list — rack 1 already exists (from spawn_group0 config). - let (code, stdout, stderr) = run(&cli, g0.mgmt_port, &g0.config_path, &["cluster", "rack", "list"]); - assert_eq!(code, 0, "rack list stderr={stderr}"); - assert!(stdout.contains('1'), "stdout={stdout}"); - - // rack add — add a second rack. - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["cluster", "rack", "add", "--id", "2", "--name", "rack-two"], - ); - assert_eq!(code, 0, "rack add stderr={stderr}"); - let (code, stdout, _) = run(&cli, g0.mgmt_port, &g0.config_path, &["cluster", "rack", "list"]); - assert_eq!(code, 0); - assert!(stdout.contains('2'), "stdout={stdout}"); - - // node add — add node 2 on rack 2. - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["cluster", "node", "add", "--id", "2", "--rack", "2"], - ); - assert_eq!(code, 0, "node add stderr={stderr}"); - let (code, stdout, _) = run(&cli, g0.mgmt_port, &g0.config_path, &["cluster", "node", "list"]); - assert_eq!(code, 0); - assert!(stdout.contains('1') && stdout.contains('2'), "stdout={stdout}"); - - // node remove — remove node 2 (no server deployed on it). - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["cluster", "node", "remove", "--id", "2"], - ); - assert_eq!(code, 0, "node remove stderr={stderr}"); - let (code, stdout, _) = run(&cli, g0.mgmt_port, &g0.config_path, &["cluster", "node", "list"]); - assert_eq!(code, 0); - assert!( - !stdout.contains("2 2"), - "node 2 should be gone: stdout={stdout}" - ); -} diff --git a/e2e-test/tests/console/cli_management.rs b/e2e-test/tests/console/cli_management.rs deleted file mode 100644 index 8ed8d954e..000000000 --- a/e2e-test/tests/console/cli_management.rs +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2026-present Gian . - -//! CLI e2e for the KV logical plane: `kv store / group / replica` verbs -//! through a system-group endpoint against a real -//! `crowdb-kv-server` with group 0 initialized. - -use std::time::Duration; - -use super::cli_fixture::{crowdb_cli_bin, run, spawn_group0}; - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -#[allow(clippy::too_many_lines)] -async fn store_group_replica_round_trip() { - let g0 = spawn_group0().await; - let cli = crowdb_cli_bin(); - std::fs::write(&g0.config_path, "invalid local topology").unwrap(); - - let store_id = "9"; - let group_id = "90"; - let replica_id = "900"; - - // store add — add store 9 on node 1. - let (code, stdout, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["kv", "store", "add", "-I", store_id, "-n", "1"], - ); - assert_eq!(code, 0, "store add stdout={stdout}\nstderr={stderr}"); - - // store list — should contain store 9. - let (code, stdout, _) = run(&cli, g0.mgmt_port, &g0.config_path, &["kv", "store", "list"]); - assert_eq!(code, 0); - assert!(stdout.contains(store_id), "stdout={stdout}"); - - // group add — create group 90 on store 9, replica 900 on node 1. - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &[ - "kv", "group", "add", "-s", store_id, "-g", group_id, "-r", replica_id, "-n", "1", - ], - ); - assert_eq!(code, 0, "group add stderr={stderr}"); - - // group list — should contain group 90. - let (code, stdout, _) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["kv", "group", "list", "-s", store_id], - ); - assert_eq!(code, 0); - assert!(stdout.contains(group_id), "stdout={stdout}"); - - // group add — a second group 91 on store 9. - let group_id_2 = "91"; - let replica_id_2 = "910"; - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &[ - "kv", - "group", - "add", - "-s", - store_id, - "-g", - group_id_2, - "-r", - replica_id_2, - "-n", - "1", - ], - ); - assert_eq!(code, 0, "group add 2 stderr={stderr}"); - - // group list — should contain both groups. - let (code, stdout, _) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["kv", "group", "list", "-s", store_id], - ); - assert_eq!(code, 0); - assert!( - stdout.contains(group_id) && stdout.contains(group_id_2), - "stdout={stdout}" - ); - - // group remove — remove group 91. - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["kv", "group", "remove", "-s", store_id, "-g", group_id_2], - ); - assert_eq!(code, 0, "group remove stderr={stderr}"); - - // store remove — remove store 9. - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["kv", "store", "remove", "-I", store_id], - ); - assert_eq!(code, 0, "store remove stderr={stderr}"); - - tokio::time::sleep(Duration::from_millis(50)).await; -} diff --git a/e2e-test/tests/console/cli_observation.rs b/e2e-test/tests/console/cli_observation.rs deleted file mode 100644 index b0f2be103..000000000 --- a/e2e-test/tests/console/cli_observation.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2026-present Gian . - -//! CLI e2e for cluster observation: `cluster status` and `cluster -//! topology` through a system-group endpoint against a real -//! `crowdb-kv-server` with the system group initialized. - -use std::time::Duration; - -use super::cli_fixture::{crowdb_cli_bin, run, spawn_group0}; - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -#[allow(clippy::too_many_lines)] -async fn cluster_status_topology_via_direct_group0() { - let g0 = spawn_group0().await; - let cli = crowdb_cli_bin(); - - // Bootstrap requires an independent versioned intent and launch registry. - let (code, _, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["cluster", "init", "-n", "1"], - ); - assert_eq!(code, 2, "cluster init stderr={stderr}"); - assert!(stderr.contains("--registry"), "stderr={stderr}"); - std::fs::write(&g0.config_path, "invalid local topology").unwrap(); - - // status — lists stores from group-0 sysdata. - let (code, stdout, stderr) = run(&cli, g0.mgmt_port, &g0.config_path, &["cluster", "status"]); - assert_eq!(code, 0, "status stderr={stderr}"); - assert!(stdout.contains("(no stores)"), "stdout={stdout}"); - - // topology — from a node's /topology endpoint. - let (code, stdout, stderr) = run( - &cli, - g0.mgmt_port, - &g0.config_path, - &["cluster", "topology", "-n", "1"], - ); - assert_eq!(code, 0, "topology stderr={stderr}"); - assert!(stdout.contains("store"), "stdout={stdout}"); - - // Status reflects the uninitialized authority without a local fallback. - let (code, stdout, _) = run(&cli, g0.mgmt_port, &g0.config_path, &["cluster", "status"]); - assert_eq!(code, 0); - assert!(stdout.contains("(no stores)"), "stdout={stdout}"); - - tokio::time::sleep(Duration::from_millis(50)).await; -} diff --git a/e2e-test/tests/console/cli_registry.rs b/e2e-test/tests/console/cli_registry.rs deleted file mode 100644 index 71d1827eb..000000000 --- a/e2e-test/tests/console/cli_registry.rs +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -#![cfg(target_os = "linux")] - -use std::os::unix::fs::PermissionsExt; -use std::path::Path; -use std::process::Command; -use std::time::Duration; - -use super::cli_fixture::{crowdb_cli_bin, spawn_group0}; -use super::fixture::client_data_dir; -use crowdb_console_shared::bootstrap_intent::BootstrapIntent; -use crowdb_console_shared::config::web::{LaunchRecord, LaunchRegistry}; -use crowdb_console_shared::launch::LaunchRuntime; -use crowdb_console_shared::lifecycle; - -struct TestProcesses(Vec); -impl Drop for TestProcesses { - fn drop(&mut self) { - for pid in &self.0 { - if lifecycle::process_is_alive(*pid) { - let _ = lifecycle::stop_pid_with_timeout(*pid, Duration::from_secs(2)); - } - } - } -} - -fn run(registry: &Path, port: u16, args: &[&str]) -> String { - let args: Vec<_> = ["kv", "server"].into_iter().chain(args.iter().copied()).collect(); - run_command(registry, port, &args) -} - -fn run_command(registry: &Path, port: u16, args: &[&str]) -> String { - let output = Command::new(crowdb_cli_bin()) - .args(["--system-ip", &crate::common::management::addresses(3)[0]]) - .arg("--registry") - .arg(registry) - .arg("--system-port") - .arg(port.to_string()) - .env("CROWDB_CLI_STATE", registry.with_file_name("invalid-legacy.toml")) - .args(args) - .output() - .unwrap(); - assert!( - output.status.success(), - "CLI failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout).unwrap() -} - -fn record(directory: &Path, service: &str) -> LaunchRecord { - let binary = directory.join("service"); - std::fs::write(&binary, "#!/bin/sh\nexec sleep 60\n").unwrap(); - std::fs::set_permissions(&binary, std::fs::Permissions::from_mode(0o700)).unwrap(); - let config = directory.join("service.toml"); - std::fs::write(&config, "").unwrap(); - LaunchRecord { - node_id: 701, - service_id: service.into(), - host: "localhost".into(), - ssh_credential_ref: None, - ssh_user: None, - ssh_port: 22, - binary_path: binary, - service_config_path: config, - workspace: directory.to_owned(), - auto_start: false, - args: Vec::new(), - readiness_url: None, - } -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -async fn registry_hardware_uses_group_zero_across_cli_invocations() { - let g0 = spawn_group0().await; - let dir = client_data_dir("cli-registry-hardware"); - let path = dir.as_path().join("launches.toml"); - LaunchRegistry { - version: 1, - launches: Vec::new(), - } - .save(&path) - .unwrap(); - - run_command( - &path, - g0.mgmt_port, - &["cluster", "rack", "add", "--id", "2", "--name", "rack-two"], - ); - assert!(run_command(&path, g0.mgmt_port, &["cluster", "rack", "list"]).contains("rack-two")); - run_command( - &path, - g0.mgmt_port, - &[ - "cluster", - "node", - "add", - "--id", - "2", - "--rack", - "2", - "--host", - "10.0.0.2", - "--ssh-credential-ref", - "ops-key", - ], - ); - assert!(run_command(&path, g0.mgmt_port, &["cluster", "node", "list"]).contains("10.0.0.2")); - run_command( - &path, - g0.mgmt_port, - &["cluster", "rack", "add", "--id", "3", "--name", "empty"], - ); - run_command(&path, g0.mgmt_port, &["cluster", "rack", "remove", "--id", "3"]); - assert!(!run_command(&path, g0.mgmt_port, &["cluster", "rack", "list"]).contains("empty")); - run_command(&path, g0.mgmt_port, &["cluster", "node", "remove", "--id", "2"]); - assert!(!run_command(&path, g0.mgmt_port, &["cluster", "node", "list"]).contains("10.0.0.2")); - run_command(&path, g0.mgmt_port, &["cluster", "rack", "remove", "--id", "2"]); - assert!(!dir.as_path().join("invalid-legacy.toml").exists()); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -async fn registry_bootstrap_uses_sealed_intent_without_legacy_topology_file() { - let g0 = spawn_group0().await; - let dir = client_data_dir("cli-registry-bootstrap"); - let path = dir.as_path().join("launches.toml"); - LaunchRegistry { - version: 1, - launches: Vec::new(), - } - .save(&path) - .unwrap(); - let source = dir.as_path().join("bootstrap-source.toml"); - let intent = BootstrapIntent::capture(&g0.bootstrap_config, &[1]).unwrap(); - intent.seal(&source).unwrap(); - std::fs::write(dir.as_path().join("invalid-legacy.toml"), "invalid legacy config").unwrap(); - - run_command( - &path, - g0.mgmt_port, - &[ - "cluster", - "init", - "--nodes", - "1", - "--bootstrap-file", - source.to_str().unwrap(), - ], - ); - assert!(!path.with_extension("bootstrap-intent.toml").exists()); - intent - .seal(&path.with_extension("bootstrap-intent.toml")) - .unwrap(); - run_command(&path, g0.mgmt_port, &["cluster", "init", "--nodes", "1"]); - assert!(!path.with_extension("bootstrap-intent.toml").exists()); - assert_eq!( - std::fs::read_to_string(dir.as_path().join("invalid-legacy.toml")).unwrap(), - "invalid legacy config" - ); - assert!(run_command(&path, g0.mgmt_port, &["cluster", "rack", "list"]).contains("rack-1")); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires the owned release KV containers"] -async fn cli_uses_launch_registry_and_runtime_identity_without_legacy_state() { - let g0 = spawn_group0().await; - let dir = client_data_dir("cli-launch-registry"); - std::fs::write(dir.as_path().join("invalid-legacy.toml"), "invalid legacy config").unwrap(); - let record = record(dir.as_path(), "kv"); - let path = dir.as_path().join("launches.toml"); - LaunchRegistry { - version: 1, - launches: vec![record.clone()], - } - .save(&path) - .unwrap(); - let runtime = LaunchRuntime::for_registry(&path).unwrap(); - let mut guard = TestProcesses(Vec::new()); - run(&path, g0.mgmt_port, &["deploy", "--node", "701"]); - let first = runtime.status(&record).await.unwrap().unwrap(); - guard.0.push(first.pid); - run(&path, g0.mgmt_port, &["start", "--node", "701"]); - assert_eq!(runtime.status(&record).await.unwrap(), Some(first)); - assert!(run(&path, g0.mgmt_port, &["list"]).contains(&first.pid.to_string())); - run(&path, g0.mgmt_port, &["restart", "--node", "701"]); - let next = runtime.status(&record).await.unwrap().unwrap(); - guard.0.push(next.pid); - assert_ne!(first, next); - run(&path, g0.mgmt_port, &["stop", "--node", "701"]); - assert!(runtime.status(&record).await.unwrap().is_none()); - run(&path, g0.mgmt_port, &["delete", "--node", "701"]); - assert!(LaunchRegistry::load(&path).unwrap().launches.is_empty()); - assert_eq!( - std::fs::read_to_string(dir.as_path().join("invalid-legacy.toml")).unwrap(), - "invalid legacy config" - ); -} diff --git a/e2e-test/tests/console/deployment.rs b/e2e-test/tests/console/deployment.rs index dde726e37..0b8d27a8a 100644 --- a/e2e-test/tests/console/deployment.rs +++ b/e2e-test/tests/console/deployment.rs @@ -45,6 +45,7 @@ async fn concurrent_admissions_allocate_unique_stable_ids_through_group_zero() { ssh_user: "crowdb".into(), operation_id: uuid.into(), confirmed: false, + host_key: None, cancelled: false, }; let a = node("12345678-1234-4234-8234-123456789abc", "127.0.0.1"); @@ -181,6 +182,7 @@ async fn authenticated_node_update_preserves_ids_and_converges_from_another_cons ssh_user: "crowdb".into(), operation_id: uuid::Uuid::new_v4().to_string(), confirmed: true, + host_key: None, cancelled: false, }; let mut node = config().nodes.remove(0); diff --git a/e2e-test/tests/console/discovery.rs b/e2e-test/tests/console/discovery.rs index c5256c479..29960110e 100644 --- a/e2e-test/tests/console/discovery.rs +++ b/e2e-test/tests/console/discovery.rs @@ -57,6 +57,7 @@ async fn chunk_kv_raw_observation_preserves_load_and_partition_state() { let cluster = KvCluster::start().await; let svc = cluster.make_service_registry_client(); let payload = ChunkKvExtra { + process_epoch: 1, node_id: Some(1), http_endpoint: Some("http://127.0.0.1:15508".into()), capacity_bytes: 10_000, @@ -69,7 +70,26 @@ async fn chunk_kv_raw_observation_preserves_load_and_partition_state() { }], partition_loads: Vec::new(), }; - svc.register_chunk_kv(7, "127.0.0.1:15507", &payload) + let lease_client = crowdb_kv_client::StreamProcessLeaseClient::new(svc.shared_kv()); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() + .try_into() + .unwrap(); + let lease = lease_client + .claim( + crowdb_protocol::chunk_stream::StreamPurpose::Wal, + 7, + [1; 16], + None, + now, + 60_000, + ) + .await + .unwrap(); + lease_client + .publish_chunk_kv_observation(lease.identity, "127.0.0.1:15507", &payload, now) .await .expect("register chunk KV"); diff --git a/e2e-test/tests/console/group_publication.rs b/e2e-test/tests/console/group_publication.rs index b829066a3..cb6992dbc 100644 --- a/e2e-test/tests/console/group_publication.rs +++ b/e2e-test/tests/console/group_publication.rs @@ -35,7 +35,22 @@ async fn verify_group_publication(drop_reply: bool, target_has_store: bool) { let created_store = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(target_has_store)); let store_state = created_store.clone(); let group_state = created_store.clone(); + let topology_state = created_store.clone(); let app = Router::new() + .route( + "/topology", + get(move || { + let hosted = topology_state.load(Ordering::SeqCst); + async move { + let stores = if hosted { + vec![serde_json::json!({"store_id": 77, "listen_addr": "127.0.0.1:10100", "groups": []})] + } else { + Vec::new() + }; + axum::Json(serde_json::json!({"stores": stores})) + } + }), + ) .route( "/stores", post(move || { diff --git a/e2e-test/tests/console/native_browser.rs b/e2e-test/tests/console/native_browser.rs index 865757263..270b4208e 100644 --- a/e2e-test/tests/console/native_browser.rs +++ b/e2e-test/tests/console/native_browser.rs @@ -53,10 +53,10 @@ async fn page_layout_and_iceberg_inspection() { } #[tokio::test] -#[ignore = "requires actual stream extent directories and a host browser"] -async fn large_journal_pagination_and_restart() { +#[ignore = "requires actual Stream chunk records and a host browser"] +async fn large_journal_records_and_restart() { let context = TestNativeContext::connect().await; - super::native_storage::start_with_geometry(&context, Some((1024 * 1024, 1))).await; + super::native_storage::start_with_geometry(&context, Some(1024 * 1024)).await; super::native_journal::TestNativeJournal::seed(&context).await; let result = crate::common::control::native_browser().await; assert_eq!(result["passed"], true); diff --git a/e2e-test/tests/console/native_capacity.rs b/e2e-test/tests/console/native_capacity.rs index 349882a83..e9dcba390 100644 --- a/e2e-test/tests/console/native_capacity.rs +++ b/e2e-test/tests/console/native_capacity.rs @@ -18,6 +18,7 @@ pub(super) async fn prepare(context: &TestNativeContext) { let profile = crate::common::control::request(json!({"action":"native-storage-profile"})).await; let capacity = profile["spare_disk_gib"].as_u64().unwrap() * 1024u64.pow(3); assert_eq!(capacity, 4 * 1024u64.pow(4)); + let loaded_by = tokio::time::Instant::now() + Duration::from_secs(3); context .api( Method::POST, @@ -38,6 +39,17 @@ pub(super) async fn prepare(context: &TestNativeContext) { .find(|(_, node, _)| *node == 1) .unwrap() .0; + tokio::time::timeout_at(loaded_by, async { + loop { + let group = hardware.get_disk_group(rack_id, 1, 581).await.unwrap().unwrap(); + if group.value.status == HwStatus::Up as i32 { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("the actual DiskDB initializes the new group before its media is admitted"); // This is a fresh, exclusively owned truncated fixture, matching the normal // native fixture's trusted legacy bootstrap rather than a public bypass. let record = DiskStorageRecord { @@ -66,7 +78,7 @@ pub(super) async fn prepare(context: &TestNativeContext) { record.validate().unwrap(); hardware.admit_disk_storage(&record).await.unwrap(); hardware.activate_disk_storage(&record).await.unwrap(); - tokio::time::timeout(Duration::from_secs(3), async { + tokio::time::timeout_at(loaded_by, async { loop { let response = context .http diff --git a/e2e-test/tests/console/native_geometry.rs b/e2e-test/tests/console/native_geometry.rs index 79cf452dd..b33c1a923 100644 --- a/e2e-test/tests/console/native_geometry.rs +++ b/e2e-test/tests/console/native_geometry.rs @@ -23,12 +23,12 @@ async fn mixed_units_reject_consumers_without_starting_services() { .await[kind] .clone(); body["kind"] = json!(kind); - body["test_single_node"] = json!(false); + body["local_deployment"] = json!(false); body["http_port"] = json!(http); body["rpc_port"] = json!(rpc); if kind == "chunk-kv" { body["metadata_store_id"] = json!(0); - body["bootstrap_group_id"] = json!(1); + body["bootstrap_group_id"] = json!(2); } let response = context .http diff --git a/e2e-test/tests/console/native_journal.rs b/e2e-test/tests/console/native_journal.rs index a762ab064..e537fa04c 100644 --- a/e2e-test/tests/console/native_journal.rs +++ b/e2e-test/tests/console/native_journal.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Actual large extent directory through the owned Chunk-KV process chain. +//! Actual ordered Stream chunk records through the owned Chunk-KV process chain. use super::native_context::TestNativeContext; use crowdb_chunk_kv_client::{ @@ -51,11 +51,11 @@ impl TestNativeJournal { panic!("large journal lost its actual value"); }; assert_eq!(record.value, value); - verify_directory(context).await; + verify_chunk_records(context).await; } } -async fn verify_directory(context: &TestNativeContext) { +async fn verify_chunk_records(context: &TestNativeContext) { let catalog = context .api( Method::GET, @@ -73,16 +73,20 @@ async fn verify_directory(context: &TestNativeContext) { catalog["generation"].as_str().unwrap() ); let observed = context.api(Method::GET, &path, Value::Null, 200).await; - if observed["journal"]["next_offset"] == 100 { - assert_eq!(observed["journal"]["extent_pages"].as_array().unwrap().len(), 100); + if observed["journal"]["chunk_record"]["logical_start"] + .as_str() + .unwrap() + .parse::() + .unwrap() + > 0 + { + assert!(observed["journal"]["next_offset"].is_null()); + assert_eq!(observed["journal"]["chunk_record"]["state"], "Active"); found = true; break; } } - assert!( - found, - "actual writes must publish more than 100 extent page fences" - ); + assert!(found, "actual writes must rotate to a later Stream chunk record"); } pub(super) async fn wait_initial_split(source: &Group0ChunkKvRangeCatalogSource) { @@ -98,7 +102,7 @@ pub(super) async fn wait_initial_split(source: &Group0ChunkKvRangeCatalogSource) .map(|entry| entry.owner.instance_id) .collect(); // First placement changes the owning stream. Produce the large current - // directory after that real handoff rather than pinning its source. + // Stream chunk records after that real handoff rather than pinning its source. if catalog.entries().len() >= 2 && owners.len() >= 2 && catalog.entries().iter().all(|entry| { @@ -115,7 +119,8 @@ pub(super) async fn wait_initial_split(source: &Group0ChunkKvRangeCatalogSource) } assert!( started.elapsed() < std::time::Duration::from_secs(60), - "native initial split did not settle before directory workload" + "native initial split did not settle before chunk record workload: {:?}", + catalog.entries() ); } } diff --git a/e2e-test/tests/console/native_load.rs b/e2e-test/tests/console/native_load.rs index 93159c6e2..0bd1dc055 100644 --- a/e2e-test/tests/console/native_load.rs +++ b/e2e-test/tests/console/native_load.rs @@ -54,6 +54,10 @@ async fn verify(context: &TestNativeContext) { "[PHASE] native 32-MiB Chunk-KV load: {}ms", started.elapsed().as_millis() ); + // Wait for a post-write heartbeat before comparing retained storage. A + // sample published during the load can omit its final packs and split child. + let load_completed_ms = + u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).unwrap().as_millis()).unwrap(); // Observe across the normal 12-second serving lease, without changing cadence. let mut interval = tokio::time::interval(Duration::from_millis(100)); let after = loop { @@ -61,7 +65,11 @@ async fn verify(context: &TestNativeContext) { let current = observations(®istry).await; assert_progress(&mut previous, current.clone()); check_value(&client, 0).await; - if started.elapsed() >= Duration::from_secs(13) { + if started.elapsed() >= Duration::from_secs(13) + && current + .iter() + .all(|row| row.last_heartbeat_ms > load_completed_ms) + { for previous in &before { let latest = current .iter() diff --git a/e2e-test/tests/console/native_storage.rs b/e2e-test/tests/console/native_storage.rs index 5e852b8f5..04394ccd3 100644 --- a/e2e-test/tests/console/native_storage.rs +++ b/e2e-test/tests/console/native_storage.rs @@ -15,7 +15,7 @@ pub(super) async fn start(context: &TestNativeContext) { start_with_geometry(context, None).await; } -pub(super) async fn start_with_geometry(context: &TestNativeContext, geometry: Option<(u64, u32)>) { +pub(super) async fn start_with_geometry(context: &TestNativeContext, geometry: Option) { start_through(context, geometry, "access-server").await; } @@ -23,7 +23,7 @@ pub(super) async fn start_prerequisites(context: &TestNativeContext) { start_through(context, None, "chunkdb").await; } -async fn start_through(context: &TestNativeContext, geometry: Option<(u64, u32)>, last: &str) { +async fn start_through(context: &TestNativeContext, geometry: Option, last: &str) { start_diskdb(context, [128 * 1024; 3]).await; for (kind, http, rpc) in [ ("diskio", None, Some(7200)), @@ -69,6 +69,7 @@ pub(super) async fn start_diskdb(context: &TestNativeContext, units: [u64; 3]) { } async fn groups(context: &TestNativeContext) { + verify_manual_metadata(context).await; context .api( Method::POST, @@ -77,12 +78,12 @@ async fn groups(context: &TestNativeContext) { 201, ) .await; - for (store, replica) in [(1, 10), (0, 20)] { + for (store, group, replica) in [(1, 1, 10), (0, 2, 20)] { context .api( Method::POST, &format!("/api/stores/{store}/groups"), - json!({"group_id":1,"replica_id":replica,"nodes":[1,2,3]}), + json!({"group_id":group,"replica_id":replica,"nodes":[1,2,3]}), 201, ) .await; @@ -103,12 +104,33 @@ async fn groups(context: &TestNativeContext) { } } +async fn verify_manual_metadata(context: &TestNativeContext) { + context + .api(Method::GET, "/api/stores/0/groups/1", Value::Null, 404) + .await; + assert!(matches!( + context + .kv + .get( + 0, + 0, + crowdb_console_shared::deployment::preparation::DEFAULT_PREPARATION_KEY, + crowdb_kv_client::ReadMode::Linearizable, + None + ) + .await + .unwrap(), + crowdb_kv_client::GetOutcome::NotFound, + )); +} + async fn disk_group(context: &TestNativeContext, node: u64) { context .api( Method::POST, &format!("/api/nodes/{node}/disk-groups"), - json!({"id":node,"store_id":u64::from(node == 1),"group_id":1,"name":"storage"}), + json!({"id":node,"store_id":u64::from(node == 1), + "group_id":if node == 1 { 1 } else { 2 },"name":"storage"}), 201, ) .await; @@ -144,7 +166,11 @@ async fn wait_diskdb(context: &TestNativeContext) { .unwrap_or_default() }) .collect(); - if groups == BTreeSet::from([1, 2, 3]) { + if groups == BTreeSet::from([1, 2, 3]) + && crowdb_console_shared::deployment::services::diskdb_storage_ready(context.kv.clone()) + .await + .is_ok() + { break; } tokio::time::sleep(Duration::from_millis(100)).await; @@ -160,7 +186,7 @@ async fn deploy( kind: &str, http: Option, rpc: Option, - geometry: Option<(u64, u32)>, + geometry: Option, ) { let mut body = context .api( @@ -173,7 +199,7 @@ async fn deploy( .clone(); body["kind"] = json!(kind); body["instance_id"] = json!(node.to_string()); - body["test_single_node"] = json!(false); + body["local_deployment"] = json!(false); if let Some(port) = http { body["http_port"] = json!(port); } @@ -184,10 +210,9 @@ async fn deploy( "diskio" => body["disk_group_id"] = json!(node), "chunk-kv" => { body["metadata_store_id"] = json!(0); - body["bootstrap_group_id"] = json!(1); - if let Some((bytes, entries)) = geometry { + body["bootstrap_group_id"] = json!(2); + if let Some(bytes) = geometry { body["stream_chunk_capacity_bytes"] = json!(bytes); - body["stream_extent_page_entries"] = json!(entries); } } "access-server" => { diff --git a/e2e-test/tests/console/server_data.rs b/e2e-test/tests/console/server_data.rs index 44fd0f1e7..5d1672004 100644 --- a/e2e-test/tests/console/server_data.rs +++ b/e2e-test/tests/console/server_data.rs @@ -38,6 +38,7 @@ async fn put_get_delete_cycle() { replica_id, initial_role: None, start_election: None, + deployment: None, }, ) .await diff --git a/e2e-test/tests/console/server_management.rs b/e2e-test/tests/console/server_management.rs index c8bf365bf..617024267 100644 --- a/e2e-test/tests/console/server_management.rs +++ b/e2e-test/tests/console/server_management.rs @@ -54,6 +54,7 @@ async fn full_store_group_remote_cycle() { replica_id, initial_role: None, start_election: None, + deployment: None, }, ) .await @@ -68,6 +69,7 @@ async fn full_store_group_remote_cycle() { replica_id: replica_id_2, initial_role: None, start_election: None, + deployment: None, }, ) .await diff --git a/e2e-test/tests/console/web_access.rs b/e2e-test/tests/console/web_access.rs index eb4ac378f..91cd7026f 100644 --- a/e2e-test/tests/console/web_access.rs +++ b/e2e-test/tests/console/web_access.rs @@ -63,7 +63,7 @@ async fn native_access_deployment_provisions_private_credentials_and_reuses_them Method::POST, "/api/nodes/1/services/deploy", json!({"kind":"access-server", "instance_id":"902", - "http_port":7502, "s3_port":7500, "health_port":7501, "test_single_node":true}), + "http_port":7502, "s3_port":7500, "health_port":7501, "local_deployment":true}), 201, ) .await, diff --git a/e2e-test/tests/console/web_diskio_discovery.rs b/e2e-test/tests/console/web_diskio_discovery.rs index 7c70b2c14..1bb7d3e6d 100644 --- a/e2e-test/tests/console/web_diskio_discovery.rs +++ b/e2e-test/tests/console/web_diskio_discovery.rs @@ -104,9 +104,14 @@ async fn diskio_discovers_disk_groups_created_after_web_deployment() { }) .await .expect("node-local DiskIO discovers both late groups"); + let readiness = get(&http, &base, "/api/chunk-storage-readiness").await; assert_eq!( - get(&http, &base, "/api/chunk-storage-readiness").await["ready"], - true + readiness["ready"], false, + "DiskIO ownership alone cannot load DiskDB media" + ); + assert!( + readiness["reason"].as_str().unwrap().contains("DiskDB"), + "{readiness}" ); assert_eq!( get(&http, &base, "/api/nodes/1/disk-groups") diff --git a/e2e-test/tests/console/web_lifecycle.rs b/e2e-test/tests/console/web_lifecycle.rs index a1c73bfe8..0d9b25a7d 100644 --- a/e2e-test/tests/console/web_lifecycle.rs +++ b/e2e-test/tests/console/web_lifecycle.rs @@ -1,6 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +use crowdb_kv_client::{HardwareClient, ServiceRegistryClient}; use reqwest::{Client, StatusCode}; use serde_json::{json, Value}; use std::path::PathBuf; @@ -73,6 +74,7 @@ async fn prepare_storage(http: &Client, base: &str) { sysmd.add_group(0, 0).await.unwrap(); sysmd.add_group(0, 1).await.unwrap(); register_voters(http, &sysmd, &addresses[..3]).await; + let registry = crowdb_kv_client::ServiceRegistryClient::from_shared(kv.clone()); let hardware = crowdb_kv_client::HardwareClient::from_shared(kv); hardware .add_rack( @@ -109,8 +111,9 @@ async fn prepare_storage(http: &Client, base: &str) { json!({"disk_id":"0000000000000000-0000000000000901","disk_type":"Hdd","capacity_bytes":16_777_216,"zone_size_bytes":8_388_608,"unit_size_bytes":4096}), StatusCode::CREATED).await; post(http, base, "/api/nodes/1/services/deploy", - json!({"kind":"diskio","instance_id":"902","rpc_port":7200,"disk_group_id":1,"test_single_node":true}), + json!({"kind":"diskio","instance_id":"902","rpc_port":7200,"disk_group_id":1,"local_deployment":true}), StatusCode::CREATED).await; + prepare_diskdb(http, base, &hardware, ®istry).await; let deadline = std::time::Instant::now() + Duration::from_secs(5); loop { let readiness: Value = http @@ -131,6 +134,46 @@ async fn prepare_storage(http: &Client, base: &str) { } } +async fn prepare_diskdb( + http: &Client, + base: &str, + hardware: &HardwareClient, + registry: &ServiceRegistryClient, +) { + post( + http, + base, + "/api/nodes/1/diskdb/deploy", + json!({"rpc_port":7100,"keepalive_interval_secs":1}), + StatusCode::CREATED, + ) + .await; + let instance_id = tokio::time::timeout(Duration::from_secs(5), async { + loop { + let instances = registry.read_all_diskdb_instances().await.unwrap(); + if let Some((instance_id, _)) = instances.first() { + assert_eq!(instances.len(), 1); + break *instance_id; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("the actual DiskDB process registers its generated identity"); + let expires_at_ms = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap() + + 60_000; + hardware + .set_owner(1, 1, 1, instance_id, expires_at_ms) + .await + .unwrap(); +} + async fn register_voters(http: &Client, sysmd: &crowdb_kv_client::CrowdbSysmdClient, addresses: &[String]) { for (index, address) in addresses.iter().enumerate() { let topology: Value = http diff --git a/e2e-test/tests/console_cli_test.py b/e2e-test/tests/console_cli_test.py index bc4bc26bf..854d91d0c 100644 --- a/e2e-test/tests/console_cli_test.py +++ b/e2e-test/tests/console_cli_test.py @@ -5,21 +5,52 @@ from pathlib import Path import tempfile import unittest -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock, patch -from console_cli_fixture import CliContainerCluster, LABEL, ROOT_LABEL +from console_cli_fixture import CliContainerCluster, LABEL, OWNER_LABEL +from console_cli_target import CliAttachedCluster +from fixture import Cluster class CliOwnershipTest(unittest.TestCase): + def test_diagnostics_export_bounded_access_metrics_before_teardown(self): + with tempfile.TemporaryDirectory() as directory: + cluster = CliAttachedCluster("image", Path(directory), Path(directory)) + cluster.addresses = ["192.0.2.1"] + response = Mock() + response.read.return_value = b'crowdb_s3_chunk_readiness_errors_total{phase="diskio"} 1\n' + with patch("console_cli_client.urllib.request.build_opener") as build, \ + patch.object(Cluster, "diagnostics") as collect: + opener = build.return_value + opener.open.return_value = MagicMock() + opener.open.return_value.__enter__.return_value = response + cluster.diagnostics() + opener.open.assert_called_once_with("http://192.0.2.1:9091/_crowdb/metrics", timeout=2) + response.read.assert_called_once_with(1024 * 1024 + 1) + collect.assert_called_once() + self.assertIn('phase="diskio"', (cluster.artifacts / "access-metrics-0.prom").read_text()) + + def test_diagnostics_preserve_errors_and_continue_other_collection(self): + with tempfile.TemporaryDirectory() as directory: + cluster = CliAttachedCluster("image", Path(directory), Path(directory)) + cluster.addresses = ["192.0.2.1"] + with patch("console_cli_client.urllib.request.build_opener") as build, \ + patch.object(Cluster, "diagnostics") as collect: + build.return_value.open.side_effect = TimeoutError("metrics unavailable") + cluster.diagnostics() + collect.assert_called_once() + self.assertEqual((cluster.artifacts / "access-metrics-0.error").read_text(), "metrics unavailable") + def test_recovery_observation_rejects_foreign_owner_before_file_access(self): cluster = object.__new__(CliContainerCluster) cluster.servers = ["owned"] cluster.root = Path("/owned/root") cluster.image = "immutable-image" - cluster.state = lambda: {"id": "expected-owner"} + cluster.run_id = "expected-cluster" + cluster.state = lambda: {"owner": "expected-owner", "image_id": cluster.image} cluster.command = Mock(return_value=json.dumps([{ "Image": cluster.image, - "Config": {"Labels": {LABEL: "foreign", ROOT_LABEL: str(cluster.root)}}}])) + "Config": {"Labels": {LABEL: "foreign", OWNER_LABEL: "expected-owner"}}}])) with self.assertRaisesRegex(ValueError, "ownership"): cluster.read_owned("node-binding.json") cluster.command.assert_called_once_with("inspect", "owned") @@ -32,16 +63,29 @@ def test_recovery_observation_rejects_paths_before_runtime_access(self): cluster.read_owned(path) cluster.command.assert_not_called() - def test_cleanup_rechecks_listed_ownership_before_removal(self): + def test_cleanup_uses_independent_host_operation_with_case_scope(self): with tempfile.TemporaryDirectory() as directory: cluster = CliContainerCluster("image", Path(directory), Path(directory)) - cluster.start() - (cluster.root / "container-cluster.json").write_text(json.dumps({"id": "owner"})) - cluster.command = Mock(side_effect=["unexpected-member", json.dumps([{ - "Config": {"Labels": {LABEL: "foreign", ROOT_LABEL: str(cluster.root)}}}])]) - with self.assertRaisesRegex(AssertionError, "ownership"): + cluster.timings["status"] = "pass" + with patch("console_cli_fixture.subprocess.run") as run: + run.return_value.returncode = 0 cluster.close() - self.assertFalse(any("rm" in call.args for call in cluster.command.call_args_list)) + arguments = run.call_args.args[0] + self.assertTrue(arguments[1].endswith("container/oci-image/docker-local.py")) + self.assertEqual(arguments[2:], ["clean", "--cluster", cluster.run_id, "--delete-data"]) + self.assertEqual(run.call_args.kwargs["env"]["CROWDB_RUNTIME_ROOT"], str(cluster.root)) + self.assertTrue(cluster.closed) + + def test_failed_cleanup_preserves_retry_state(self): + with tempfile.TemporaryDirectory() as directory: + cluster = CliContainerCluster("image", Path(directory), Path(directory)) + with patch("console_cli_fixture.subprocess.run") as run: + run.return_value.returncode = 1 + run.return_value.stdout = "" + run.return_value.stderr = "daemon unavailable" + with self.assertRaisesRegex(RuntimeError, "daemon unavailable"): + cluster.close() + self.assertFalse(cluster.closed) def test_recovery_observation_rejects_invalid_node_before_inspection(self): cluster = object.__new__(CliContainerCluster) diff --git a/e2e-test/tests/containerd_profile_test.py b/e2e-test/tests/containerd_profile_test.py index 3ea1667c3..32fc3801e 100644 --- a/e2e-test/tests/containerd_profile_test.py +++ b/e2e-test/tests/containerd_profile_test.py @@ -13,7 +13,7 @@ class ContainerdProfileTest(unittest.TestCase): def test_profile_budget_and_selection_fail_before_runtime_access(self): - for args in (("--concurrency","2"),("--concurrency","1","--node-cpus",".75","--cpus","1.49"), + for args in (("--concurrency","1","--node-cpus","2","--cpus","4","--memory-mib","8192"),("--concurrency","2"),("--concurrency","1","--node-cpus",".75","--cpus","1.49"), ("--concurrency","1","--memory-mib","4096"),("--concurrency","1","--layer","kv")): with patch.object(sys,"argv",["run.py","--profile","containerd-cluster",*args]), \ patch("containerd_profile.docker") as docker, contextlib.redirect_stderr(io.StringIO()): @@ -22,7 +22,7 @@ def test_profile_budget_and_selection_fail_before_runtime_access(self): for profile, nodes in (("containerd-cluster",3),("containerd-host",1)): output=io.StringIO() with patch.object(sys,"argv",["run.py","--profile",profile,"--concurrency","1", - "--node-cpus",".75","--cpus","4","--memory-mib","6144","--list"]), \ + "--node-cpus","1","--cpus","4","--memory-mib","8192","--list"]), \ patch("containerd_profile.docker") as docker, contextlib.redirect_stdout(output): run.main() docker.assert_not_called() diff --git a/e2e-test/tests/deployment.rs b/e2e-test/tests/deployment.rs index afcbbfb1f..6a7793c36 100644 --- a/e2e-test/tests/deployment.rs +++ b/e2e-test/tests/deployment.rs @@ -24,7 +24,14 @@ async fn selected_kv_services_recover_acknowledged_data() { #[tokio::test] #[ignore = "requires the normal release-monitor container fixture"] async fn selected_storage_services_recover_acknowledged_data() { - witness(true).await; + witness(false).await; + let address = crate::common::management::addresses(1)[0].clone(); + let registry = crowdb_kv_client::ServiceRegistryClient::new(CrowdbKvClient::new(ClientConfig::new( + vec![format!("http://{address}:10000")], + ))); + let instances = registry.read_all_diskdb_instances().await.unwrap(); + assert_eq!(instances.len(), 1); + assert_eq!(instances[0].0, 1); } #[tokio::test] @@ -59,13 +66,13 @@ async fn witness_with_write(storage: bool, write: bool) { )]))); let disks = hardware.list_all_disks().await.unwrap(); if storage { - assert_eq!(disks.len(), 4); + assert_eq!(disks.len(), 1); assert!(disks .iter() .all(|disk| disk.value.zone_count == 1 && disk.capacity_bytes() == 16 * 1024 * 1024 * 1024)); - let owner = hardware.get_owner(1, 1, 101).await.unwrap().unwrap(); + let owner = hardware.get_owner(1, 1, 1).await.unwrap().unwrap(); assert_eq!(owner.instance_id, 1); - let binding = hardware.get_bind(1, 1, 101).await.unwrap().unwrap(); + let binding = hardware.get_bind(1, 1, 1).await.unwrap().unwrap(); assert_eq!((binding.store_id, binding.group_id), (0, 1)); } else { assert!(disks.is_empty()); diff --git a/e2e-test/tests/deployment/anonymous.rs b/e2e-test/tests/deployment/anonymous.rs index 1087c6a4d..7228511c2 100644 --- a/e2e-test/tests/deployment/anonymous.rs +++ b/e2e-test/tests/deployment/anonymous.rs @@ -12,7 +12,13 @@ async fn default_anonymous_volume_is_ready_and_monitor_death_exits_container() { .unwrap() .iter() .any(|mount| mount["Destination"] == "/opt/crowdb/data" && mount["Type"] == "volume")); - assert_eq!(info["manifest"]["state"], "ready"); + assert_eq!(info["status"]["phase"], "ready"); + assert_eq!(info["accepted"]["prepared"], true); + assert_eq!(info["accepted"]["node_id"], 1); + assert_eq!( + info["status"]["observation"]["cluster_id"], + info["accepted"]["bootstrap"]["cluster_id"] + ); assert!(info["log"] .as_str() .unwrap() diff --git a/e2e-test/tests/deployment/balance.rs b/e2e-test/tests/deployment/balance.rs index e8745ed48..ae1ef3168 100644 --- a/e2e-test/tests/deployment/balance.rs +++ b/e2e-test/tests/deployment/balance.rs @@ -2,13 +2,13 @@ // Licensed under the Apache License, Version 2.0. use crowdb_kv_client::{ClientConfig, CrowdbKvClient, GetOutcome, ReadMode}; -use crowdb_protocol::chunk_kv::DomainMonitorDescriptor; +use crowdb_protocol::chunk_kv::{ChunkKvRangeBalancePolicy, DomainMonitorDescriptor, SplitTransition}; use crowdb_protocol::key::{ChunkKvSplitKey, DomainMonitorKey, TextKey}; use std::time::{Duration, Instant}; #[tokio::test] #[ignore = "requires the complete release single-node profile"] -async fn single_node_disables_local_chunk_kv_split_planning() { +async fn single_node_preserves_default_chunk_kv_balance_policy() { super::witness_with_write(true, true).await; let address = crate::common::management::addresses(1)[0].clone(); let client = CrowdbKvClient::new(ClientConfig::new(vec![format!("http://{address}:10000")])); @@ -24,7 +24,10 @@ async fn single_node_disables_local_chunk_kv_split_planning() { panic!("single-node Chunk-KV monitor descriptor is missing"); }; let monitor: DomainMonitorDescriptor = serde_json::from_slice(&value).unwrap(); - assert!(monitor.chunk_kv_range_balance.is_none()); + assert_eq!( + monitor.chunk_kv_range_balance, + Some(ChunkKvRangeBalancePolicy::default()) + ); let deadline = Instant::now() + Duration::from_secs(12); loop { let observed = client @@ -34,7 +37,7 @@ async fn single_node_disables_local_chunk_kv_split_planning() { ChunkKvSplitKey::text_prefix_all().as_bytes(), &[], &[], - 1, + 128, ReadMode::Linearizable, None, false, @@ -42,10 +45,15 @@ async fn single_node_disables_local_chunk_kv_split_planning() { ) .await .unwrap(); - assert!( - observed.items.is_empty(), - "disabled balancing created a split transition" - ); + for (_, value) in &observed.items { + let split: SplitTransition = serde_json::from_slice(value).unwrap(); + assert!( + split.validate().is_ok(), + "default local split must preserve its durable identity: {split:?}" + ); + assert_eq!(split.parent_owner.instance_id, 1); + assert_eq!(split.child.owner.instance_id, 1); + } assert!(!observed.truncated && !observed.timed_out); if Instant::now() >= deadline { break; @@ -61,10 +69,7 @@ async fn single_node_disables_local_chunk_kv_split_planning() { let path = entry.unwrap().path(); if path.is_file() { count += 1; - let body = std::fs::read(path).unwrap(); - assert!(!body - .windows(b"local split planned".len()) - .any(|window| window == b"local split planned")); + assert!(std::fs::metadata(path).unwrap().len() > 0); } } assert!(count > 0, "the actual KV service must produce diagnostic logs"); diff --git a/e2e-test/tests/deployment/bootstrap.rs b/e2e-test/tests/deployment/bootstrap.rs index 419ab081c..e6437048f 100644 --- a/e2e-test/tests/deployment/bootstrap.rs +++ b/e2e-test/tests/deployment/bootstrap.rs @@ -3,26 +3,31 @@ #[tokio::test] #[ignore = "requires the interrupted normal release-monitor container"] -async fn interrupted_bootstrap_preserves_completed_steps() { +async fn interrupted_setup_preserves_completed_steps() { let root = std::path::PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").unwrap()); let before: serde_json::Value = - serde_json::from_slice(&std::fs::read(root.join("interrupted-manifest.json")).unwrap()).unwrap(); + serde_json::from_slice(&std::fs::read(root.join("interrupted-setup.json")).unwrap()).unwrap(); let after: serde_json::Value = - serde_json::from_slice(&std::fs::read(root.join("recovered-manifest.json")).unwrap()).unwrap(); - assert_eq!(before["state"], "initializing"); - assert_eq!(after["state"], "ready"); - assert_eq!(before["deployment_id"], after["deployment_id"]); - let steps = after["steps"].as_array().unwrap(); - assert!(steps.iter().all(|step| step["complete"] == true)); - let completed = before["steps"] + serde_json::from_slice(&std::fs::read(root.join("recovered-setup.json")).unwrap()).unwrap(); + assert_eq!(before["accepted"], after["accepted"]); + assert_eq!(before["setup"]["request"], after["setup"]["request"]); + assert_eq!(before["setup"]["status"]["phase"], "running"); + assert_eq!(after["setup"]["status"]["phase"], "ready"); + let completed = before["setup"]["status"]["services"] .as_array() .unwrap() .iter() - .filter(|step| step["complete"] == true) + .filter(|step| step["phase"] == "ready") .collect::>(); assert!(!completed.is_empty()); - assert!(completed.iter().all(|prior| steps - .iter() - .any(|step| step["name"] == prior["name"] && step["complete"] == true))); + let steps = after["setup"]["status"]["services"].as_array().unwrap(); + for prior in completed { + let retained = steps.iter().find(|step| step["kind"] == prior["kind"]).unwrap(); + assert_eq!(retained["phase"], "ready"); + if prior["kind"] != "paxos-kv" { + assert!(prior["operation_id"].is_string()); + } + assert_eq!(retained["operation_id"], prior["operation_id"]); + } super::witness_with_write(true, true).await; } diff --git a/e2e-test/tests/deployment/clients.rs b/e2e-test/tests/deployment/clients.rs index 2765c7a9f..d07ddd097 100644 --- a/e2e-test/tests/deployment/clients.rs +++ b/e2e-test/tests/deployment/clients.rs @@ -15,7 +15,7 @@ async fn official_clients_and_protocol_layout_survive_restart() { let runtime: serde_json::Value = serde_json::from_slice(&std::fs::read(root.join("deployment-runtime.json")).unwrap()).unwrap(); assert_eq!(runtime["modes"]["secrets/server.env"], "600"); - assert_eq!(runtime["modes"]["secrets/client.env"], "600"); + assert_eq!(runtime["modes"]["console/secrets/client.env"], "600"); assert_eq!(runtime["modes"]["crash"], "700"); for role in ["monitor", "kv"] { assert_eq!(runtime["cwd"][role], "/opt/crowdb/data/crash"); @@ -57,8 +57,8 @@ async fn official_clients_and_protocol_layout_survive_restart() { async fn verify_public_services(address: &str) { let client = crate::common::management::client(); let web = format!("http://{address}:9090"); - let authority: serde_json::Value = client - .get(format!("{web}/api/authority")) + let report: serde_json::Value = client + .get(format!("{web}/api/cluster/status")) .send() .await .unwrap() @@ -67,10 +67,17 @@ async fn verify_public_services(address: &str) { .json() .await .unwrap(); - assert_eq!(authority["source"], "group0"); - assert_eq!(authority["available"], true); - let preview: serde_json::Value = client - .get(format!("{web}/api/preview")) + assert_eq!(report["phase"], "ready"); + assert_eq!(report["observation"]["bound"], true); + let authority = report["observation"]["dependencies"] + .as_array() + .unwrap() + .iter() + .find(|fact| fact["id"] == "cluster/authority") + .unwrap(); + assert_eq!(authority["state"], "ready"); + let services: serde_json::Value = client + .get(format!("{web}/api/servers")) .send() .await .unwrap() @@ -79,8 +86,7 @@ async fn verify_public_services(address: &str) { .json() .await .unwrap(); - assert_eq!(preview["source"], "group0"); - assert!(!preview["services"].as_array().unwrap().is_empty()); + assert!(!services.as_array().unwrap().is_empty()); let stores: serde_json::Value = client .get(format!("{web}/api/stores")) .send() @@ -96,16 +102,32 @@ async fn verify_public_services(address: &str) { .unwrap() .iter() .any(|store| store["store_id"] == 0)); - for path in ["/api/stores", "/api/racks", "/api/node/admit"] { + let mode: serde_json::Value = client + .get(format!("{web}/api/mode")) + .send() + .await + .unwrap() + .error_for_status() + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(mode, serde_json::json!({"mode":"node","readonly":false})); + for (path, expected) in [ + ("/api/stores", 422), + ("/api/racks", 400), + ("/api/node/admit", 422), + ] { + let response = client + .post(format!("{web}{path}")) + .json(&serde_json::json!({})) + .send() + .await + .unwrap(); assert_eq!( - client - .post(format!("{web}{path}")) - .json(&serde_json::json!({})) - .send() - .await - .unwrap() - .status(), - reqwest::StatusCode::FORBIDDEN + response.status().as_u16(), + expected, + "malformed management request: {path}" ); } assert_eq!( @@ -120,7 +142,7 @@ async fn verify_public_services(address: &str) { } async fn verify_layout() { - let endpoint = format!("{}:12200", crate::common::management::addresses(1)[0]); + let endpoint = format!("{}:7301", crate::common::management::addresses(1)[0]); let transport = ChunkdbRpcTransport::new(); let mut start_token = None; let mut seen = [false; 2]; diff --git a/e2e-test/tests/deployment/failures.rs b/e2e-test/tests/deployment/failures.rs index b917b6885..59f024ab8 100644 --- a/e2e-test/tests/deployment/failures.rs +++ b/e2e-test/tests/deployment/failures.rs @@ -14,31 +14,10 @@ async fn restart_exhaustion_exits_the_container() { assert!(result["log"] .as_str() .unwrap() - .contains("\"kind\":\"restart_exhausted\"")); + .contains("web exceeded monitor restart budget")); assert_eq!(result["secret_free"], true); - for event in [ - "starting", - "ready", - "bootstrap_step_started", - "bootstrap_step_completed", - "child_started", - "child_stopped", - "child_exited", - "probe_failed", - "restarting", - "draining", - "stopped", - "restart_exhausted", - ] { - assert!( - result["events"] - .as_array() - .unwrap() - .iter() - .any(|value| value == event), - "missing {event}" - ); - } + assert_eq!(result["after"]["phase"], "failed"); + assert_eq!(result["after"]["services"]["web"]["restart_attempts"], 5); } #[tokio::test] @@ -48,9 +27,6 @@ async fn recovery_rejects_changed_durable_credentials() { let result = crate::common::control::monitor_failure("reject-changed-credentials").await; assert_eq!(result["state"]["Status"], "exited"); assert_ne!(result["state"]["ExitCode"], 0); - assert_eq!(result["ready_before"], result["ready_after"]); - assert!(result["log"] - .as_str() - .unwrap() - .contains("server credentials are incomplete")); + assert_eq!(result["credential_rejection"], true); + assert_eq!(result["secret_free"], true); } diff --git a/e2e-test/tests/deployment/kv_bootstrap.rs b/e2e-test/tests/deployment/kv_bootstrap.rs index d8c0311b8..d601feebd 100644 --- a/e2e-test/tests/deployment/kv_bootstrap.rs +++ b/e2e-test/tests/deployment/kv_bootstrap.rs @@ -1,78 +1,72 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_monitor::{ - kv_step_names, BootstrapManifest, BootstrapSession, DeploymentProfile, KvBootstrap, ManifestState, - MonitorLog, -}; +use crate::common::management::{client, request}; use reqwest::Method; use serde_json::{json, Value}; -use crate::common::management::{client, request}; - #[tokio::test] #[ignore = "requires the selected release KV container"] async fn monitor_creates_real_kv_groups_then_validates_after_restart() { let root = std::path::PathBuf::from(std::env::var("CROWDB_E2E_ARTIFACTS").unwrap()); - let profile = DeploymentProfile::load(root.join("profile.toml")).unwrap(); - let manifest_bytes = std::fs::read(root.join("managed-manifest.json")).unwrap(); - let manifest: BootstrapManifest = serde_json::from_slice(&manifest_bytes).unwrap(); - assert_eq!(manifest.state(), ManifestState::Ready); - assert_eq!(manifest.next_step(), None); - for name in kv_step_names(&profile).unwrap() { - assert_eq!(manifest.step_complete(&name), Some(true)); - } + let bytes = std::fs::read(root.join("managed-node.json")).unwrap(); + let accepted: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(accepted["prepared"], true); + assert_eq!(accepted["node_id"], 1); let write = std::env::var("CROWDB_E2E_PHASE").as_deref() == Ok("write"); - let witness = root.join("initial-kv-manifest.json"); + let witness = root.join("initial-kv-node.json"); if write { - std::fs::write(&witness, &manifest_bytes).unwrap(); + std::fs::write(&witness, &bytes).unwrap(); } else { - let initial: BootstrapManifest = serde_json::from_slice(&std::fs::read(witness).unwrap()).unwrap(); - assert_eq!(manifest, initial, "restart must preserve accepted initialization"); + let initial: Value = serde_json::from_slice(&std::fs::read(witness).unwrap()).unwrap(); + assert_eq!(accepted, initial, "restart must preserve accepted initialization"); } let origin = format!("http://{}:10000", crate::common::management::addresses(1)[0]); let http = client(); - let accepted = request( + let result = request( &http, &origin, Method::POST, "/system/prepare", - json!({"replica_id": 1, "bootstrap": manifest.system_bootstrap_identity()}), + json!({"replica_id": 1, "bootstrap": accepted["bootstrap"]}), 200, ) .await; - assert_eq!(accepted["bootstrap"], json!(manifest.system_bootstrap_identity())); - for group in &profile.groups { + assert_eq!(result["bootstrap"], accepted["bootstrap"]); + for group in [0, 1] { let ready = request( &http, &origin, Method::GET, - &format!("/stores/0/groups/{}/ready", group.group_id), + &format!("/stores/0/groups/{group}/ready"), Value::Null, 200, ) .await; assert_eq!(ready["ready"], true); - assert_eq!(ready["leader_id"], group.replica_id); + assert_eq!(ready["leader_id"], 1); assert_eq!(ready["voting_replicas"], 1); } - // A client-side manifest is a different bootstrap, even with identical - // replica geometry. It must not adopt the container's existing system group. - let data = root.join("client-data/kv-bootstrap"); - std::fs::create_dir_all(&data).unwrap(); - let names = kv_step_names(&profile).unwrap(); - let steps = names.iter().map(String::as_str).collect::>(); - let mut session = BootstrapSession::open(&data, b"profile", b"config", &steps).unwrap(); - std::fs::create_dir_all(data.join("log")).unwrap(); - let mut events = MonitorLog::open(&data.join("log"), profile.logs.clone()) + let mut foreign = accepted["bootstrap"].clone(); + foreign["operation_id"] = json!(uuid::Uuid::new_v4().to_string()); + let conflict = http + .post(format!("{origin}/system/prepare")) + .json(&json!({"replica_id": 1, "bootstrap": foreign})) + .send() .await .unwrap(); - let error = KvBootstrap::new(&origin) - .unwrap() - .reconcile(&mut session, &profile, &mut events) - .await - .unwrap_err(); - assert!(error.to_string().contains("conflicting bootstrap"), "{error}"); - assert_eq!(session.manifest().next_step(), Some("kv-group-0-0")); + assert_eq!(conflict.status(), 409); + let detail = conflict.text().await.unwrap(); + assert!(detail.contains("bootstrap"), "{detail}"); + let retained = request( + &http, + &origin, + Method::POST, + "/system/prepare", + json!({"replica_id": 1, "bootstrap": accepted["bootstrap"]}), + 200, + ) + .await; + assert_eq!(retained["bootstrap"], accepted["bootstrap"]); super::witness_with_write(false, write).await; } diff --git a/e2e-test/tests/deployment/listeners.rs b/e2e-test/tests/deployment/listeners.rs index 299eb582f..5a5ffc7bc 100644 --- a/e2e-test/tests/deployment/listeners.rs +++ b/e2e-test/tests/deployment/listeners.rs @@ -7,7 +7,7 @@ fn verify(protocol: &str, port: &str) { serde_json::from_slice(&std::fs::read(root.join("listener-recovery.json")).unwrap()).unwrap(); assert_eq!(result["protocol"], protocol); assert_eq!(result["fixed_listener"], port); - assert_eq!(result["event"], "child_exited"); + assert_eq!(result["signal"], "KILL"); let before_generation = result["before"]["generation"].as_u64().unwrap(); let after_generation = result["after"]["generation"].as_u64().unwrap(); assert!(after_generation > before_generation); @@ -17,11 +17,17 @@ fn verify(protocol: &str, port: &str) { #[test] #[ignore = "requires an owned fixed-listener release profile"] fn fixed_s3_listener_recovers_after_explicit_termination() { - verify("s3", "0.0.0.0:9091"); + verify( + "s3", + &format!("{}:9091", crate::common::management::addresses(1)[0]), + ); } #[test] #[ignore = "requires an owned fixed-listener release profile"] fn fixed_iceberg_listener_recovers_after_explicit_termination() { - verify("iceberg", "0.0.0.0:9092"); + verify( + "iceberg", + &format!("{}:9092", crate::common::management::addresses(1)[0]), + ); } diff --git a/e2e-test/tests/deployment/storage.rs b/e2e-test/tests/deployment/storage.rs index 7f813cf73..32d20fc67 100644 --- a/e2e-test/tests/deployment/storage.rs +++ b/e2e-test/tests/deployment/storage.rs @@ -2,9 +2,7 @@ // Licensed under the Apache License, Version 2.0. use crowdb_kv_client::{ClientConfig, CrowdbKvClient, CrowdbSysmdClient}; -use crowdb_monitor::{ - verify_chunk_services, BootstrapSession, DeploymentProfile, LogicalBootstrap, MonitorLog, -}; +use crowdb_monitor::{verify_chunk_services, DeploymentProfile}; use std::path::PathBuf; fn root() -> PathBuf { @@ -47,35 +45,21 @@ async fn chunk_services_validate_live_registration_and_logical_topology() { std::fs::write(path, original).unwrap(); } verify_chunk_services(&seed(), &config_profile).await.unwrap(); - verify_logical_topology(&root, &profile).await; + verify_logical_topology().await; super::witness_with_write(true, std::env::var("CROWDB_E2E_PHASE").as_deref() == Ok("write")).await; } -async fn verify_logical_topology(root: &std::path::Path, profile: &DeploymentProfile) { +async fn verify_logical_topology() { let sysmd = CrowdbSysmdClient::new(CrowdbKvClient::new(ClientConfig::new(vec![seed()]))); assert_eq!(sysmd.list_stores().await.unwrap().len(), 1); - assert_eq!(sysmd.list_groups_in_store(0).await.unwrap().len(), 2); + let initial = sysmd.list_groups_in_store(0).await.unwrap(); + assert_eq!(initial.len(), 2); assert_eq!(sysmd.list_replicas_in_group(0, 0).await.unwrap().len(), 1); assert_eq!(sysmd.list_replicas_in_group(0, 1).await.unwrap().len(), 1); - let data = root.join("client-data/logical-bootstrap"); - std::fs::create_dir_all(&data).unwrap(); - let mut session = BootstrapSession::open(&data, b"profile", b"config", &["logical-topology"]).unwrap(); - std::fs::create_dir_all(data.join("log")).unwrap(); - let mut events = MonitorLog::open(&data.join("log"), profile.logs.clone()) - .await - .unwrap(); - let bootstrap = LogicalBootstrap::new(seed()); - bootstrap - .reconcile(&mut session, profile, &mut events) - .await - .unwrap(); sysmd.add_group(0, 99).await.unwrap(); - assert!(bootstrap - .reconcile(&mut session, profile, &mut events) - .await - .is_err()); assert_eq!(sysmd.list_groups_in_store(0).await.unwrap().len(), 3); sysmd.remove_group(0, 99).await.unwrap(); + assert_eq!(sysmd.list_groups_in_store(0).await.unwrap(), initial); } #[tokio::test] diff --git a/e2e-test/tests/deployment/supervision.rs b/e2e-test/tests/deployment/supervision.rs index fb69fe2e2..9a41cd5a0 100644 --- a/e2e-test/tests/deployment/supervision.rs +++ b/e2e-test/tests/deployment/supervision.rs @@ -4,6 +4,12 @@ async fn recover(service: &str, signal: &str) { super::witness_with_write(true, true).await; let observed = crate::common::control::recover_service(service, signal).await; + if signal == "STOP" { + assert_eq!(observed["hung"]["healthy"], false); + assert_eq!(observed["hung"]["pid"], observed["before"]["pid"]); + assert_eq!(observed["hung"]["generation"], observed["before"]["generation"]); + assert_eq!(observed["recovery_trigger"], "explicit-termination"); + } assert_ne!(observed["before"]["pid"], observed["after"]["pid"]); assert!( observed["after"]["generation"].as_u64().unwrap() diff --git a/e2e-test/tests/deployment/supplied_media.rs b/e2e-test/tests/deployment/supplied_media.rs index 151d46459..d6351f69c 100644 --- a/e2e-test/tests/deployment/supplied_media.rs +++ b/e2e-test/tests/deployment/supplied_media.rs @@ -10,6 +10,22 @@ use crowdb_protocol::{common::HwStatus, key::DiskKey, mgmt::node::storage::DiskB use serde_json::Value; use std::time::Duration; +async fn assert_selected_host(kv: &CrowdbKvClient, specification: &Value, host: &str) { + let (registry, _) = crowdb_console_shared::deployment::registry::read(kv) + .await + .unwrap() + .expect("current confirmed node registry"); + assert_eq!(registry.cluster_id, specification["cluster_id"].as_str().unwrap()); + let node = registry + .nodes + .iter() + .find(|node| { + node.node_id == specification["node_id"].as_u64().unwrap() && node.confirmed && !node.cancelled + }) + .expect("confirmed supplied-media host"); + assert_eq!(node.physical_host_id, host); +} + #[tokio::test] #[ignore = "requires an owned Linux host profile and its accepted disposable media"] async fn retained_data_probe() { @@ -19,12 +35,10 @@ async fn retained_data_probe() { let mode = specification["mode"].as_str().unwrap(); assert!(matches!(mode, "write" | "verify")); let host = specification["physical_host_id"].as_str().unwrap(); - assert!(matches!( - host, - "containerd-acceptance-host" | "docker-acceptance-host" - )); let seeds: Vec = serde_json::from_value(specification["seeds"].clone()).unwrap(); - let hardware = HardwareClient::new(CrowdbKvClient::new(ClientConfig::new(seeds.clone()))); + let kv = CrowdbKvClient::new(ClientConfig::new(seeds.clone())); + assert_selected_host(&kv, &specification, host).await; + let hardware = HardwareClient::new(kv); let disks = specification["disks"].as_array().unwrap(); assert_eq!(disks.len(), 2); let unit = 1024 * 1024; diff --git a/e2e-test/tests/deployment/validation.rs b/e2e-test/tests/deployment/validation.rs index 887374065..b8534ddb0 100644 --- a/e2e-test/tests/deployment/validation.rs +++ b/e2e-test/tests/deployment/validation.rs @@ -21,7 +21,7 @@ fn invalid_profile_is_rejected() { } #[test] -#[ignore = "requires the owned invalid-manifest release container"] -fn invalid_manifest_is_rejected() { - rejected("bootstrap manifest cannot be decoded"); +#[ignore = "requires the owned invalid-policy release container"] +fn invalid_node_policy_is_rejected() { + rejected("expected value at line 1 column 1"); } diff --git a/e2e-test/tests/diskio/errors.rs b/e2e-test/tests/diskio/errors.rs index cc5d40be7..269622be6 100644 --- a/e2e-test/tests/diskio/errors.rs +++ b/e2e-test/tests/diskio/errors.rs @@ -97,6 +97,7 @@ async fn assert_missing_creation_time( disk_id: Some(&disk), size: 4, write_create_time_ms: 0, + sync: false, ..FBDiskWriteRequestArgs::default() }, ); diff --git a/e2e-test/tests/diskio/semantic.rs b/e2e-test/tests/diskio/semantic.rs index 69c2fdb0e..caee77ff6 100644 --- a/e2e-test/tests/diskio/semantic.rs +++ b/e2e-test/tests/diskio/semantic.rs @@ -144,6 +144,11 @@ async fn assert_payload_and_status(client: &Arc, disk_id: DiskId) ) .await .expect("durable semantic write"); + assert_eq!(client.status().fsync_operations, 0); + client + .fsync(disk_id, client.normal_options()) + .await + .expect("explicit semantic fsync"); let read = client .read( target, diff --git a/e2e-test/tests/diskio/semantic_views.rs b/e2e-test/tests/diskio/semantic_views.rs index 11dafc6a1..d43ad4909 100644 --- a/e2e-test/tests/diskio/semantic_views.rs +++ b/e2e-test/tests/diskio/semantic_views.rs @@ -27,7 +27,8 @@ pub(super) async fn assert_large_view_range(client: &DiskioClient, target: Segme .await .expect("multiple bounded view frames"); assert_eq!(client.status().write_operations - writes_before, 3); - assert_eq!(client.status().fsync_operations - syncs_before, 1); + // Synchronized frames return durable completion without an extra fsync RPC. + assert_eq!(client.status().fsync_operations - syncs_before, 0); let read = client .read( target, @@ -100,7 +101,7 @@ pub(super) async fn assert_independent_strip_writes_overlap() { .unwrap(); pending.await.unwrap(); assert_eq!(client.status().write_operations, 4); - assert_eq!(client.status().fsync_operations, 2); + assert_eq!(client.status().fsync_operations, 0); for target in targets { let read = client .read( @@ -166,7 +167,7 @@ pub(super) async fn assert_view_frames_are_ordered(target: SegmentTarget, aligne .unwrap(); pending.await.unwrap().unwrap(); assert_eq!(client.status().write_operations, 2); - assert_eq!(client.status().fsync_operations, 1); + assert_eq!(client.status().fsync_operations, 0); let read = client .read( target, diff --git a/e2e-test/tests/failure_logs_test.py b/e2e-test/tests/failure_logs_test.py index 1f1117df3..38097d0d3 100644 --- a/e2e-test/tests/failure_logs_test.py +++ b/e2e-test/tests/failure_logs_test.py @@ -16,6 +16,30 @@ class FailureLogsTest(unittest.TestCase): + def test_transient_warning_only_logs_do_not_become_failure_summaries(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + artifacts = root / "artifacts" + artifacts.mkdir() + data = root / "node1" + logs = data / "log" / "chunk-kv-1" + logs.mkdir(parents=True) + (logs / "server.log").write_text( + "2026-10-16T12:00:00Z WARN worker: retrying error=connection refused\n") + (logs / "service.log").write_text( + "[2026-10-16 12:00:00] [warning] retrying connection\n") + (artifacts / "prepare-inspect.error").write_text( + "Error: No such container: prepare\n") + output = io.StringIO() + with patch.dict("os.environ", {"CROWDB_E2E_PRINT_SERVICE_ERRORS": "1"}), \ + contextlib.redirect_stdout(output): + print_failure(artifacts, [data], "case", RuntimeError("browser failed"), redact) + text = output.getvalue() + self.assertIn("[FAILED CASE] case", text) + self.assertNotIn("FAILURE SUMMARY", text) + self.assertNotIn("retrying", text) + self.assertNotIn("No such container", text) + def test_service_errors_stacks_raw_stderr_and_archives_are_redacted(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/e2e-test/tests/image_identity_test.py b/e2e-test/tests/image_identity_test.py index 926f2ddeb..9552cf89a 100644 --- a/e2e-test/tests/image_identity_test.py +++ b/e2e-test/tests/image_identity_test.py @@ -3,6 +3,7 @@ """Verify image provenance across client commits and dirty server edits.""" import subprocess +import os import tempfile import unittest from pathlib import Path @@ -34,6 +35,45 @@ def commit(self): self.git("-c", "user.name=test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture") + def snapshot(self): + with tempfile.TemporaryDirectory() as directory: + environment = dict(os.environ, GIT_INDEX_FILE=str(Path(directory) / "index")) + for arguments in (("read-tree", "HEAD"), ("add", "--all", "--", ".")): + subprocess.run(["git", "-C", str(self.root), *arguments], env=environment, check=True) + return subprocess.check_output(["git", "-C", str(self.root), "write-tree"], + env=environment, text=True).strip() + + def test_tree_verification_preserves_git_objects_and_real_index_with_dirty_image_inputs(self): + extra = self.root / "app/server/src/extra.rs" + extra.write_text("image contains this untracked server input\n") + tree = self.snapshot() + index = (self.root / ".git/index").read_bytes() + objects = set((self.root / ".git/objects").rglob("*")) + (self.root / "e2e-test/client.py").write_text("new host-only client\n") + (self.root / "e2e-test/new-client.py").write_text("new untracked client\n") + identity = verify_server_sources(self.root, self.revision, tree) + self.assertTrue(identity["server_sources_match"]) + self.assertEqual((self.root / ".git/index").read_bytes(), index) + self.assertEqual(set((self.root / ".git/objects").rglob("*")), objects) + self.assertEqual(self.git("rev-parse", "HEAD"), self.revision) + + def test_tree_comparison_rejects_new_changed_or_missing_server_inputs(self): + extra = self.root / "app/server/src/extra.rs" + extra.write_text("accepted image input\n") + tree = self.snapshot() + extra.write_text("changed image input\n") + with self.assertRaisesRegex(ValueError, "extra.rs"): + verify_server_sources(self.root, self.revision, tree) + extra.write_text("accepted image input\n") + newer = self.root / "app/server/src/newer.rs" + newer.write_text("new server input\n") + with self.assertRaisesRegex(ValueError, "newer.rs"): + verify_server_sources(self.root, self.revision, tree) + newer.unlink() + extra.unlink() + with self.assertRaisesRegex(ValueError, "extra.rs"): + verify_server_sources(self.root, self.revision, tree) + def test_client_and_component_test_changes_reuse_release_image(self): (self.root / "e2e-test/client.py").write_text("new client\n") (self.root / "app/server/tests/api_test.rs").unlink() @@ -69,12 +109,16 @@ def test_sdk_packaging_does_not_exempt_server_inputs(self): verify_server_sources(self.root, revision) def test_host_container_cli_does_not_exempt_shared_deployment_or_runtime_inputs(self): - source = self.root / "app/crowdb-cli/src/commands/container_cluster/runtime.rs" + source = self.root / "app/crowdb-cli/src/commands/cluster/local/docker.rs" source.parent.mkdir(parents=True) source.write_text("host lifecycle\n") + registry = self.root / "app/crowdb-cli/src/commands/kv/server/registry.rs" + registry.parent.mkdir(parents=True) + registry.write_text("host process controls\n") self.commit() revision = self.git("rev-parse", "HEAD") source.write_text("changed host lifecycle\n") + registry.write_text("isolated host process controls\n") self.assertTrue(verify_server_sources(self.root, revision)["server_sources_match"]) shared = self.root / "lib/crowdb-console-shared/src/deployment.rs" shared.parent.mkdir(parents=True) @@ -124,6 +168,19 @@ def test_unified_host_tasks_do_not_exempt_image_build_inputs(self): with self.assertRaisesRegex(ValueError, "pixi.toml"): verify_server_sources(self.root, revision) + def test_dev_example_does_not_exempt_shipped_examples_or_runtime_features(self): + original = self.manifest.read_text() + fixture = ('[[example]]\nname = "measurement"\n' + 'path = "tests/common/measurement.rs"\nrequired-features = ["test-util"]\n') + self.manifest.write_text(original + fixture) + self.assertTrue(verify_server_sources(self.root, self.revision)["server_sources_match"]) + for changed in (fixture.replace('tests/common/', 'examples/'), + fixture.replace('["test-util"]', '["runtime"]'), + fixture.replace('required-features = ["test-util"]\n', '')): + self.manifest.write_text(original + changed) + with self.assertRaisesRegex(ValueError, "Cargo.toml"): + verify_server_sources(self.root, self.revision) + def test_host_benchmark_hook_does_not_exempt_shared_runtime_or_default_features(self): directory = self.root / "lib/crowdb-console-shared" driver = directory / "src/ops/s3_bench.rs" @@ -236,6 +293,9 @@ def test_unpackaged_operator_documentation_reuses_image_but_entrypoint_changes_d (oci / "README.md").write_text("operator instructions\n") (oci / "runtime.py").write_text("host runtime commands\n") (oci / "run-node.py").write_text("host launcher\n") + host_modules = ("host_inputs.py", "host_runtime.py", "host_lifecycle.py", "host_cleanup.py", "host-clean.py") + for name in host_modules: + (oci / name).write_text("host CLI/cleanup only\n") (oci / "Dockerfile").write_text("release image recipe\n") (directory / "README.md").write_text("operator instructions\n") (directory / "dev-container.sh").write_text("host development container helper\n") @@ -247,6 +307,8 @@ def test_unpackaged_operator_documentation_reuses_image_but_entrypoint_changes_d revision = self.git("rev-parse", "HEAD") (oci / "runtime.py").write_text("host CPU shares commands\n") (oci / "run-node.py").write_text("host CPU sharing options\n") + for name in host_modules: + (oci / name).write_text("changed host CLI/cleanup\n") (oci / "README.md").write_text("root containerd acceptance instructions\n") (directory / "README.md").write_text("new service-selection instructions\n") (directory / "dev-container.sh").write_text("changed host cleanup command\n") diff --git a/e2e-test/tests/native_containerd_test.py b/e2e-test/tests/native_containerd_test.py new file mode 100644 index 000000000..dc8c90141 --- /dev/null +++ b/e2e-test/tests/native_containerd_test.py @@ -0,0 +1,99 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Native host provenance, early validation and owned-daemon teardown boundaries.""" + +import contextlib +import io +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import tomllib +import unittest +from unittest.mock import MagicMock, patch + +import run +from oci_native_runtime import NativeRuntime, docker_free +from oci_host_node import HostNode + + +class NativeContainerdTest(unittest.TestCase): + def test_host_cleanup_only_skips_absent_receipts_and_rejects_invalid_ownership(self): + node = HostNode.__new__(HostNode) + node.cluster = 'test' + node.env = {'CROWDB_RUNTIME_ROOT':'/unused'} + with patch('oci_host_node.read_private', return_value=None), patch.object(node, 'record') as record: + node.cleanup() + record.assert_not_called() + for error in ('corrupt receipt', 'foreign owner'): + with patch('oci_host_node.read_private', return_value={}), \ + patch.object(node, 'record', side_effect=ValueError(error)), \ + patch('oci_host_node.clean') as clean: + with self.assertRaisesRegex(ValueError, error): node.cleanup() + clean.assert_not_called() + + def test_native_profile_listing_and_invalid_budgets_have_no_runtime_effects(self): + for options in [('--concurrency','2'), ('--concurrency','1','--memory-mib','2048'), + ('--concurrency','1','--layer','kv')]: + with patch.object(sys,'argv',['run.py','--profile','containerd-host',*options]), \ + patch('native_containerd_profile.docker_free') as preflight, \ + contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): run.main() + preflight.assert_not_called() + output=io.StringIO() + with patch.object(sys,'argv',['run.py','--profile','containerd-host','--concurrency','1','--list']), \ + patch('native_containerd_profile.docker_free') as preflight, contextlib.redirect_stdout(output): + run.main() + preflight.assert_not_called() + self.assertIn('local_protocols_and_production_metadata_recreation',output.getvalue()) + + def test_docker_client_socket_daemon_and_container_host_cannot_claim_native_execution(self): + for condition in ('client','socket','daemon','container'): + with self.subTest(condition=condition), \ + patch('oci_native_runtime.shutil.which',return_value='/docker' if condition=='client' else None), \ + patch('oci_native_runtime.Path.exists',autospec=True, + side_effect=lambda path: str(path)==('/var/run/docker.sock' if condition=='socket' else '/.dockerenv') + if condition in ('socket','container') else False), \ + patch('oci_native_runtime.Path.glob',return_value=[MagicMock(read_text=lambda:'dockerd')] if condition=='daemon' else []): + with self.assertRaises(ValueError): docker_free() + with patch('oci_native_runtime.shutil.which',return_value=None), \ + patch('oci_native_runtime.Path.exists',autospec=True,side_effect=lambda path:str(path)=='/.dockerenv'), \ + patch('oci_native_runtime.Path.glob',return_value=[]): + docker_free(allow_container_host=True) + with self.assertRaisesRegex(ValueError,'substitute'): docker_free() + + def test_owned_daemon_is_closed_after_body_failure_without_touching_other_processes(self): + with tempfile.TemporaryDirectory() as temporary: + root=Path(temporary) + prefix=root/'tools' + (prefix/'bin').mkdir(parents=True) + for name in ('containerd','containerd-shim-runc-v2','nerdctl','runc'): + (prefix/'bin'/name).touch() + process=MagicMock(pid=4242) + process.poll.return_value=None + runtime=MagicMock() + runtime.invoke.return_value='{"ID":"owned","Driver":"native"}' + with patch.dict(os.environ,{'CROWDB_OCI_RUNTIME_PREFIX':str(prefix)}), \ + patch('oci_native_runtime.docker_free'), patch('oci_native_runtime.os.geteuid',return_value=0), \ + patch('oci_native_runtime.Runtime',return_value=runtime), \ + patch('oci_native_runtime.subprocess.run',return_value=MagicMock(stdout='version=4\nimports=["/etc/containerd/conf.d/*.toml"]\ndisabled_plugins=[]\n[plugins."io.containerd.server.v1.grpc"]\naddress="/run/containerd/containerd.sock"\n[plugins."io.containerd.server.v1.ttrpc"]\naddress="/run/containerd/containerd.sock.ttrpc"\n')), \ + patch('oci_native_runtime.subprocess.Popen',return_value=process), \ + patch('oci_native_runtime.os.killpg') as kill: + with self.assertRaisesRegex(RuntimeError,'test body'), NativeRuntime(root/'owned','test'): + raise RuntimeError('test body') + config = tomllib.loads((root/'owned/config.toml').read_text()) + self.assertEqual(config['imports'], []) + self.assertTrue(config['plugins']['io.containerd.server.v1.grpc']['address'].startswith('/tmp/crowdb-ctd-')) + self.assertEqual(config['plugins']['io.containerd.server.v1.ttrpc']['address'], config['plugins']['io.containerd.server.v1.grpc']['address'] + '.ttrpc') + self.assertIn('io.containerd.cri.v1.runtime',config['disabled_plugins']) + kill.assert_called_once_with(4242,signal.SIGTERM) + process.wait.assert_called_once_with(timeout=10) + + def test_host_fixture_imports_cannot_shadow_the_e2e_artifact_publisher(self): + result=subprocess.run([sys.executable,'-c', + 'import native_containerd_profile; import publish; print(publish.__file__)'], + env=dict(os.environ,PYTHONPATH=str(Path(__file__).resolve().parents[1])), + capture_output=True,text=True,check=True,timeout=10) + self.assertEqual(Path(result.stdout.strip()).resolve(),Path(__file__).resolve().parents[1]/'publish.py') diff --git a/e2e-test/tests/oci_reference_test.py b/e2e-test/tests/oci_reference_test.py new file mode 100644 index 000000000..c567a314b --- /dev/null +++ b/e2e-test/tests/oci_reference_test.py @@ -0,0 +1,143 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from fixture import LABEL +import oci_reference_runtime as runtime +from scenarios import case_nodes, case_resources, select_cases + + +class OciReferenceOwnershipTest(unittest.TestCase): + def cluster(self, implementation="docker"): + return SimpleNamespace(implementation=implementation, run_id="owned", image_id="expected", + expected_names=["owned-node1"], node_count=1, + runtime_accepted=False, command=Mock()) + + def test_preexisting_scope_is_rejected_without_adoption_or_effects(self): + cluster = self.cluster() + with patch.object(runtime, "inventory", return_value=[{"Name": "foreign"}]): + with self.assertRaisesRegex(ValueError, "already contains"): + runtime.start(cluster) + self.assertFalse(cluster.runtime_accepted) + cluster.command.assert_not_called() + + def test_unaccepted_start_does_not_authorize_cleanup(self): + cluster = self.cluster() + runtime.close(cluster) + cluster.command.assert_not_called() + + def test_changed_image_is_rejected_before_any_cleanup_effect(self): + cluster = self.cluster() + cluster.runtime_accepted = True + foreign = {"Name": "/owned-node1", "Image": "replaced", + "Config": {"Labels": {LABEL: "owned"}}} + with patch.object(runtime, "inventory", return_value=[foreign]): + with self.assertRaisesRegex(ValueError, "ownership or image"): + runtime.close(cluster) + cluster.command.assert_not_called() + + def test_unrelated_network_attachment_prevents_container_and_network_removal(self): + cluster = self.cluster() + cluster.runtime_accepted = True + container = {"Id": "node-id", "Name": "/owned-node1", "Image": "expected", + "Config": {"Labels": {LABEL: "owned"}}} + network = {"Id": "bridge-id", "Containers": {"foreign-id": {}}} + with patch.object(runtime, "inventory", side_effect=lambda _, kind: { + "container": [container], "network": [network]}[kind]): + with self.assertRaisesRegex(ValueError, "unrelated attachment"): + runtime.close(cluster) + cluster.command.assert_not_called() + + def test_compose_project_label_alone_does_not_authorize_an_unrelated_service(self): + cluster = self.cluster("compose") + container = {"Image": "expected", "Config": {"Labels": { + runtime.PROJECT: "owned", "com.docker.compose.service": "foreign"}}} + with self.assertRaisesRegex(ValueError, "unrelated service"): + runtime.validate_container(cluster, container) + + def test_compose_service_label_does_not_authorize_an_unexpected_container(self): + cluster = self.cluster("compose") + container = {"Name": "/foreign", "Image": "expected", "Config": {"Labels": { + runtime.PROJECT: "owned", "com.docker.compose.service": "node"}}} + with self.assertRaisesRegex(ValueError, "unexpected reference container name"): + runtime.validate_container(cluster, container) + + def test_missing_owned_storage_rejects_recreation_before_container_removal(self): + cluster = self.cluster() + container = {"Id": "node-id", "Name": "/owned-node1", "Image": "expected", + "Config": {"Labels": {LABEL: "owned"}}} + with patch.object(runtime, "inventory", side_effect=lambda _, kind: { + "container": [container], "network": [], "volume": []}[kind]): + with self.assertRaisesRegex(ValueError, "retained volumes"): + runtime.recreate(cluster) + cluster.command.assert_not_called() + + def test_unexpected_owned_volume_rejects_cleanup_before_any_removal(self): + cluster = self.cluster() + cluster.runtime_accepted = True + container = {"Id": "node-id", "Name": "/owned-node1", "Image": "expected", + "Config": {"Labels": {LABEL: "owned"}}} + with patch.object(runtime, "inventory", side_effect=lambda _, kind: { + "container": [container], "network": [], "volume": [{"Name": "unrelated"}]}[kind]): + with self.assertRaisesRegex(ValueError, "unrelated volume"): + runtime.close(cluster) + cluster.command.assert_not_called() + + def test_all_reference_examples_have_registered_owners_and_resource_reservations(self): + cases = select_cases(scope="deployment.reference") + self.assertEqual({case.oci_reference for case in cases}, + {"docker-one", "docker-three", "compose-one", "compose-three"}) + self.assertEqual({case.oci_reference for case in cases if case.oci_editable_ui}, + {"docker-one", "docker-three"}) + self.assertEqual(len(cases), 6) + for case in cases: + count = 1 if case.oci_reference.endswith("one") else 3 + self.assertEqual(case_nodes(case), 3 if case.oci_shortcuts else 4 if case.oci_address_recovery else count) + cpu, memory = case_resources(case, 0.25) + self.assertGreaterEqual(cpu, count) + self.assertGreaterEqual(memory, count * 1536 + 512) + + def test_pending_free_regression_has_one_owned_normal_node_case(self): + cases = select_cases(scope="console.cli.container", + case_name="console::cli_container::pending_free_accounting") + self.assertEqual(len(cases), 1) + self.assertTrue(cases[0].cli_workload) + self.assertEqual(cases[0].oci_reference, "docker-one") + self.assertEqual(case_nodes(cases[0]), 1) + + def test_cold_recovery_reserves_the_later_nonvoter_without_reducing_existing_nodes(self): + case = select_cases(case_name='deployment::reference::direct_docker_three_nodes')[0] + self.assertTrue(case.oci_address_recovery) + self.assertEqual(case_nodes(case), 4) + self.assertEqual(case_resources(case, .25), (3.75, 3 * 1536 + 768 + 512)) + + def test_recreated_names_reuse_exact_retained_volumes_on_the_accepted_new_network(self): + cluster = self.cluster() + cluster.expected_names = ['owned-node1-cold'] + cluster.reference_volumes = ['owned-node1-data'] + cluster.reference_network = 'owned-recovered' + cluster.port_base, cluster.image, cluster.containers = 20000, 'expected', [] + runtime.create_docker_nodes(cluster, [0]) + self.assertEqual(runtime.expected_volumes(cluster), {'owned-node1-data'}) + create = cluster.command.call_args.args + self.assertIn('type=volume,source=owned-node1-data,target=/opt/crowdb/data', create) + self.assertEqual(create[create.index('--network') + 1], 'owned-recovered') + self.assertNotIn('owned-node1-cold-data', create) + + def test_cold_network_scope_still_rejects_foreign_names_and_attachments(self): + cluster = self.cluster() + cluster.reference_networks = {'owned', 'owned-recovered'} + for name, peers in [('foreign', {}), ('owned-recovered', {'foreign-id':{}})]: + with patch.object(runtime, 'inventory', return_value=[{'Name':name,'Containers':peers}]): + with self.assertRaises(ValueError): + runtime.validate_networks(cluster, [{'Id':'owned-id'}]) + cluster.command.assert_not_called() + + def test_shortcut_acceptance_reserves_the_maximum_live_topology(self): + case = select_cases(case_name='deployment::reference::direct_docker_one_node')[0] + self.assertTrue(case.oci_shortcuts) + self.assertEqual(case_nodes(case), 3) + self.assertEqual(case_resources(case, .25), (3.5, 3 * 1536 + 512)) diff --git a/e2e-test/tests/oci_workloads_test.py b/e2e-test/tests/oci_workloads_test.py new file mode 100644 index 000000000..8a022f378 --- /dev/null +++ b/e2e-test/tests/oci_workloads_test.py @@ -0,0 +1,55 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import copy +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +from oci_reference_workloads import quiesce_chunk_allocators + + +class AllocatorWorkloadPreconditionTest(unittest.TestCase): + def services(self): + return [{"id": kind + "-1", "service_type": kind, "pid": index + 1} + for index, kind in enumerate(("paxos-kv", "diskdb", "diskio", "chunkdb", + "chunk-kv", "access-server"))] + + def test_stops_producers_in_dependency_order_and_preserves_allocator_processes(self): + services, actions = self.services(), [] + def request(origin, path, body=None): + self.assertEqual(origin, "http://owned:9090") + if path == "/api/servers": + return copy.deepcopy(services) + service_id = path.split("/")[-2] + self.assertEqual(path, f"/api/services/{service_id}/stop") + self.assertEqual(body, {}) + actions.append(service_id) + next(item for item in services if item["id"] == service_id)["pid"] = None + with tempfile.TemporaryDirectory() as directory: + cluster = SimpleNamespace(node_count=1, artifacts=Path(directory), + endpoint=lambda _: "http://owned:9090") + with patch("oci_reference_workloads.request", side_effect=request): + quiesce_chunk_allocators(cluster) + proof = json.loads((cluster.artifacts / "allocator-workload-precondition.json").read_text()) + self.assertEqual(actions, ["access-server-1", "chunk-kv-1", "chunkdb-1"]) + self.assertEqual(proof["stopped_producers"], actions) + self.assertEqual(proof["preserved_services"], [{"id": kind + "-1", "pid": index + 1} + for index, kind in enumerate(("paxos-kv", "diskdb", "diskio"))]) + + def test_missing_producer_rejects_before_stopping_any_service(self): + cluster = SimpleNamespace(node_count=1, endpoint=lambda _: "http://owned:9090") + services = [item for item in self.services() if item["service_type"] != "chunkdb"] + with patch("oci_reference_workloads.request", return_value=services) as request: + with self.assertRaisesRegex(AssertionError, "one owned chunkdb"): + quiesce_chunk_allocators(cluster) + request.assert_called_once_with("http://owned:9090", "/api/servers") + + def test_other_topologies_reject_before_contacting_the_target(self): + with patch("oci_reference_workloads.request", new=Mock()) as request: + with self.assertRaisesRegex(ValueError, "owned single node"): + quiesce_chunk_allocators(SimpleNamespace(node_count=3)) + request.assert_not_called() diff --git a/e2e-test/tests/regression_fixture_test.py b/e2e-test/tests/regression_fixture_test.py new file mode 100644 index 000000000..934a3ea1f --- /dev/null +++ b/e2e-test/tests/regression_fixture_test.py @@ -0,0 +1,225 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Measurement ownership guards; these tests do not certify OCI deployments.""" + +import argparse +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import types +import unittest +from unittest.mock import patch + +SPEC = importlib.util.spec_from_file_location( + "measurement_fixture", Path(__file__).resolve().parents[2] / "tools/benchmark/fixture.py") +fixture = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(fixture) + + +class RegressionFixtureTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.state = self.root / "console.toml" + self.descriptor = self.state.with_suffix(".fixture.json") + self.value = {"pid": os.getpid(), "start": fixture.process_start(os.getpid()), + "endpoint": "http://127.0.0.1:1", "token": "owner", "root": str(self.root)} + + def write_descriptor(self): + self.descriptor.write_text(json.dumps(self.value)) + self.descriptor.chmod(0o600) + + def test_current_process_identity_is_required(self): + self.write_descriptor() + self.assertEqual(fixture.descriptor(self.state), self.value) + self.value["start"] = "foreign-start" + self.write_descriptor() + with self.assertRaises(fixture.FixtureStopped): + fixture.descriptor(self.state) + + def test_descriptor_rejects_public_file_symlink_and_oversized_file(self): + self.write_descriptor() + self.descriptor.chmod(0o644) + with self.assertRaisesRegex(ValueError, "private regular"): + fixture.descriptor(self.state) + self.descriptor.unlink() + self.descriptor.symlink_to(self.state) + with self.assertRaisesRegex(ValueError, "private regular"): + fixture.descriptor(self.state) + self.descriptor.unlink() + self.write_descriptor() + self.descriptor.write_text(" " * 65537) + with self.assertRaisesRegex(ValueError, "bounded private"): + fixture.descriptor(self.state) + + def test_dead_owner_cannot_spawn_a_fixture(self): + args = argparse.Namespace(state=self.state, owner_pid=None, arguments=["--kind", "kv"]) + with patch.object(fixture.subprocess, "Popen") as spawn: + with self.assertRaisesRegex(ValueError, "live measurement owner"): + fixture.setup(args) + spawn.assert_not_called() + + def test_failed_client_drops_stale_report_and_does_not_inherit_cluster_target(self): + self.write_descriptor() + report = self.root / "client-runtime/artifacts/cli/bench-kv/report.json" + report.parent.mkdir(parents=True) + report.write_text('{"total_ops": 999}') + args = argparse.Namespace(state=self.state, action="client", arguments=["bench", "kv", "bad"]) + with patch.dict(os.environ, {"CROWDB_CLUSTER": "foreign"}), \ + patch.object(fixture.subprocess, "call", return_value=2) as call: + self.assertEqual(fixture.run(args), 2) + self.assertNotIn("CROWDB_CLUSTER", call.call_args.kwargs["env"]) + self.assertEqual(call.call_args.args[0][1:3], + ["--management-endpoint", self.value["endpoint"]]) + self.assertFalse(report.exists()) + self.assertFalse((self.root / "measurements").exists(), "failed parsing retained stale logs") + + def test_invalid_benchmark_family_cannot_delete_arbitrary_directories(self): + self.write_descriptor() + retained = self.root / "client-runtime/artifacts/cli/keep" + retained.mkdir(parents=True) + witness = retained / "witness" + witness.write_text("preserved") + args = argparse.Namespace(state=self.state, action="client", + arguments=["bench", "kv/../keep"]) + with patch.object(fixture.subprocess, "call", return_value=2): + self.assertEqual(fixture.run(args), 2) + self.assertEqual(witness.read_text(), "preserved") + + def test_stop_with_stale_identity_never_sends_a_control_request(self): + self.value["start"] = "foreign-start" + self.write_descriptor() + args = argparse.Namespace(state=self.state, action="stop") + with patch.object(fixture.HTTP, "open") as request: + self.assertEqual(fixture.run(args), 0) + request.assert_not_called() + + def check_service_logs(self): + script = ("source \"$1\"\n" + "regression_fixture_log_root() { printf '%s\\n' \"$TEST_FIXTURE_ROOT\"; }\n" + "regression_require_no_service_errors\n") + common = Path(__file__).resolve().parents[2] / "tools/benchmark/bench-regression-common.sh" + return subprocess.run(["bash", "-c", script, "log-validation", str(common)], + env=dict(os.environ, REGRESSION_LOG_ROOT=str(self.root), TEST_FIXTURE_ROOT=str(self.root)), + capture_output=True, text=True, timeout=5) + + def test_service_log_scan_excludes_binary_error_strings(self): + services = self.root / "services" + services.mkdir() + (services / "server").write_bytes(b"\0DiskNotExist\0") + (services / "server.log").write_text("normal lifecycle\n") + result = self.check_service_logs() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_service_log_errors_are_printed_and_fail_validation(self): + services = self.root / "services" + services.mkdir() + (services / "server.log").write_text("DiskNotExist: lost registered disk\n") + result = self.check_service_logs() + self.assertNotEqual(result.returncode, 0) + self.assertIn("DiskNotExist: lost registered disk", result.stdout) + + def test_service_log_scan_failure_is_not_treated_as_no_errors(self): + result = self.check_service_logs() + self.assertNotEqual(result.returncode, 0) + self.assertIn("grep status 2", result.stderr) + + +# Exercise the exact inline validator shipped with the benchmark script. +STREAM_SCRIPT = Path(__file__).resolve().parents[2] / "tools/benchmark/bench-chunk-stream-regression.sh" +STREAM_PROGRAM = STREAM_SCRIPT.read_text().split("<<'PY_STREAM_REGRESSION'\n", 1)[1].split("\nPY_STREAM_REGRESSION\n", 1)[0] +regression = types.ModuleType("stream_regression") +exec(compile(STREAM_PROGRAM, str(STREAM_SCRIPT), "exec"), regression.__dict__) + + +def stream_transcript(): + result = dict(workload="append", operations=4000, bytes=16384000, errors=0, seconds=10, + ops_s=400, mib_s=1.56, avg_us=2000, p50_us=1900, p99_us=3000, + rss_start_kib=1024, rss_end_kib=2048, rss_peak_kib=3072, + max_queue_requests=1, max_queue_bytes=4096, batches=4000, batch_requests=4000, + rollovers=0, metadata_publications=1, cache_hits=0, cache_misses=0, + physical_reads=0, reclaimed_bytes=0, watchdogs=0) + phases = {name: 0 for name in regression.REQUIRED_PHASES} + phases.update({"stream.metadata.publish.lh": 1, "stream.append.batch.lh": 4000, + "stream.append.mirror.lh": 4000, "chunkio.diskio.write.e2e.lh": 8000}) + return ("bench-runtime: pid=123 allowed_cpus=0-3 tokio_workers=32\n" + + "chunk-stream: " + " ".join(f"{key}={value}" for key, value in result.items()) + "\n" + + "\n".join(f"stream-phase: name={name} count={count} avg_us=0 p50_us=0 p99_us=0 max_us=0" + for name, count in sorted(phases.items()))) + + +class StreamRegressionTest(unittest.TestCase): + def analyze(self, text, **kwargs): + return regression.analyze(text, "append_4k_1t", 10, **kwargs) + + def test_complete_workload_passes_and_retains_all_distributions(self): + value = self.analyze(stream_transcript()) + self.assertTrue(value["passed"], value["checks"]) + self.assertEqual(value["phases"]["stream.metadata.publish.lh"]["count"], 1) + self.assertIn("stream.chunk.seal_drain.lh", value["phases"]) + + def test_missing_duplicate_or_nonfinite_results_cannot_pass(self): + for text in ("", stream_transcript() + "\n" + stream_transcript(), + stream_transcript().replace(" p99_us=3000", ""), + stream_transcript().replace("ops_s=400", "ops_s=nan"), + stream_transcript().replace("errors=0", "errors=-1")): + with self.subTest(text=text[:100]), self.assertRaises(ValueError): + self.analyze(text) + + def test_missing_phase_is_not_interpreted_as_zero_checkpoint_work(self): + text = "\n".join(line for line in stream_transcript().splitlines() + if "stream.checkpoint.cursor.lh" not in line) + with self.assertRaisesRegex(ValueError, "missing phases"): + self.analyze(text) + + def test_publication_cursor_and_required_mirror_guards_fail(self): + for before, after, check in ( + ("metadata_publications=1", "metadata_publications=0", "publications"), + ("name=stream.checkpoint.cursor.lh count=0", "name=stream.checkpoint.cursor.lh count=1", + "no_cursor_checkpoint"), + ("name=chunkio.diskio.write.e2e.lh count=8000", + "name=chunkio.diskio.write.e2e.lh count=4000", "mirror_writes")): + with self.subTest(check=check): + value = self.analyze(stream_transcript().replace(before, after)) + self.assertFalse(value["passed"]) + self.assertFalse(value["checks"][check]) + + def test_short_or_failed_runs_and_original_performance_bounds_fail(self): + for before, after, check in (("seconds=10", "seconds=0.1", "full_duration"), + ("ops_s=400", "ops_s=249", "throughput"), + ("p99_us=3000", "p99_us=10001", "p99"), + ("rss_peak_kib=3072", "rss_peak_kib=300000", "rss")): + with self.subTest(check=check): + value = self.analyze(stream_transcript().replace(before, after)) + self.assertFalse(value["passed"]) + self.assertFalse(value["checks"][check]) + self.assertFalse(self.analyze(stream_transcript(), status=124)["passed"]) + + def test_trim_may_checkpoint_its_retained_boundary(self): + text = stream_transcript().replace("workload=append", "workload=gc") + text = text.replace("mib_s=1.56", "mib_s=2000").replace("reclaimed_bytes=0", "reclaimed_bytes=1048576") + text = text.replace("name=stream.checkpoint.cursor.lh count=0", "name=stream.checkpoint.cursor.lh count=1") + value = regression.analyze(text, "gc_576m", 10) + self.assertTrue(value["passed"], value["checks"]) + + def test_missing_case_receipt_cannot_disappear_from_summary(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "conditions.json").write_text(json.dumps(dict(cases=["append_4k_1t"]))) + self.assertEqual(regression.summarize(argparse.Namespace(root=root, results=root / "results.tsv")), 1) + self.assertFalse(json.loads((root / "results.json").read_text())["complete"]) + + def test_failed_cleanup_cannot_pass_a_successful_workload(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + log = root / "workload.log" + log.write_text(stream_transcript()) + (root / "cpu.json").write_text("{}") + (root / "console.fixture-stopped.json").write_text('{"ok":false,"error":"stale owner"}') + args = argparse.Namespace(log=log, label="append_4k_1t", duration=10, status=0) + self.assertEqual(regression.record(args), 1) + self.assertFalse(json.loads((root / "result.json").read_text())["checks"]["cleanup"]) diff --git a/e2e-test/tests/report_test.py b/e2e-test/tests/report_test.py index 498ad500e..dfe0f3043 100644 --- a/e2e-test/tests/report_test.py +++ b/e2e-test/tests/report_test.py @@ -11,6 +11,26 @@ class ReportTest(unittest.TestCase): + def test_expected_nonzero_exit_is_passed_only_when_fixture_lifecycle_also_passed(self): + for fixture_status, actual, expected, outcome in ( + ("pass", 1, 1, "pass"), ("pass", 2, 1, "failed"), ("failed", 1, 1, "failed"), + ): + with self.subTest(fixture=fixture_status, actual=actual, expected=expected): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + case = root / "crowdb-e2e-owned" + case.mkdir() + for name, value in { + "case.json": {"case": "waiting", "scope": "deployment.reference"}, + "client-check.json": {"phase": "check", "seconds": 1.0, + "exit_code": actual, "expected_exit_code": expected}, + "timing.json": {"status": fixture_status, "total": 2.0}, + }.items(): + (case / name).write_text(json.dumps(value)) + write_report(root, {"status": fixture_status}) + results = json.loads((root / "results.json").read_text()) + self.assertEqual(results["cases"][0]["status"], outcome) + def test_failed_cleanup_is_a_failed_case_even_after_client_passed(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/e2e-test/tests/resource_dispatch_test.py b/e2e-test/tests/resource_dispatch_test.py new file mode 100644 index 000000000..2513ccd71 --- /dev/null +++ b/e2e-test/tests/resource_dispatch_test.py @@ -0,0 +1,96 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import concurrent.futures +import threading +import unittest + +from resource_dispatch import batch_slots, run_with_resources + + +class ResourceDispatchTest(unittest.TestCase): + def test_large_case_is_exclusive_but_small_cases_share_the_budget(self): + cases = ("large", "left", "right", "last") + costs = {case: (3.75, 5888) if case in ("large", "last") else (1.5, 2560) + for case in cases} + started = {case: threading.Event() for case in cases} + finished = {case: threading.Event() for case in cases} + release = threading.Event() + + def execute(case): + started[case].set() + if case == "large": + self.assertTrue(release.wait(5)) + elif case in ("left", "right"): + self.assertTrue(finished["large"].is_set()) + self.assertTrue(started["right" if case == "left" else "left"].wait(5)) + else: + self.assertTrue(all(finished[peer].is_set() for peer in cases[:-1])) + finished[case].set() + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as owner: + result = owner.submit(run_with_resources, cases, 2, execute, costs.__getitem__, + (4, 8192), threading.Event()) + try: + self.assertTrue(started["large"].wait(5)) + self.assertFalse(started["left"].is_set()) + self.assertFalse(started["right"].is_set()) + finally: + release.set() + self.assertEqual(result.result(timeout=10), []) + self.assertTrue(all(event.is_set() for event in finished.values())) + + def test_memory_budget_prevents_parallel_dispatch_even_with_idle_cpu(self): + completed = [] + + def execute(case): + self.assertEqual(completed, list(range(case))) + completed.append(case) + + self.assertEqual(run_with_resources(range(3), 3, execute, lambda _: (1, 4096), + (4, 6144), threading.Event()), []) + self.assertEqual(completed, [0, 1, 2]) + + def test_completion_refills_a_slot_without_waiting_for_an_unrelated_peer(self): + first = threading.Event() + next_case = threading.Event() + + def execute(case): + if case == 0: + first.set() + self.assertTrue(next_case.wait(5)) + elif case == 1: + self.assertTrue(first.wait(5)) + else: + next_case.set() + + self.assertEqual(run_with_resources(range(3), 2, execute, lambda _: (1.5, 2560), + (4, 8192), threading.Event()), []) + + def test_failure_drains_owned_peer_and_does_not_dispatch_later_cases(self): + stopped = threading.Event() + ready = threading.Barrier(2, timeout=5) + finished = threading.Event() + + def execute(case): + self.assertLess(case, 2) + ready.wait() + if case == 0: + stopped.set() + raise ValueError("original failure") + self.assertTrue(stopped.wait(5)) + finished.set() + + errors = run_with_resources(range(3), 2, execute, lambda _: (1, 1024), (4, 8192), stopped) + self.assertEqual([str(error) for error in errors], ["original failure"]) + self.assertTrue(finished.is_set()) + + def test_oversized_case_is_rejected_before_any_fixture_starts(self): + with self.assertRaisesRegex(ValueError, "budget"): + run_with_resources([0], 2, lambda _: self.fail("allocated oversized fixture"), + lambda _: (1.5, 7168), (4, 6144), threading.Event()) + + def test_browser_batch_stays_within_both_budgets(self): + costs = {"small": (1.5, 2560), "large": (3.5, 5120)} + self.assertEqual(batch_slots(["small", "large"], 2, costs.__getitem__, (4, 8192)), 1) + self.assertEqual(batch_slots(["small"] * 2, 2, costs.__getitem__, (4, 8192)), 2) diff --git a/e2e-test/tests/runner_progression_test.py b/e2e-test/tests/runner_progression_test.py new file mode 100644 index 000000000..2382e8540 --- /dev/null +++ b/e2e-test/tests/runner_progression_test.py @@ -0,0 +1,79 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +from scenarios import Case, run_cases + + +class RunnerProgressionTest(unittest.TestCase): + def run_selection(self, fail_fast, stable=False, slots=1): + started = [] + with tempfile.TemporaryDirectory() as temporary: + class Fixture: + def __init__(self, case): + self.case = case + self.artifacts = Path(temporary) / case.name + self.artifacts.mkdir() + + def __enter__(self): + return self + + def __exit__(self, *_): + pass + + def client(self, *_): + started.append(self.case.name) + if self.case.name == 'first': + raise ValueError('original failure') + + cases = [Case('test', name, native_client=stable) for name in ('first', 'next', 'last')] + with patch('scenarios.select_cases', return_value=cases), \ + patch('scenarios.allocate_cluster', side_effect=lambda case, *_: Fixture(case)): + with self.assertRaisesRegex(ValueError, 'original failure'): + run_cases('image', temporary, 'bundle', slots, fail_fast=fail_fast) + return started + + def test_first_failure_stops_queued_cases_without_interrupting_executor(self): + self.assertEqual(self.run_selection(True), ['first']) + + def test_default_collects_all_independent_cases(self): + self.assertEqual(self.run_selection(False), ['first', 'next', 'last']) + + def test_fail_fast_finishes_already_owned_browser_peers(self): + self.assertCountEqual(self.run_selection(True, stable=True, slots=2), ['first', 'next']) + + def test_runner_uses_each_case_cost_and_keeps_browser_networks_separate(self): + peers = threading.Barrier(2, timeout=5) + finished = set() + with tempfile.TemporaryDirectory() as temporary: + cases = [Case('test', name) for name in ('left', 'right')] + cases.append(Case('deployment.browser', 'browser')) + + class Fixture: + def __init__(self, case): + self.name = case.name + self.artifacts = Path(temporary) / self.name + self.artifacts.mkdir() + + def __enter__(self): + return self + + def __exit__(self, *_): + finished.add(self.name) + + def client(self, *_): + peers.wait() + + def browser(self): + self_owner.assertEqual(finished, {'left', 'right'}) + + self_owner = self + with patch('scenarios.select_cases', return_value=cases), \ + patch('scenarios.allocate_cluster', side_effect=lambda case, *_: Fixture(case)): + run_cases('image', temporary, 'bundle', 2, resource_budget=(4, 4096)) + self.assertEqual(finished, {'left', 'right', 'browser'}) diff --git a/e2e-test/tests/s3_cli_diagnostics_test.py b/e2e-test/tests/s3_cli_diagnostics_test.py new file mode 100644 index 000000000..c3683990d --- /dev/null +++ b/e2e-test/tests/s3_cli_diagnostics_test.py @@ -0,0 +1,27 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import importlib.util +import subprocess +import unittest +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('s3_cli_recipes', + Path(__file__).resolve().parents[1] / 'clients/s3/clients.py') +recipes = importlib.util.module_from_spec(spec) +spec.loader.exec_module(recipes) + + +class S3CliDiagnosticsTest(unittest.TestCase): + def test_failed_cli_keeps_both_outputs_and_command_without_credentials(self): + result = subprocess.CompletedProcess([], 1, b'listing stdout', b'failure secret-value') + with patch.object(recipes.subprocess, 'run', return_value=result): + with self.assertRaises(AssertionError) as captured: + recipes.run('aws', ['s3', 'ls', 's3://bucket/prefix?X-Amz-Signature=signed'], + {'AWS_SECRET_ACCESS_KEY': 'secret-value'}) + message = str(captured.exception) + for expected in ['listing stdout', 'failure ', "'s3', 'ls'"]: + self.assertIn(expected, message) + self.assertNotIn('secret-value', message) + self.assertNotIn('Signature=signed', message) diff --git a/e2e-test/tests/s3_http_exchange_test.py b/e2e-test/tests/s3_http_exchange_test.py new file mode 100644 index 000000000..d89784a13 --- /dev/null +++ b/e2e-test/tests/s3_http_exchange_test.py @@ -0,0 +1,37 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. + +import importlib.util +import unittest +from pathlib import Path +from unittest.mock import Mock + +spec = importlib.util.spec_from_file_location('s3_http_exchange', + Path(__file__).resolve().parents[1] / 'clients/s3/http_exchange.py') +exchange = importlib.util.module_from_spec(spec) +spec.loader.exec_module(exchange) + + +class S3HttpExchangeTest(unittest.TestCase): + def test_early_rejection_is_read_after_peer_stops_accepting_body(self): + for failure in [BrokenPipeError('body rejected'), ConnectionResetError('body rejected')]: + connection = Mock() + connection.request.side_effect = failure + connection.getresponse.return_value.status = 403 + response = exchange.request_response(connection, 'GET', '/key', b'body', {}) + self.assertEqual(response.status, 403) + self.assertIs(response, connection.getresponse.return_value) + + def test_incomplete_send_cannot_be_reported_as_success(self): + connection = Mock() + connection.request.side_effect = BrokenPipeError('incomplete body') + connection.getresponse.return_value.status = 200 + with self.assertRaisesRegex(BrokenPipeError, 'incomplete body'): + exchange.request_response(connection, 'PUT', '/key', b'body', {}) + + def test_disconnect_without_an_http_response_still_fails(self): + connection = Mock() + connection.request.side_effect = BrokenPipeError('incomplete body') + connection.getresponse.side_effect = ConnectionResetError('no response') + with self.assertRaisesRegex(ConnectionResetError, 'no response'): + exchange.request_response(connection, 'PUT', '/key', b'body', {}) diff --git a/e2e-test/tests/selection_test.py b/e2e-test/tests/selection_test.py index c0c2f6b12..c761ffcde 100644 --- a/e2e-test/tests/selection_test.py +++ b/e2e-test/tests/selection_test.py @@ -50,7 +50,7 @@ def test_missing_case_wrong_scope_and_single_case_isolation_fail_before_allocati def test_console_operations_combines_host_client_scopes(self): cases = json.loads(self.invoke("--list", "--scope", "console.operations")) - self.assertEqual(len(cases), 159) + self.assertEqual(len(cases), 153) self.assertEqual(sum(case["scope"] == "console.managed" for case in cases), 2) exact = json.loads(self.invoke("--list", "--scope", "console.operations", "--case", cases[0]["case"])) self.assertEqual(exact, [cases[0]]) @@ -76,7 +76,7 @@ def test_partition_cli_covers_complete_domains_and_rejects_narrowing(self): def test_cli_container_replay_reserves_three_complete_nodes(self): from scenarios import CONSOLE_OPERATION_CASES, case_nodes, case_resources cases = [case for case in CONSOLE_OPERATION_CASES if case.cli_replay] - self.assertEqual({case.cli_replay for case in cases}, {"bound", "storage"}) + self.assertEqual({case.cli_replay for case in cases}, {"bound", "storage", "terminal-writer", "attachment", "observation", "hardware", "logical", "data"}) for case in cases: self.assertEqual(case_nodes(case), 3) self.assertEqual(case_resources(case, 0.25), (1.5, 6656)) @@ -108,12 +108,14 @@ def test_layers_select_only_their_domains_and_count_dependencies(self): ["dataset.http", "dataset.http", "dataset.http", "dataset.sdk", "dataset.sdk"]) self.assertTrue(all(case["nodes"] == 13 for case in dataset)) console = json.loads(self.invoke("--list", "--layer", "console")) - self.assertEqual(len(console), 159) + self.assertEqual(len(console), 153) wal_nodes = {"remove_leader_from_five_node_group_elects_new_leader": 6, "remove_unreachable_leader_retains_group0_membership": 6, "replica_add_remove_wires_peers_bidirectionally": 3, "restart_3node_1group": 4, "restart_5node_1group": 6, "restart_5node_2group": 6, "restart_6node_2group_overlap": 7} for case in console: - if case["scope"] == "console.native.browser.six-node": + if case["case"] == "console::cli_container::pending_free_accounting": + self.assertEqual(case["nodes"], 1) + elif case["scope"] == "console.native.browser.six-node": self.assertEqual(case["nodes"], 6) elif case["case"].endswith("independent_subsets"): self.assertEqual(case["nodes"], 5) diff --git a/e2e-test/tests/special_profiles_test.py b/e2e-test/tests/special_profiles_test.py index 466a85d10..38bdcb07b 100644 --- a/e2e-test/tests/special_profiles_test.py +++ b/e2e-test/tests/special_profiles_test.py @@ -6,46 +6,46 @@ import json from pathlib import Path import sys +import tempfile import unittest from unittest.mock import patch import run -from special_profiles import cleanup_owned_host, cleanup_owned_media +from special_profiles import cleanup_owned_hosts class HostProfileTest(unittest.TestCase): - def test_media_helper_cleanup_checks_all_three_ownership_boundaries(self): - for field in ("image", "owner", "mount", None): - info = {"Image": "foreign" if field == "image" else "image", - "Config": {"Labels": {"org.crowdb.e2e.run": "foreign" if field == "owner" else "owned"}}, - "Mounts": [{"Destination": "/owned", "Source": "/foreign" if field == "mount" else "/owned"}]} - with self.subTest(field=field), patch("special_profiles.docker", side_effect=["owned-media", json.dumps([info]), ""]) as docker: - if field: - with self.assertRaisesRegex(ValueError, "ownership"): - cleanup_owned_media("owned", Path("/owned"), "image") - self.assertEqual(docker.call_count, 2) - else: - cleanup_owned_media("owned", Path("/owned"), "image") - self.assertEqual(docker.call_args.args, ("rm", "-f", "owned-media")) - def test_selection_and_resource_validation_precede_docker(self): - for args in (("--concurrency","2"),("--concurrency","1","--cpus","1"), - ("--concurrency","1","--memory-mib","1024"), - ("--concurrency","1","--layer","kv")): - with patch.object(sys,"argv",["run.py","--profile","host",*args]), \ - patch("special_profiles.docker") as docker, contextlib.redirect_stderr(io.StringIO()): + for args in (('--concurrency','2'), ('--concurrency','1','--cpus','1'), + ('--concurrency','1','--memory-mib','2048'), + ('--concurrency','1','--layer','kv')): + with patch.object(sys,'argv',['run.py','--profile','host',*args]), \ + patch('special_profiles.docker') as docker, contextlib.redirect_stderr(io.StringIO()): with self.assertRaises(SystemExit): run.main() docker.assert_not_called() output = io.StringIO() - with patch.object(sys,"argv",["run.py","--profile","host","--concurrency","1","--list"]), \ - patch("special_profiles.docker") as docker, contextlib.redirect_stdout(output): + with patch.object(sys,'argv',['run.py','--profile','host','--concurrency','1','--list']), \ + patch('special_profiles.docker') as docker, contextlib.redirect_stdout(output): run.main() docker.assert_not_called() - self.assertEqual(json.loads(output.getvalue())[0]["scope"],"deployment.host") + self.assertEqual(json.loads(output.getvalue())[0]['scope'],'deployment.host') + + def test_cleanup_rejects_invalid_ownership_and_still_cleans_the_other_known_case(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + for case in ('protocols','supplied'): + path = root / case / 'private/runtime/persistent/oci-hosts/owned.json' + path.parent.mkdir(parents=True) + path.write_text('{}') + for failure in ('changed image', 'foreign owner', 'different daemon', 'different mount'): + with patch('special_profiles.read_private', return_value={}), \ + patch('special_profiles.selected', side_effect=[ValueError(failure), {'verified':'receipt'}]), \ + patch('special_profiles.clean') as clean: + with self.assertRaisesRegex(RuntimeError, failure): cleanup_owned_hosts(root) + clean.assert_called_once_with({'verified':'receipt'}) - def test_interruption_cleanup_refuses_foreign_data_or_changed_image(self): - for image, source in (("foreign-image","/owned/node"),("image","/foreign/node")): - info = {"Image":image,"Mounts":[{"Destination":"/opt/crowdb/data","Source":source}]} - with patch("special_profiles.docker",side_effect=["owned",json.dumps([info])]) as docker: - with self.assertRaisesRegex(ValueError,"ownership"): - cleanup_owned_host("owned",Path("/owned"),"image") - self.assertEqual(docker.call_count,2) + def test_cleanup_does_not_search_the_runtime_when_no_known_receipt_exists(self): + with tempfile.TemporaryDirectory() as temporary, \ + patch('special_profiles.selected') as selected, patch('special_profiles.clean') as clean: + cleanup_owned_hosts(Path(temporary)) + selected.assert_not_called() + clean.assert_not_called() diff --git a/e2e-test/tests/storage_readiness_test.py b/e2e-test/tests/storage_readiness_test.py index 778699af0..5d2d3f753 100644 --- a/e2e-test/tests/storage_readiness_test.py +++ b/e2e-test/tests/storage_readiness_test.py @@ -4,9 +4,14 @@ import tempfile import unittest -from unittest.mock import patch +from unittest.mock import Mock, call, patch from chunkdb_fixture import ChunkdbCluster, ChunkdbStorageCluster +from chunk_kv_fixture import ChunkKvCluster +from access_iceberg_fixture import AccessIcebergCluster +from access_dataset_fixture import AccessDatasetCluster +from s3_node_fixture import S3NodeCluster +from s3_recovery import run_recovery class StorageReadinessTest(unittest.TestCase): @@ -48,3 +53,72 @@ def metadata_restart(): cluster.crash_restart() self.assertEqual(starts, cluster.servers[5:11]) ready.assert_called_once_with("bundle", "storage-ready-recovered", "chunkdb::io::ready") + + def test_three_node_consumers_start_after_storage_ownership_is_ready(self): + with tempfile.TemporaryDirectory() as directory: + cluster = S3NodeCluster("image", directory, "bundle") + cluster.addresses = [f"10.20.30.{index + 2}" for index in range(14)] + events = [] + cluster.client = lambda _bundle, phase, _case: events.append(phase) + cluster.diskdb = lambda node: events.append(("diskdb", node)) + cluster.diskio = lambda node: events.append(("diskio", node)) + cluster.chunkdb = lambda node: events.append(("chunkdb", node)) + cluster.chunk_kv = lambda: events.append("chunk-kv") + cluster.access = lambda: events.append("access") + with patch("s3_node_fixture.Cluster.start"): + cluster.start() + self.assertEqual(events, ["seed", + *(("diskdb", node) for node in range(3)), + *(("diskio", node) for node in range(3)), + "storage-ready", + *(("chunkdb", node) for node in range(3)), + "chunk-kv", "access"]) + + def test_three_node_service_indexes_survive_dependency_ordered_launch(self): + from allocation import node_data + with tempfile.TemporaryDirectory() as directory: + cluster = S3NodeCluster("image", directory, "bundle") + base = "10.20.30." + def start_metadata(): + cluster.addresses = [base + str(index + 2) for index in range(3)] + for index in range(3): + node_data(cluster, index) + cluster.servers.append(f"{cluster.run_id}-node{index}") + def launch(_cluster, index, data, arguments, address, **_kwargs): + cluster.servers.append(f"{cluster.run_id}-node{index}") + return address + def ready(_bundle, phase, _case): + if phase == "storage-ready": + self.assertEqual(len(cluster.addresses), 14) + self.assertEqual(cluster.addresses[9:12], [base + str(i + 2) for i in range(9, 12)]) + cluster.client = ready + cluster.diskdb = lambda node: cluster.launch_service(node + 3, ()) + cluster.diskio = lambda node: cluster.launch_service(node + 9, ()) + cluster.chunkdb = lambda node: cluster.launch_service(node + 6, ()) + cluster.chunk_kv = lambda: cluster.launch_service(12, ()) + cluster.access = lambda: cluster.launch_service(13, ()) + with patch("s3_node_fixture.Cluster.start", side_effect=start_metadata), patch("s3_node_fixture.start_node", side_effect=launch): + cluster.start() + self.assertEqual(cluster.servers, [f"{cluster.run_id}-node{i}" for i in range(14)]) + self.assertEqual([path.parent.name for path in cluster.data_paths], [f"node{i}" for i in range(14)]) + mounts = __import__("json").loads((cluster.artifacts / "mounts.json").read_text()) + self.assertEqual(mounts["host_data"], [str(path) for path in cluster.data_paths]) + + def test_s3_chunk_kv_restart_selects_fenced_recovery_readiness(self): + cluster = Mock() + cluster.timings = {} + run_recovery(cluster, "chunk-kv") + cluster.ready_chunk_kv.assert_called_once_with(recovering=True) + self.assertEqual(cluster.control.call_args_list, [ + call({"action": "kill", "node": 11}), + call({"action": "start", "node": 11})]) + + def test_protocol_restart_selects_fenced_recovery_readiness(self): + for kind in (AccessIcebergCluster, AccessDatasetCluster): + with self.subTest(kind=kind.__name__): + cluster = object.__new__(kind) + cluster.chunk_kv_index = 11 + cluster.ready_chunk_kv = Mock() + with patch.object(ChunkKvCluster, "control", return_value={"ok": True}): + cluster.control({"action": "start", "node": 11}) + cluster.ready_chunk_kv.assert_called_once_with(recovering=True) diff --git a/e2e-test/tests/test_dispatch_test.py b/e2e-test/tests/test_dispatch_test.py index 675946621..ab0eadec4 100644 --- a/e2e-test/tests/test_dispatch_test.py +++ b/e2e-test/tests/test_dispatch_test.py @@ -51,6 +51,15 @@ def test_compilation_cache_excludes_rootful_oci_state_and_runtime_artifacts(self self.assertIn('cache-directories: target/debug', block) self.assertIn("shared-key: container-e2e-client-v2-${{ hashFiles('pixi.lock') }}", block) + def test_platform_domains_have_separate_runners_and_compilation_only_cache(self): + block = self.workflow.split(' ContainerStorage:', 1)[1] + self.assertIn('max-parallel: 4', block) + self.assertIn('- group: dataset\n domains: dataset\n', block) + self.assertIn('- group: deployment\n domains: deployment\n', block) + self.assertNotIn('domains: dataset deployment', block) + self.assertIn('cache-targets: false', block) + self.assertIn('cache-directories: |\n target/debug\n sdk/target\n', block) + def test_missing_and_duplicate_domains_are_rejected(self): for workflow in ( self.workflow.replace("pixi run test-e2e-s3", "echo removed"), diff --git a/lib/crowdb-access-iceberg/Cargo.toml b/lib/crowdb-access-iceberg/Cargo.toml index 0807448f2..8883d347d 100644 --- a/lib/crowdb-access-iceberg/Cargo.toml +++ b/lib/crowdb-access-iceberg/Cargo.toml @@ -47,4 +47,4 @@ uuid = { version = "1", features = ["v4"] } [dev-dependencies] crowdb-access-iceberg = { path = ".", features = ["test-util"] } arc-swap = "1.9" -tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync", "time", "test-util"] } diff --git a/lib/crowdb-access-iceberg/src/catalog/recovery.rs b/lib/crowdb-access-iceberg/src/catalog/recovery.rs index 84a18633a..b40433912 100644 --- a/lib/crowdb-access-iceberg/src/catalog/recovery.rs +++ b/lib/crowdb-access-iceberg/src/catalog/recovery.rs @@ -17,7 +17,7 @@ impl CatalogRepository { operation: ManagementOperation, now_ms: u64, ) -> Result, CatalogError> { - let started = std::time::Instant::now(); + let started = tokio::time::Instant::now(); if operation.terminal() { self.audit(&operation, now_ms).await?; if operation.phase == ManagementPhase::Conflict { diff --git a/lib/crowdb-access-iceberg/src/catalog/repository.rs b/lib/crowdb-access-iceberg/src/catalog/repository.rs index 04b9371f8..72dcb8bbc 100644 --- a/lib/crowdb-access-iceberg/src/catalog/repository.rs +++ b/lib/crowdb-access-iceberg/src/catalog/repository.rs @@ -1,5 +1,5 @@ use std::sync::Arc; -use std::time::Instant; +use tokio::time::Instant; use crate::error::ValidationError; use crate::key::{CatalogId, CatalogScope, IcebergKey, OperationId, SystemScope}; diff --git a/lib/crowdb-access-iceberg/src/file/blocks.rs b/lib/crowdb-access-iceberg/src/file/blocks.rs index fbf9e420d..6f1e14896 100644 --- a/lib/crowdb-access-iceberg/src/file/blocks.rs +++ b/lib/crowdb-access-iceberg/src/file/blocks.rs @@ -5,7 +5,7 @@ use crowdb_chunk_client::{ SmallWriteMetricsSnapshot, }; use crowdb_protocol::chunkdb::rpc::Location; -use crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES; +use crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES; use sha2::{Digest, Sha256}; use std::sync::Arc; @@ -16,7 +16,7 @@ use super::{ChunkRoot, FileIdentity}; mod intent; pub const MAX_FILE_BLOCK_BYTES: usize = 256 * 1024; -pub const NATIVE_FILE_BLOCK_BYTES: usize = MAX_FRAME_PAYLOAD_BYTES; +pub const NATIVE_FILE_BLOCK_BYTES: usize = MAX_FRAME_V2_PAYLOAD_BYTES; #[derive(Debug, thiserror::Error)] pub enum FileIoError { diff --git a/lib/crowdb-access-iceberg/src/storage.rs b/lib/crowdb-access-iceberg/src/storage.rs index 6657ebd8a..24edddf67 100644 --- a/lib/crowdb-access-iceberg/src/storage.rs +++ b/lib/crowdb-access-iceberg/src/storage.rs @@ -16,7 +16,7 @@ use crowdb_chunk_kv_client::{ use crowdb_common::ec::EcScheme; use crowdb_kv_client::{ClientConfig as KvConfig, CrowdbKvClient}; use crowdb_protocol::chunkdb::rpc::{ChunkType, Location}; -use crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES; +use crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES; use crate::catalog::{CatalogRepository, ClearBounds, RoutedCatalogStore}; use crate::file::{FileBlockStore, NativeFileBlocks}; @@ -143,7 +143,7 @@ pub async fn prepare_file_writer( large_write: &LargeWritePolicy, ) -> Result { if let Some(length) = small_length { - if length <= MAX_FRAME_PAYLOAD_BYTES { + if length <= MAX_FRAME_V2_PAYLOAD_BYTES { let mut writer = chunks .prepare_small_write_for_key(length, location_key.as_bytes()) .await?; diff --git a/lib/crowdb-access-iceberg/tests/common/store.rs b/lib/crowdb-access-iceberg/tests/common/store.rs index bbed28412..b8bb553a0 100644 --- a/lib/crowdb-access-iceberg/tests/common/store.rs +++ b/lib/crowdb-access-iceberg/tests/common/store.rs @@ -12,6 +12,7 @@ use crowdb_protocol::chunk_kv::ClientRequestId; #[derive(Default)] pub struct TestStore { + pub yield_operations: bool, pub values: ArcSwap, StoredValue>>, pub fail_after: AtomicUsize, pub file_record_reply_loss: AtomicBool, @@ -109,6 +110,21 @@ impl TestStore { } } + async fn pause_namespace_authority(&self, key: &[u8], expected: Option<&[u8]>, value: &[u8]) { + if let Ok(crowdb_access_iceberg::record::StorageRecord::NamespaceAuthority(authority)) = + crowdb_access_iceberg::key::IcebergKey::decode(key) + .and_then(|key| crowdb_access_iceberg::record::StorageRecord::decode(&key, value)) + { + if authority.pending_operation.is_some() && expected.is_some() { + if let Some(barrier) = &self.namespace_update_barrier { + if self.namespace_update_visits.fetch_add(1, Ordering::SeqCst) < 2 { + barrier.wait().await; + } + } + } + } + } + async fn pause_stage_transition(&self, key: &[u8], expected: Option<&[u8]>) { if let (Some(barrier), Some(expected)) = (&self.stage_transition_barrier, expected) { if let Ok(crowdb_access_iceberg::record::StorageRecord::TableCreateOperation(operation)) = @@ -169,6 +185,9 @@ impl CatalogStore for TestStore { }) } async fn get(&self, key: &[u8]) -> Result, StoreError> { + if self.yield_operations { + tokio::task::yield_now().await; + } self.reads.fetch_add(1, Ordering::SeqCst); Ok(self.values.load().get(key).cloned()) } @@ -181,6 +200,9 @@ impl CatalogStore for TestStore { identity: ClientRequestId, ) -> Result { identity.validate().unwrap(); + if self.yield_operations { + tokio::task::yield_now().await; + } if self.deny_gc_workspace(key) { return Err(StoreError::Budget); } @@ -226,18 +248,7 @@ impl CatalogStore for TestStore { } } } - if let Ok(crowdb_access_iceberg::record::StorageRecord::NamespaceAuthority(authority)) = - crowdb_access_iceberg::key::IcebergKey::decode(key) - .and_then(|key| crowdb_access_iceberg::record::StorageRecord::decode(&key, value)) - { - if authority.pending_operation.is_some() && expected.is_some() { - if let Some(barrier) = &self.namespace_update_barrier { - if self.namespace_update_visits.fetch_add(1, Ordering::SeqCst) < 2 { - barrier.wait().await; - } - } - } - } + self.pause_namespace_authority(key, expected, value).await; if let Ok(crowdb_access_iceberg::record::StorageRecord::Active(root)) = crowdb_access_iceberg::key::IcebergKey::decode(key) .and_then(|key| crowdb_access_iceberg::record::StorageRecord::decode(&key, value)) @@ -277,6 +288,9 @@ impl CatalogStore for TestStore { return Err(StoreError::Response); } self.pause_table_head(key, expected, value, true).await; + if self.yield_operations { + tokio::task::yield_now().await; + } return Ok(CasOutcome::Applied(revision)); } } diff --git a/lib/crowdb-access-iceberg/tests/maintenance_timing_test.rs b/lib/crowdb-access-iceberg/tests/maintenance_timing_test.rs index 895f66672..da0857c05 100644 --- a/lib/crowdb-access-iceberg/tests/maintenance_timing_test.rs +++ b/lib/crowdb-access-iceberg/tests/maintenance_timing_test.rs @@ -10,7 +10,7 @@ use crowdb_access_iceberg::catalog::{ use crowdb_access_iceberg::key::OperationId; use crowdb_access_iceberg::operation::{ManagementAction, ManagementRequest, RequestIdentity}; -#[tokio::test] +#[tokio::test(start_paused = true)] async fn maintenance_observation_includes_time_spent_awaiting_fence_cas() { let store = Arc::new(common::TestStore::default()); let repository = CatalogRepository::new(store.clone(), ClearBounds::default()).unwrap(); diff --git a/lib/crowdb-access-iceberg/tests/management_test.rs b/lib/crowdb-access-iceberg/tests/management_test.rs index 741f3c428..3bdd06481 100644 --- a/lib/crowdb-access-iceberg/tests/management_test.rs +++ b/lib/crowdb-access-iceberg/tests/management_test.rs @@ -46,7 +46,7 @@ fn request(action: ManagementAction, epoch: u64, name: &str) -> ManagementReques } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn completed_clear_admits_one_retired_catalog_task() { let store = Arc::new(TestStore::default()); let catalog = repository(&store); @@ -95,7 +95,7 @@ async fn completed_clear_admits_one_retired_catalog_task() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn explicit_activation_preserves_catalog_and_replays_across_restarts() { let store = Arc::new(TestStore::default()); let catalog = repository(&store); @@ -179,7 +179,7 @@ async fn explicit_activation_preserves_catalog_and_replays_across_restarts() { assert_eq!(replacement.config_generation, 1); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn initialize_rename_clear_and_old_result_replay_preserve_identity() { let store = Arc::new(TestStore::default()); let repository = repository(&store); @@ -243,7 +243,7 @@ async fn initialize_rename_clear_and_old_result_replay_preserve_identity() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn every_lost_initialize_response_recovers_on_another_server() { for fail_after in 1..=8 { let store = Arc::new(TestStore::default()); @@ -266,7 +266,7 @@ async fn every_lost_initialize_response_recovers_on_another_server() { } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn every_lost_activation_response_recovers_one_profile_without_replacing_tables() { for failure in 1..=8 { let store = Arc::new(TestStore::default()); @@ -314,7 +314,7 @@ async fn every_lost_activation_response_recovers_one_profile_without_replacing_t } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn every_lost_clear_response_preserves_one_replacement_and_persisted_grace() { for failure in 1..=10 { let store = Arc::new(TestStore::default()); @@ -362,7 +362,7 @@ async fn every_lost_clear_response_preserves_one_replacement_and_persisted_grace } } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn denied_clear_and_digest_reuse_do_not_mutate() { let store = Arc::new(TestStore::default()); let initialize = request(ManagementAction::Initialize, 0, "catalog"); @@ -462,7 +462,7 @@ async fn concurrent_initializers() { assert_eq!(repository(&store).status().await.unwrap(), (root, authority)); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn delayed_fence_and_crash_restart_cannot_shorten_persisted_grace() { let store = Arc::new(TestStore::default()); let bounds = ClearBounds { @@ -538,7 +538,7 @@ async fn delayed_fence_and_crash_restart_cannot_shorten_persisted_grace() { ); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn explicit_clear_can_raise_delegation_bounds_without_shortening_old_reader_grace() { let store = Arc::new(TestStore::default()); let old = repository(&store) @@ -572,6 +572,7 @@ async fn explicit_clear_can_raise_delegation_bounds_without_shortening_old_reade assert_eq!(transition.bounds.delegated_access_ms, 50); assert_eq!(authority.admission_bounds, transition.bounds); let restarted = repository(&store); + let writes = store.writes.load(Ordering::SeqCst); assert!(matches!( restarted .execute( @@ -582,6 +583,7 @@ async fn explicit_clear_can_raise_delegation_bounds_without_shortening_old_reade .await, Err(CatalogError::Busy) )); + assert_eq!(store.writes.load(Ordering::SeqCst), writes); let ready = restarted .execute(clear, ManagementPrivilege::Clear, transition.complete_after_ms) .await @@ -590,7 +592,7 @@ async fn explicit_clear_can_raise_delegation_bounds_without_shortening_old_reade assert_ne!(ready.catalog, old.catalog); } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn clear_epoch_exhaustion_and_missing_confirmation_do_not_write() { use crowdb_access_iceberg::key::{IcebergKey, SystemScope}; use crowdb_access_iceberg::record::StorageRecord; diff --git a/lib/crowdb-access-iceberg/tests/namespace_create_test.rs b/lib/crowdb-access-iceberg/tests/namespace_create_test.rs index 606d6a417..ae47f5cbc 100644 --- a/lib/crowdb-access-iceberg/tests/namespace_create_test.rs +++ b/lib/crowdb-access-iceberg/tests/namespace_create_test.rs @@ -29,6 +29,34 @@ fn request(fixture: &TestNamespace, names: &[&str]) -> NamespaceCreateRequest { } } +#[tokio::test] +async fn simultaneous_helpers_preserve_the_same_namespace_result() { + let mut fixture = TestNamespace::new().await; + fixture.store = std::sync::Arc::new(common::TestStore { + values: arc_swap::ArcSwap::from(fixture.store.values.load_full()), + yield_operations: true, + ..Default::default() + }); + let request = request(&fixture, &["reference"]); + let creators = std::array::from_fn::<_, 3, _>(|_| NamespaceCreator::new(fixture.store.clone())); + let (first, second, third) = tokio::join!( + creators[0].create(&request), + creators[1].create(&request), + creators[2].create(&request) + ); + let first = first.unwrap(); + assert_eq!(first.status, 200); + assert_eq!(second.unwrap(), first); + assert_eq!(third.unwrap(), first); + let selected = NamespaceRepository::new(fixture.store.clone()) + .load(fixture.context, &request.identifier) + .await + .unwrap() + .unwrap(); + assert_eq!(selected.properties, request.properties); + assert_eq!(selected.pending_operation, None); +} + #[tokio::test] async fn native_create_publishes_top_level_and_nested_names_with_stable_replay() { let fixture = TestNamespace::new().await; diff --git a/lib/crowdb-access-s3/src/auth.rs b/lib/crowdb-access-s3/src/auth.rs index 2fe39e104..6cba5c482 100644 --- a/lib/crowdb-access-s3/src/auth.rs +++ b/lib/crowdb-access-s3/src/auth.rs @@ -78,6 +78,11 @@ pub enum AuthError { #[async_trait::async_trait] pub trait RequestAuthenticator: Send + Sync { + /// Current provider availability, when the authenticator can observe it. + fn is_ready(&self) -> Option { + None + } + async fn authenticate(&self, request: RawAuthRequest<'_>) -> Result<(), AuthError>; /// Returns an authenticated streaming seed whose full body must be verified. @@ -109,6 +114,10 @@ impl Default for TrustedNetworkAuthenticator { #[async_trait::async_trait] impl RequestAuthenticator for TrustedNetworkAuthenticator { + fn is_ready(&self) -> Option { + Some(true) + } + async fn authenticate(&self, _request: RawAuthRequest<'_>) -> Result<(), AuthError> { Ok(()) } diff --git a/lib/crowdb-access-s3/src/auth/sigv4.rs b/lib/crowdb-access-s3/src/auth/sigv4.rs index f0f70b96e..9ba997e68 100644 --- a/lib/crowdb-access-s3/src/auth/sigv4.rs +++ b/lib/crowdb-access-s3/src/auth/sigv4.rs @@ -29,9 +29,17 @@ pub struct Credential { pub trait CredentialProvider: Send + Sync { fn lookup(&self, access_key: &str) -> Option; + + fn is_ready(&self) -> bool { + true + } } impl CredentialProvider for std::sync::Arc { + fn is_ready(&self) -> bool { + (**self).is_ready() + } + fn lookup(&self, access_key: &str) -> Option { (**self).lookup(access_key) } @@ -56,6 +64,10 @@ impl

SigV4Verifier

{ #[async_trait::async_trait] impl RequestAuthenticator for SigV4Verifier

{ + fn is_ready(&self) -> Option { + Some(self.provider.is_ready()) + } + async fn authenticate(&self, request: RawAuthRequest<'_>) -> Result<(), AuthError> { let now = SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/lib/crowdb-access-s3/src/auth/snapshot.rs b/lib/crowdb-access-s3/src/auth/snapshot.rs index 97afec0e6..d28c377e0 100644 --- a/lib/crowdb-access-s3/src/auth/snapshot.rs +++ b/lib/crowdb-access-s3/src/auth/snapshot.rs @@ -82,6 +82,10 @@ impl CredentialCache { } impl CredentialProvider for CredentialCache { + fn is_ready(&self) -> bool { + self.is_ready_at(unix_seconds()) + } + fn lookup(&self, access_key: &str) -> Option { self.lookup_at(access_key, unix_seconds()) } diff --git a/lib/crowdb-access-s3/src/metrics.rs b/lib/crowdb-access-s3/src/metrics.rs index 416109c7b..58ba53421 100644 --- a/lib/crowdb-access-s3/src/metrics.rs +++ b/lib/crowdb-access-s3/src/metrics.rs @@ -283,11 +283,50 @@ impl S3Metrics { append_chunk_metrics(&mut output, chunks.large_write_buffer_metrics()); append_small_write_metrics(&mut output, &chunks.small_write_metrics()); append_read_flow_metrics(&mut output, chunks.read_flow_metrics()); + append_readiness_metrics(&mut output, chunks.readiness_metrics()); } output } } +fn append_readiness_metrics( + output: &mut String, + metrics: crowdb_chunk_client::metrics::ReadinessMetricsSnapshot, +) { + for (phase, snapshot) in [ + ("total", metrics.total), + ("hardware", metrics.hardware), + ("chunkdb", metrics.chunkdb), + ("diskio", metrics.diskio), + ] { + let labels = format!("phase=\"{phase}\""); + append_metric( + output, + "crowdb_s3_chunk_readiness_observations_total", + &labels, + snapshot.observations, + ); + append_metric( + output, + "crowdb_s3_chunk_readiness_errors_total", + &labels, + snapshot.errors, + ); + for (statistic, value) in [ + ("p50", snapshot.p50_ns), + ("p99", snapshot.p99_ns), + ("max", snapshot.max_ns), + ] { + append_metric( + output, + "crowdb_s3_chunk_readiness_latency_ns", + &format!("{labels},statistic=\"{statistic}\""), + value, + ); + } + } +} + fn append_request_metrics(output: &mut String, snapshot: &S3MetricsSnapshot) { let metric_matrices = [ ("crowdb_s3_requests_total", &snapshot.requests), @@ -569,6 +608,11 @@ impl S3Health { self.live.store(false, Ordering::Release); } + #[must_use] + pub fn is_live(&self) -> bool { + self.live.load(Ordering::Acquire) + } + #[must_use] pub fn snapshot(&self, metrics: &S3Metrics, native: Option<&NativeBodyAllocator>) -> S3HealthSnapshot { let native_metrics = native.map(NativeBodyAllocator::metrics_snapshot); @@ -625,6 +669,18 @@ pub struct S3Readiness { } impl S3Readiness { + #[must_use] + pub const fn unknown() -> Self { + Self { + listener: DependencyHealth::Unknown, + metadata: DependencyHealth::Unknown, + chunks: DependencyHealth::Unknown, + native_pool: DependencyHealth::Unknown, + cleanup: DependencyHealth::Unknown, + authentication: DependencyHealth::Unknown, + } + } + #[must_use] pub const fn is_ready(self) -> bool { self.listener.can_admit() @@ -642,12 +698,13 @@ pub enum DependencyHealth { Ready, Busy, Unavailable, + Unknown, } impl DependencyHealth { #[must_use] pub const fn can_admit(self) -> bool { - !matches!(self, Self::Unavailable) + matches!(self, Self::Ready | Self::Busy) } #[must_use] @@ -656,6 +713,7 @@ impl DependencyHealth { Self::Ready => "ready", Self::Busy => "busy", Self::Unavailable => "unavailable", + Self::Unknown => "unknown", } } } @@ -664,6 +722,7 @@ fn load_health(value: &AtomicU8) -> DependencyHealth { match value.load(Ordering::Acquire) { value if value == DependencyHealth::Ready as u8 => DependencyHealth::Ready, value if value == DependencyHealth::Busy as u8 => DependencyHealth::Busy, + value if value == DependencyHealth::Unknown as u8 => DependencyHealth::Unknown, _ => DependencyHealth::Unavailable, } } diff --git a/lib/crowdb-access-s3/src/native_buffer.rs b/lib/crowdb-access-s3/src/native_buffer.rs index 69da8b543..8268091e3 100644 --- a/lib/crowdb-access-s3/src/native_buffer.rs +++ b/lib/crowdb-access-s3/src/native_buffer.rs @@ -23,7 +23,7 @@ use crowdb_common::metrics::{Bandwidth, MetricName}; use crowdb_protocol::common::ChunkId; use crowdb_protocol::frame::{ encode_frame_regions, prepare_frame_regions, set_frame_chunk_id, FrameError, FrameMagic, - FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES, + FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES, }; use hyper::body::{Bytes, Http1BodyReceiveBuffer, Http1BodyReceiveProvider}; @@ -132,7 +132,7 @@ impl NativeBodyReceiver { )); } let frame_offset = slot * MAX_FRAME_BYTES; - let payload_offset = frame_offset + FRAME_HEADER_PREFIX_BYTES; + let payload_offset = frame_offset + FRAME_V2_HEADER_BYTES; let write_time_ms = SystemTime::now() .duration_since(UNIX_EPOCH) .map_or(0, |duration| { @@ -143,17 +143,17 @@ impl NativeBodyReceiver { unsafe { let header = std::slice::from_raw_parts_mut( owner.pointer.as_ptr().add(frame_offset), - FRAME_HEADER_PREFIX_BYTES, + FRAME_V2_HEADER_BYTES, ); let payload = std::slice::from_raw_parts( owner.pointer.as_ptr().add(payload_offset), - MAX_FRAME_PAYLOAD_BYTES, + MAX_FRAME_V2_PAYLOAD_BYTES, ); let footer = std::slice::from_raw_parts_mut( owner .pointer .as_ptr() - .add(payload_offset + MAX_FRAME_PAYLOAD_BYTES), + .add(payload_offset + MAX_FRAME_V2_PAYLOAD_BYTES), FRAME_FOOTER_BYTES, ); prepare_frame_regions(state.magic, payload, write_time_ms, header, footer) @@ -215,8 +215,8 @@ impl NativeBodyReceiver { owner: Arc, data: &Bytes, ) -> io::Result<()> { - for (slot, payload) in data.chunks(MAX_FRAME_PAYLOAD_BYTES).enumerate() { - let payload_offset = slot * MAX_FRAME_BYTES + FRAME_HEADER_PREFIX_BYTES; + for (slot, payload) in data.chunks(MAX_FRAME_V2_PAYLOAD_BYTES).enumerate() { + let payload_offset = slot * MAX_FRAME_BYTES + FRAME_V2_HEADER_BYTES; // SAFETY: the reserved owner is exclusively initialized here and // every destination payload slot is disjoint and large enough. unsafe { @@ -230,13 +230,13 @@ impl NativeBodyReceiver { io::Error::new(io::ErrorKind::InvalidData, "native payload length exceeds u16") })?; state.completed_slots += 1; - if payload.len() == MAX_FRAME_PAYLOAD_BYTES { + if payload.len() == MAX_FRAME_V2_PAYLOAD_BYTES { Self::prepare_full_slot(state, &owner, slot)?; } } state.next_slot = state.completed_slots; if let Some(last) = state.completed_slots.checked_sub(1) { - if usize::from(state.payload_lengths[last]) < MAX_FRAME_PAYLOAD_BYTES { + if usize::from(state.payload_lengths[last]) < MAX_FRAME_V2_PAYLOAD_BYTES { state.append_slot = Some(last); } } @@ -282,8 +282,14 @@ impl NativeBodyReceiver { /// Select frame identity before socket reception and hand off whole owners. /// # Errors - /// Returns an error if the receive state is concurrently in use. + /// Returns an error for incompatible framing or concurrent receive-state use. pub fn enable_owner_handoff_with_magic(&self, magic: FrameMagic) -> io::Result<()> { + if !matches!(magic, FrameMagic::RepoSmallV2 | FrameMagic::RepoLargeV2) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "native receive requires ordinary object revision 2", + )); + } let _guard = self.enter_state()?; // SAFETY: the guard exclusively owns the existing receive state. let state = unsafe { &mut *self.state.get() }; @@ -399,7 +405,7 @@ fn completed_owner(state: &mut ReceiverState, owner: Arc) -> io::Re .copied() .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "native owner has no completed slots"))?; let physical_len = (lengths.len() - 1) * MAX_FRAME_BYTES - + FRAME_HEADER_PREFIX_BYTES + + FRAME_V2_HEADER_BYTES + usize::from(last_payload) + FRAME_FOOTER_BYTES; state.completed_slots = 0; @@ -442,6 +448,9 @@ impl FramedWriteBuffer for NativeFramedOwner { } fn prepare_frames(&mut self, magic: FrameMagic, write_time_ms: u64) -> Result<(), FrameError> { + if !matches!(magic, FrameMagic::RepoSmallV2 | FrameMagic::RepoLargeV2) { + return Err(FrameError::InvalidRole); + } if self.magic != magic { self.prepared = false; self.prepared_slots = 0; @@ -455,13 +464,13 @@ impl FramedWriteBuffer for NativeFramedOwner { let frame_offset = index .checked_mul(MAX_FRAME_BYTES) .ok_or(FrameError::LengthOverflow)?; - let payload_offset = frame_offset + FRAME_HEADER_PREFIX_BYTES; + let payload_offset = frame_offset + FRAME_V2_HEADER_BYTES; // SAFETY: socket receive has completed this slot. Header and // footer regions are disjoint from immutable payload views. unsafe { let header = std::slice::from_raw_parts_mut( self.owner.pointer.as_ptr().add(frame_offset), - FRAME_HEADER_PREFIX_BYTES, + FRAME_V2_HEADER_BYTES, ); let payload = std::slice::from_raw_parts(self.owner.pointer.as_ptr().add(payload_offset), payload_len); @@ -483,21 +492,24 @@ impl FramedWriteBuffer for NativeFramedOwner { chunk_id: ChunkId, write_time_ms: u64, ) -> Result, FrameError> { + if !matches!(magic, FrameMagic::RepoSmallV2 | FrameMagic::RepoLargeV2) { + return Err(FrameError::InvalidRole); + } let payload_len = self .frame_payload_len(index) .ok_or(FrameError::InvalidLocationRange)?; let frame_offset = index .checked_mul(MAX_FRAME_BYTES) .ok_or(FrameError::LengthOverflow)?; - let payload_offset = frame_offset + FRAME_HEADER_PREFIX_BYTES; - let frame_len = FRAME_HEADER_PREFIX_BYTES + payload_len + FRAME_FOOTER_BYTES; + let payload_offset = frame_offset + FRAME_V2_HEADER_BYTES; + let frame_len = FRAME_V2_HEADER_BYTES + payload_len + FRAME_FOOTER_BYTES; // SAFETY: the provider never exposes the reserved header/footer bytes. // This consumed slot is their only writer; payload views alias only // the disjoint initialized payload range. unsafe { let header = std::slice::from_raw_parts_mut( self.owner.pointer.as_ptr().add(frame_offset), - FRAME_HEADER_PREFIX_BYTES, + FRAME_V2_HEADER_BYTES, ); let payload = std::slice::from_raw_parts(self.owner.pointer.as_ptr().add(payload_offset), payload_len); @@ -505,7 +517,7 @@ impl FramedWriteBuffer for NativeFramedOwner { self.owner.pointer.as_ptr().add(payload_offset + payload_len), FRAME_FOOTER_BYTES, ); - if self.prepared { + if self.prepared && self.magic == magic { set_frame_chunk_id(chunk_id, footer)?; } else { encode_frame_regions(magic, chunk_id, payload, write_time_ms, header, footer)?; @@ -595,8 +607,8 @@ impl Http1BodyReceiveProvider for NativeBodyReceiver { }, |slot| (slot, usize::from(state.payload_lengths[slot])), ); - let payload_offset = slot * MAX_FRAME_BYTES + FRAME_HEADER_PREFIX_BYTES + initial_len; - let capacity = requested.min(MAX_FRAME_PAYLOAD_BYTES - initial_len).min( + let payload_offset = slot * MAX_FRAME_BYTES + FRAME_V2_HEADER_BYTES + initial_len; + let capacity = requested.min(MAX_FRAME_V2_PAYLOAD_BYTES - initial_len).min( state .owner_bytes .saturating_sub(payload_offset + FRAME_FOOTER_BYTES), @@ -618,7 +630,7 @@ impl Http1BodyReceiveProvider for NativeBodyReceiver { state.owner = None; } debug_assert_eq!( - FRAME_HEADER_PREFIX_BYTES + MAX_FRAME_PAYLOAD_BYTES + FRAME_FOOTER_BYTES, + FRAME_V2_HEADER_BYTES + MAX_FRAME_V2_PAYLOAD_BYTES + FRAME_FOOTER_BYTES, MAX_FRAME_BYTES ); Poll::Ready(Ok(Box::new(NativeReceiveRegion { @@ -671,7 +683,7 @@ impl Http1BodyReceiveProvider for NativeBodyReceiver { if issued.initial_len == 0 { state.completed_slots += 1; } - if payload_len < MAX_FRAME_PAYLOAD_BYTES { + if payload_len < MAX_FRAME_V2_PAYLOAD_BYTES { state.append_slot = Some(issued.slot); } else { Self::prepare_full_slot(state, &owner, issued.slot)?; @@ -713,7 +725,8 @@ impl Http1BodyReceiveProvider for NativeBodyReceiver { )); } let owner_bytes = state.owner_bytes; - let logical_capacity = owner_bytes - state.payload_lengths.len() * 34; + let logical_capacity = + owner_bytes - state.payload_lengths.len() * (FRAME_V2_HEADER_BYTES + FRAME_FOOTER_BYTES); if data.len() > logical_capacity { return Err(io::Error::new( io::ErrorKind::InvalidData, diff --git a/lib/crowdb-access-s3/src/native_buffer/allocator.rs b/lib/crowdb-access-s3/src/native_buffer/allocator.rs index c3ac69b78..e758a72d0 100644 --- a/lib/crowdb-access-s3/src/native_buffer/allocator.rs +++ b/lib/crowdb-access-s3/src/native_buffer/allocator.rs @@ -87,12 +87,17 @@ impl NativeBodyAllocator { &self, payload_bytes: usize, ) -> Result { - let physical = crowdb_protocol::frame::framed_physical_length(payload_bytes as u64) + let physical = crowdb_protocol::frame::FrameLayout::OrdinaryV2 + .physical_length(payload_bytes as u64) .ok() .and_then(|bytes| usize::try_from(bytes).ok()) .filter(|bytes| *bytes <= self.state.budget_bytes) .ok_or(NativeBufferConfigError::InvalidOwnerSize)?; - Ok(self.receiver_with_owner_bytes(physical.max(34))) + Ok(self.receiver_with_owner_bytes( + physical.max( + crowdb_protocol::frame::FRAME_V2_HEADER_BYTES + crowdb_protocol::frame::FRAME_FOOTER_BYTES, + ), + )) } /// Attach the process bandwidth series before sharing this allocator. @@ -131,7 +136,7 @@ impl NativeBodyAllocator { payload_lengths: vec![0; owner_bytes.div_ceil(MAX_FRAME_BYTES)], append_slot: None, credit_wait_started: None, - magic: FrameMagic::RepoLargeV1, + magic: FrameMagic::RepoLargeV2, owner_bytes, }), state_in_use: AtomicBool::new(false), diff --git a/lib/crowdb-access-s3/src/streaming.rs b/lib/crowdb-access-s3/src/streaming.rs index e2242ceaa..2c6d81733 100644 --- a/lib/crowdb-access-s3/src/streaming.rs +++ b/lib/crowdb-access-s3/src/streaming.rs @@ -366,7 +366,7 @@ fn prepare_native_owner(owner: &mut NativeFramedOwner) -> Result<(), PutOutcome> u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) }); owner - .prepare_frames(FrameMagic::RepoLargeV1, write_time_ms) + .prepare_frames(FrameMagic::RepoLargeV2, write_time_ms) .map_err(|error| put_error(PutErrorCode::ChunkWrite, error)) } diff --git a/lib/crowdb-access-s3/tests/native_buffer_test.rs b/lib/crowdb-access-s3/tests/native_buffer_test.rs index b6d3b778a..b314b4cc2 100644 --- a/lib/crowdb-access-s3/tests/native_buffer_test.rs +++ b/lib/crowdb-access-s3/tests/native_buffer_test.rs @@ -7,9 +7,49 @@ use std::time::Duration; use crowdb_access_s3::native_buffer::NativeBodyAllocator; use crowdb_chunk_client::FramedWriteBuffer; use crowdb_protocol::common::ChunkId; -use crowdb_protocol::frame::{parse_frame, FrameMagic, MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_protocol::frame::{parse_frame, FrameMagic, MAX_FRAME_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES}; use hyper::body::{Bytes, Http1BodyReceiveProvider}; +#[tokio::test] +async fn native_owner_rejects_wrong_purposes_before_touching_reserved_regions() { + let allocator = NativeBodyAllocator::new(MAX_FRAME_BYTES, MAX_FRAME_BYTES).unwrap(); + let provider = allocator.object_receiver(); + for magic in [ + FrameMagic::BtreePageV2, + FrameMagic::PageIndexV2, + FrameMagic::StreamV2, + ] { + assert!(provider.enable_owner_handoff_with_magic(magic).is_err()); + } + provider + .enable_owner_handoff_with_magic(FrameMagic::RepoSmallV2) + .unwrap(); + let mut allocation = poll_fn(|cx| provider.poll_next_buffer(cx, 3)).await.unwrap(); + for (slot, value) in allocation.spare_capacity_mut().iter_mut().zip(*b"abc") { + slot.write(value); + } + allocation.advance(3).unwrap(); + drop(provider.on_data_ready(allocation).unwrap()); + let mut owner = provider.finish_owner_when_ready().await.unwrap().unwrap(); + for magic in [ + FrameMagic::BtreePageV2, + FrameMagic::PageIndexV2, + FrameMagic::StreamV2, + ] { + assert!(owner.prepare_frames(magic, 42).is_err()); + assert!(owner + .finalize_frame(0, magic, ChunkId { high: 7, low: 11 }, 42) + .is_err()); + } + owner.prepare_frames(FrameMagic::RepoSmallV2, 42).unwrap(); + let range = owner + .finalize_frame(0, FrameMagic::RepoSmallV2, ChunkId { high: 7, low: 11 }, 42) + .unwrap(); + let bytes = owner.views(range).unwrap().concat(); + let frame = parse_frame(&bytes, ChunkId { high: 7, low: 11 }).unwrap(); + assert_eq!(frame.payload, b"abc"); +} + #[tokio::test] async fn native_frame_retains_and_releases_allocator_credit() { let allocator = NativeBodyAllocator::new(MAX_FRAME_BYTES, MAX_FRAME_BYTES).unwrap(); @@ -137,12 +177,12 @@ async fn native_eof_owner_finalizes_reserved_frame_bytes_in_place() { assert_eq!(owner.frame_count(), 1); let chunk_id = ChunkId { high: 17, low: 29 }; let range = owner - .finalize_frame(0, FrameMagic::RepoLargeV1, chunk_id, 123) + .finalize_frame(0, FrameMagic::RepoLargeV2, chunk_id, 123) .unwrap(); let frame = owner.views(range).unwrap().pop().unwrap(); assert_eq!(parse_frame(&frame, chunk_id).unwrap().payload, payload); - assert_eq!(frame.as_ptr().wrapping_add(14), payload.as_ptr()); + assert_eq!(frame.as_ptr().wrapping_add(15), payload.as_ptr()); drop(frame); drop(owner); @@ -159,25 +199,25 @@ async fn full_owner_is_one_contiguous_buffer_across_chunk_views() { provider.enable_owner_handoff(); let mut payloads = Vec::new(); for value in [0x31, 0x72] { - let mut allocation = poll_fn(|cx| provider.poll_next_buffer(cx, MAX_FRAME_PAYLOAD_BYTES)) + let mut allocation = poll_fn(|cx| provider.poll_next_buffer(cx, MAX_FRAME_V2_PAYLOAD_BYTES)) .await .unwrap(); allocation .spare_capacity_mut() .fill(std::mem::MaybeUninit::new(value)); - allocation.advance(MAX_FRAME_PAYLOAD_BYTES).unwrap(); + allocation.advance(MAX_FRAME_V2_PAYLOAD_BYTES).unwrap(); payloads.push(provider.on_data_ready(allocation).unwrap()); } let mut owner = provider.take_ready_owner().unwrap(); assert_eq!(owner.frame_count(), 2); - assert_eq!(owner.logical_len(), (2 * MAX_FRAME_PAYLOAD_BYTES) as u64); + assert_eq!(owner.logical_len(), (2 * MAX_FRAME_V2_PAYLOAD_BYTES) as u64); let first_chunk = ChunkId { high: 1, low: 2 }; let second_chunk = ChunkId { high: 3, low: 4 }; let first = owner - .finalize_frame(0, FrameMagic::RepoLargeV1, first_chunk, 5) + .finalize_frame(0, FrameMagic::RepoLargeV2, first_chunk, 5) .unwrap(); let second = owner - .finalize_frame(1, FrameMagic::RepoLargeV1, second_chunk, 6) + .finalize_frame(1, FrameMagic::RepoLargeV2, second_chunk, 6) .unwrap(); assert_eq!(first, 0..MAX_FRAME_BYTES); assert_eq!(second, MAX_FRAME_BYTES..owner_bytes); @@ -217,11 +257,11 @@ async fn prefetched_payload_starts_the_first_native_owner_and_next_fill_appends( let mut owner = provider.finish_owner().unwrap().unwrap(); let chunk_id = ChunkId { high: 41, low: 43 }; let range = owner - .finalize_frame(0, FrameMagic::RepoLargeV1, chunk_id, 47) + .finalize_frame(0, FrameMagic::RepoLargeV2, chunk_id, 47) .unwrap(); let frame = owner.views(range).unwrap().pop().unwrap(); - assert_ne!(frame.as_ptr().wrapping_add(14), payload.as_ptr()); + assert_ne!(frame.as_ptr().wrapping_add(15), payload.as_ptr()); assert_eq!( parse_frame(&frame, chunk_id).unwrap().payload, b"prefetched-body-next" @@ -269,7 +309,7 @@ async fn prefetched_body_waits_for_credit_and_still_starts_at_body_offset_zero() .unwrap(); let chunk_id = ChunkId { high: 51, low: 53 }; let frame_range = owner - .finalize_frame(0, FrameMagic::RepoLargeV1, chunk_id, 59) + .finalize_frame(0, FrameMagic::RepoLargeV2, chunk_id, 59) .unwrap(); let frame = owner.views(frame_range).unwrap().pop().unwrap(); assert_eq!( @@ -308,7 +348,7 @@ async fn prefetched_body_at_eof_waits_for_credit_before_finalizing() { .unwrap(); let chunk_id = ChunkId { high: 61, low: 67 }; let frame_range = owner - .finalize_frame(0, FrameMagic::RepoLargeV1, chunk_id, 71) + .finalize_frame(0, FrameMagic::RepoLargeV2, chunk_id, 71) .unwrap(); let frame = owner.views(frame_range).unwrap().pop().unwrap(); assert_eq!(parse_frame(&frame, chunk_id).unwrap().payload, b"only-body"); @@ -321,7 +361,7 @@ async fn one_mib_owner_boundaries_follow_body_not_prefetched_header_size() { let allocator = NativeBodyAllocator::new(MIB, MIB).unwrap(); let provider = allocator.object_receiver(); provider.enable_owner_handoff(); - let prefix = Bytes::from(vec![b'p'; MAX_FRAME_PAYLOAD_BYTES - 2]); + let prefix = Bytes::from(vec![b'p'; MAX_FRAME_V2_PAYLOAD_BYTES - 2]); provider.on_prefetched_data(prefix.clone()).unwrap(); let mut first = poll_fn(|cx| provider.poll_next_buffer(cx, 4)).await.unwrap(); @@ -341,37 +381,37 @@ async fn one_mib_owner_boundaries_follow_body_not_prefetched_header_size() { let chunk_id = ChunkId { high: 73, low: 79 }; assert_eq!(owner.frame_count(), 2); let first_range = owner - .finalize_frame(0, FrameMagic::RepoLargeV1, chunk_id, 0) + .finalize_frame(0, FrameMagic::RepoLargeV2, chunk_id, 0) .unwrap(); let second_range = owner .finalize_frame( 1, - FrameMagic::RepoLargeV1, + FrameMagic::RepoLargeV2, chunk_id, - MAX_FRAME_PAYLOAD_BYTES as u64, + MAX_FRAME_V2_PAYLOAD_BYTES as u64, ) .unwrap(); assert_eq!(first_range, 0..MAX_FRAME_BYTES); assert_eq!(second_range.start, MAX_FRAME_BYTES); - assert_eq!(owner.logical_len(), (MAX_FRAME_PAYLOAD_BYTES + 2) as u64); + assert_eq!(owner.logical_len(), (MAX_FRAME_V2_PAYLOAD_BYTES + 2) as u64); let first_frame = owner.views(first_range).unwrap().pop().unwrap(); let second_frame = owner.views(second_range).unwrap().pop().unwrap(); assert_eq!( parse_frame(&first_frame, chunk_id).unwrap().payload.len(), - MAX_FRAME_PAYLOAD_BYTES + MAX_FRAME_V2_PAYLOAD_BYTES ); - assert_eq!(&first_frame[14..14 + prefix.len()], &prefix[..]); + assert_eq!(&first_frame[15..15 + prefix.len()], &prefix[..]); assert_eq!(parse_frame(&second_frame, chunk_id).unwrap().payload, b"34"); assert_eq!(allocator.allocation_count(), 1); } #[tokio::test] async fn small_native_objects_keep_one_owner_across_socket_reads_and_frames() { - for length in [1024, 512 * 1024, MAX_FRAME_PAYLOAD_BYTES, 1_200_000] { + for length in [1024, 512 * 1024, MAX_FRAME_V2_PAYLOAD_BYTES, 1_200_000] { let allocator = NativeBodyAllocator::new(2 * 1024 * 1024, 1024 * 1024).unwrap(); let provider = allocator.object_receiver_for_payload(length).unwrap(); provider - .enable_owner_handoff_with_magic(FrameMagic::RepoSmallV1) + .enable_owner_handoff_with_magic(FrameMagic::RepoSmallV2) .unwrap(); let mut fragments = Vec::new(); let mut remaining = length; @@ -392,14 +432,14 @@ async fn small_native_objects_keep_one_owner_across_socket_reads_and_frames() { .take_ready_owner() .unwrap_or_else(|| provider.finish_owner().unwrap().unwrap()); assert!(provider.take_ready_owner().is_none(), "owner is handed off once"); - let count = length.div_ceil(MAX_FRAME_PAYLOAD_BYTES); + let count = length.div_ceil(MAX_FRAME_V2_PAYLOAD_BYTES); assert_eq!(owner.frame_count(), count); - owner.prepare_frames(FrameMagic::RepoSmallV1, 123).unwrap(); + owner.prepare_frames(FrameMagic::RepoSmallV2, 123).unwrap(); let chunk = ChunkId { high: 11, low: 31 }; let mut end = 0; for index in 0..count { let range = owner - .finalize_frame(index, FrameMagic::RepoSmallV1, chunk, 123) + .finalize_frame(index, FrameMagic::RepoSmallV2, chunk, 123) .unwrap(); assert_eq!(range.start, end); end = range.end; @@ -407,19 +447,19 @@ async fn small_native_objects_keep_one_owner_across_socket_reads_and_frames() { let views = owner.views(0..end).unwrap(); assert_eq!(views.len(), 1, "one object must produce one contiguous view"); let bytes = &views[0]; - assert_eq!(bytes.len(), length + count * 34); - assert_eq!(bytes.as_ptr().wrapping_add(14), fragments[0].as_ptr()); + assert_eq!(bytes.len(), length + count * 35); + assert_eq!(bytes.as_ptr().wrapping_add(15), fragments[0].as_ptr()); let mut offset = 0; for index in 0..count { let payload_len = owner.frame_payload_len(index).unwrap(); - let frame_len = payload_len + 34; + let frame_len = payload_len + 35; let frame = parse_frame(&bytes[offset..offset + frame_len], chunk).unwrap(); assert!(frame.payload.iter().all(|byte| *byte == 0x5a)); offset += frame_len; } assert_eq!(allocator.metrics_snapshot().prefix_copy_bytes, 0); assert_eq!(allocator.allocation_count(), 1); - assert_eq!(allocator.retained_bytes(), length + count * 34); + assert_eq!(allocator.retained_bytes(), length + count * 35); drop(views); drop(owner); drop(fragments); diff --git a/lib/crowdb-access-s3/tests/streaming_test.rs b/lib/crowdb-access-s3/tests/streaming_test.rs index 6828d567f..255ab8325 100644 --- a/lib/crowdb-access-s3/tests/streaming_test.rs +++ b/lib/crowdb-access-s3/tests/streaming_test.rs @@ -19,7 +19,7 @@ use crowdb_access_s3::streaming::{ }; use crowdb_chunk_client::{ChunkIoWriter, FeedStatus, FramedWriteBuffer, IoError}; use crowdb_protocol::chunkdb::rpc::Location; -use crowdb_protocol::frame::{MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_protocol::frame::{MAX_FRAME_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES}; use hyper::body::{Body, Bytes, Frame, Http1BodyReceiveProvider, SizeHint}; #[test] @@ -292,13 +292,13 @@ async fn native_body_hashes_payload_and_hands_owner_to_writer_once() { let allocator = NativeBodyAllocator::new(MAX_FRAME_BYTES, MAX_FRAME_BYTES).unwrap(); let receiver = allocator.object_receiver(); receiver.enable_owner_handoff(); - let mut allocation = poll_fn(|cx| receiver.poll_next_buffer(cx, MAX_FRAME_PAYLOAD_BYTES)) + let mut allocation = poll_fn(|cx| receiver.poll_next_buffer(cx, MAX_FRAME_V2_PAYLOAD_BYTES)) .await .unwrap(); allocation .spare_capacity_mut() .fill(std::mem::MaybeUninit::new(0x5a)); - allocation.advance(MAX_FRAME_PAYLOAD_BYTES).unwrap(); + allocation.advance(MAX_FRAME_V2_PAYLOAD_BYTES).unwrap(); let payload = receiver.on_data_ready(allocation).unwrap(); let mut body = TestBody { frames: VecDeque::from([payload]), @@ -320,9 +320,12 @@ async fn native_body_hashes_payload_and_hands_owner_to_writer_once() { assert_eq!(writer.generic_frames, 0); assert_eq!(writer.owner_frames, 1); - assert_eq!(writer.logical_bytes, MAX_FRAME_PAYLOAD_BYTES as u64); + assert_eq!(writer.logical_bytes, MAX_FRAME_V2_PAYLOAD_BYTES as u64); assert_eq!(checksum.len(), 16); - assert_eq!(metrics.snapshot().checksum_bytes, MAX_FRAME_PAYLOAD_BYTES as u64); + assert_eq!( + metrics.snapshot().checksum_bytes, + MAX_FRAME_V2_PAYLOAD_BYTES as u64 + ); } #[tokio::test] diff --git a/lib/crowdb-chunk-client/Cargo.toml b/lib/crowdb-chunk-client/Cargo.toml index b95b5b310..04d7036c1 100644 --- a/lib/crowdb-chunk-client/Cargo.toml +++ b/lib/crowdb-chunk-client/Cargo.toml @@ -34,8 +34,10 @@ serde = { version = "1", features = ["derive"] } # crate's own integration tests. crowdb-chunk-client = { path = ".", features = ["test-util"] } crowdb-test-harness = { path = "../crowdb-test-harness" } +crowdb-diskio-client = { path = "../crowdb-diskio-client", features = ["test-util"] } +flatbuffers = { workspace = true } crowdb-diskdb-client = { path = "../crowdb-diskdb-client" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "test-util"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "test-util", "net"] } bincode = "1" serde_json = "1" diff --git a/lib/crowdb-chunk-client/src/chunk.rs b/lib/crowdb-chunk-client/src/chunk.rs index de3904948..46de1c757 100644 --- a/lib/crowdb-chunk-client/src/chunk.rs +++ b/lib/crowdb-chunk-client/src/chunk.rs @@ -17,7 +17,7 @@ pub mod chunk_reader; pub mod chunk_writer; pub(crate) mod client_recovery; pub mod ec_strip_writer; -pub mod mirror_chunk_writer; +pub mod frame_observation; pub mod mirror_flow; pub mod mirror_strip_writer; pub mod parity_writer; @@ -32,8 +32,12 @@ pub use chunk_reader::{ }; pub use chunk_writer::ChunkWriter; pub use ec_strip_writer::EcStripWriter; -pub use mirror_chunk_writer::{MirrorChunkWriter, STREAM_CHUNK_BYTES}; pub use mirror_flow::MirrorStripFlow; pub use mirror_strip_writer::MirrorStripWriter; pub use strip::{StripResult, StripWriter}; pub use strip_reader::StripReader; + +/// Maximum physical capacity of one Stream/Wal chunk. +pub const STREAM_CHUNK_BYTES: u64 = 256 * 1024 * 1024; +/// Physical allocation granularity of a growing Stream/Wal chunk. +pub const STREAM_STRIP_BYTES: u64 = 1024 * 1024; diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs index 39c47171f..33be7ab41 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader.rs @@ -16,19 +16,23 @@ use crowdb_protocol::chunkdb::rpc::{ }; use crowdb_protocol::common::ChunkId; use crowdb_protocol::diskdb::rpc::Segment; -use crowdb_protocol::frame::{ - parse_frame, parse_frame_views, ChunkLocation, FrameMagic, FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, - MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES, -}; use tokio::sync::Semaphore; use tokio::task::JoinSet; use super::client_recovery::ClientRecovery; -use super::read_credit::{retain, ReadBudget, ReadLease, StreamSlots}; +use super::read_credit::{ReadBudget, ReadLease, StreamSlots}; use super::strip_reader::StripReader; -use crate::metrics::ReadFlowMetrics; use crate::{ChunkAllocator, DiskWriter, IoError, ReadError, ReadResult}; +mod framing; +mod layout; +mod layout_cache; +mod streaming; +mod verified; + +use layout::{strip_end, strip_read_parts, validate_strip_layout}; +use layout_cache::FrameLayoutCache; + const KIB: u64 = 1024; const DEFAULT_STREAM_WINDOW: usize = 1024 * 1024; @@ -129,6 +133,7 @@ pub struct ChunkReader { policy: ChunkReadPolicy, flow_metrics: Arc, stream_budget: Arc, + frame_layouts: Arc, } impl ChunkReader { @@ -168,6 +173,7 @@ impl ChunkReader { policy, flow_metrics, stream_budget, + frame_layouts: Arc::default(), }) } @@ -367,201 +373,6 @@ impl ChunkReader { }) } - /// Reads one variable-length frame through the normal mirror/EC recovery - /// path and verifies both its integrity trailer and its public kind. - /// - /// Stream and tree metadata keep a compact physical frame location rather - /// than a synthetic fixed-stride `Location`. They use this entry point so - /// a checksum mismatch still marks the serving replica unavailable and - /// retries from a protected source before any payload is exposed. - #[allow( - clippy::too_many_lines, - reason = "layout acquisition, validation, failure observation, and retries share one frame-read state machine" - )] - pub async fn read_verified_frame( - &self, - chunk_id: ChunkId, - frame_offset: u64, - frame_length: u64, - expected_magic: FrameMagic, - ) -> ReadResult { - if frame_length < (FRAME_HEADER_PREFIX_BYTES + FRAME_FOOTER_BYTES) as u64 - || frame_length > MAX_FRAME_BYTES as u64 - { - return Err(ReadError::InvalidLocations( - "frame length is outside the public frame bounds".into(), - )); - } - let frame_end = frame_offset - .checked_add(frame_length) - .ok_or_else(|| ReadError::InvalidLocations("frame end overflows".into()))?; - for attempt in 1..=self.policy.max_layout_retries { - let query_started = Instant::now(); - let response = match self.query_chunk_timed(chunk_id).await { - Ok(response) => response, - Err(error) => { - tracing::warn!( - chunk_high = chunk_id.high, - chunk_low = chunk_id.low, - frame_offset, - frame_length, - attempt, - error = %error, - "verified frame layout query failed" - ); - return Err(map_metadata_error(error)); - } - }; - let layout_validity_ms = response.layout_validity_ms; - let validity = Duration::from_millis(layout_validity_ms); - let deadline = query_started + validity.saturating_sub(self.policy.layout_safety_margin); - let mut chunk = response - .chunk - .ok_or_else(|| ReadError::ChunkDeleted(format!("{}:{}", chunk_id.high, chunk_id.low)))?; - tracing::debug!( - chunk_high = chunk_id.high, - chunk_low = chunk_id.low, - frame_offset, - frame_length, - attempt, - layout_validity_ms, - strip_count = chunk.strips.len(), - "verified frame layout acquired" - ); - let (physical, observations) = match self - .read_chunk_range_partial(&chunk, frame_offset, frame_length, frame_offset) - .await - { - Ok(result) => result, - Err(error) => { - tracing::warn!( - chunk_high = chunk_id.high, - chunk_low = chunk_id.low, - frame_offset, - frame_length, - attempt, - error = %error, - "verified frame physical read failed" - ); - return Err(error); - } - }; - let bytes = match extract_physical_range(&physical.ranges, frame_offset..frame_end) { - Ok(bytes) if physical.failures.is_empty() => bytes, - Ok(_) | Err(_) => { - let failed_start = physical.failures.first().map(|failure| failure.start); - let failed_end = physical.failures.first().map(|failure| failure.end); - let failed_error = physical.failures.first().map(|failure| failure.error.to_string()); - let expired = Instant::now() >= deadline; - let observation_error = if expired { - None - } else { - self.mark_observed_failures(&mut chunk, observations).await.err() - }; - tracing::warn!( - chunk_high = chunk_id.high, - chunk_low = chunk_id.low, - frame_offset, - frame_length, - attempt, - completed_ranges = physical.ranges.len(), - failed_ranges = physical.failures.len(), - failed_start = ?failed_start, - failed_end = ?failed_end, - failed_error = ?failed_error, - expired, - observation_error = ?observation_error, - "verified frame read did not produce a complete range" - ); - if expired || observation_error.is_some() { - return Err(ReadError::DataLoss( - "frame bytes could not be reconstructed".into(), - )); - } - continue; - } - }; - let decode_started = Instant::now(); - let parsed_frame = parse_frame(&bytes, chunk_id); - let parse_ns = u64::try_from(decode_started.elapsed().as_nanos()).unwrap_or(u64::MAX); - self.flow_metrics.frame_decode_wait_ns.inc_by(parse_ns); - self.flow_metrics.frame_parse_wait_ns.inc_by(parse_ns); - match parsed_frame { - Ok(frame) if frame.header.magic == expected_magic => { - if Instant::now() >= deadline { - tracing::warn!( - chunk_high = chunk_id.high, - chunk_low = chunk_id.low, - frame_offset, - frame_length, - attempt, - elapsed_ms = query_started.elapsed().as_millis(), - layout_validity_ms, - "verified frame read exceeded layout validity" - ); - continue; - } - self.mark_observed_failures(&mut chunk, observations).await?; - return Ok(bytes); - } - Ok(frame) => { - tracing::warn!( - chunk_high = chunk_id.high, - chunk_low = chunk_id.low, - frame_offset, - frame_length, - attempt, - expected_magic = ?expected_magic, - actual_magic = ?frame.header.magic, - "verified frame kind disagrees with its location" - ); - return Err(ReadError::DataLoss( - "frame kind disagrees with its location".into(), - )); - } - Err(error) => { - let served_segments: Vec = observations - .iter() - .flat_map(|observation| observation.served_segments.iter().copied()) - .collect(); - let corrupt = mark_served_segments_corrupt(observations); - let corrupt_segments = corrupt.len(); - let expired = Instant::now() >= deadline; - let observation_error = if corrupt_segments == 0 || expired { - None - } else { - self.mark_observed_failures(&mut chunk, corrupt).await.err() - }; - tracing::warn!( - chunk_high = chunk_id.high, - chunk_low = chunk_id.low, - frame_offset, - frame_length, - attempt, - error = %error, - corrupt_segments, - served_segments = ?served_segments, - expired, - observation_error = ?observation_error, - "verified frame parsing failed" - ); - if corrupt_segments == 0 || expired || observation_error.is_some() { - return Err(ReadError::DataLoss(format!("invalid chunk frame: {error}"))); - } - } - } - } - tracing::warn!( - chunk_high = chunk_id.high, - chunk_low = chunk_id.low, - frame_offset, - frame_length, - attempts = self.policy.max_layout_retries, - "verified frame read exhausted valid chunk layouts" - ); - Err(ReadError::LayoutExpired) - } - async fn read_location_partial( &self, location: &Location, @@ -626,128 +437,6 @@ impl ChunkReader { Err(ReadError::LayoutExpired) } - async fn read_framed_location( - &self, - location: &Location, - local_start: u64, - length: u64, - logical_start: u64, - first_layout: Option, - ) -> ReadResult { - let chunk_id = location - .chunk_id - .ok_or_else(|| ReadError::InvalidLocations("location has no chunk ID".into()))?; - if local_start - .checked_add(length) - .map_or(true, |end| end > location.logical_length) - { - return Err(ReadError::InvalidLocations( - "framed location logical range is invalid".into(), - )); - } - let framed = ChunkLocation { - chunk_id, - frame_offset: location.offset, - logical_length: location.logical_length, - }; - if framed - .physical_length() - .map_err(|error| ReadError::InvalidLocations(error.to_string()))? - != location.length - { - return Err(ReadError::InvalidLocations( - "framed location length is inconsistent".into(), - )); - } - let physical_range = framed - .physical_range_for_subrange(local_start..local_start.saturating_add(length)) - .map_err(|error| ReadError::InvalidLocations(error.to_string()))?; - let selected_logical_start = - (local_start / MAX_FRAME_PAYLOAD_BYTES as u64) * MAX_FRAME_PAYLOAD_BYTES as u64; - self.read_verified_framed_range( - framed, - physical_range, - selected_logical_start, - local_start..local_start + length, - logical_start, - first_layout, - ) - .await - } - - async fn read_verified_framed_range( - &self, - framed: ChunkLocation, - physical_range: Range, - selected_logical_start: u64, - requested: Range, - output_logical_start: u64, - mut first_layout: Option, - ) -> ReadResult { - let chunk_id = framed.chunk_id; - for _ in 0..self.policy.max_layout_retries { - let layout = match first_layout.take().filter(LayoutSnapshot::usable) { - Some(layout) => layout, - None => self.query_layout(chunk_id).await?, - }; - let (physical, observations) = self - .read_chunk_range_partial( - &layout.chunk, - physical_range.start, - physical_range.end - physical_range.start, - physical_range.start, - ) - .await?; - let decode_started = Instant::now(); - let decoded = decode_framed_partial( - &physical, - framed, - selected_logical_start, - requested.clone(), - output_logical_start, - &self.flow_metrics, - ); - self.flow_metrics - .frame_decode_wait_ns - .inc_by(u64::try_from(decode_started.elapsed().as_nanos()).unwrap_or(u64::MAX)); - let parsed = match decoded { - Ok(parsed) => parsed, - Err(error) => { - layout.invalidate(); - let corrupt = mark_served_segments_corrupt(observations); - if corrupt.is_empty() || Instant::now() >= layout.deadline { - return Err(error); - } - let mut chunk = (*layout.chunk).clone(); - if self.mark_observed_failures(&mut chunk, corrupt).await.is_err() { - return Err(error); - } - continue; - } - }; - if Instant::now() >= layout.deadline { - layout.invalidate(); - continue; - } - if observations - .iter() - .any(|observation| !observation.failed_segments.is_empty()) - { - layout.invalidate(); - let mut chunk = (*layout.chunk).clone(); - if self - .mark_observed_failures(&mut chunk, observations) - .await - .is_err() - { - continue; - } - } - return Ok(parsed); - } - Err(ReadError::LayoutExpired) - } - async fn read_chunk_range_partial( &self, chunk: &Chunk, @@ -911,97 +600,6 @@ impl ChunkReader { } } -fn decode_framed_partial( - physical: &PartialReadResult, - framed: ChunkLocation, - selected_logical_start: u64, - requested: Range, - output_logical_start: u64, - flow_metrics: &ReadFlowMetrics, -) -> ReadResult { - let first_frame = selected_logical_start / MAX_FRAME_PAYLOAD_BYTES as u64; - let last_frame = (requested.end - 1) / MAX_FRAME_PAYLOAD_BYTES as u64; - let mut result = PartialReadResult::default(); - let mut magic = None; - for frame_index in first_frame..=last_frame { - let frame_logical_start = frame_index * MAX_FRAME_PAYLOAD_BYTES as u64; - let payload_len = (framed.logical_length - frame_logical_start).min(MAX_FRAME_PAYLOAD_BYTES as u64); - let frame_start = framed - .frame_offset - .checked_add(frame_index * MAX_FRAME_BYTES as u64) - .ok_or_else(|| ReadError::InvalidLocations("frame offset overflows".into()))?; - let frame_end = frame_start - .checked_add(FRAME_HEADER_PREFIX_BYTES as u64 + payload_len + FRAME_FOOTER_BYTES as u64) - .ok_or_else(|| ReadError::InvalidLocations("frame end overflows".into()))?; - let wanted_start = requested.start.max(frame_logical_start); - let wanted_end = requested.end.min(frame_logical_start + payload_len); - let output_start = output_logical_start + wanted_start - requested.start; - let output_end = output_logical_start + wanted_end - requested.start; - if physical - .failures - .iter() - .any(|failure| failure.start < frame_end && failure.end > frame_start) - { - result.failures.push(FailedReadRange { - start: output_start, - end: output_end, - error: ReadError::DataLoss("frame bytes could not be reconstructed".into()), - }); - continue; - } - let views = extract_physical_views(&physical.ranges, frame_start..frame_end)?; - let parse_started = Instant::now(); - let parsed = if views.len() == 1 { - parse_frame(&views[0], framed.chunk_id).map(|frame| (frame.header, frame.physical_length)) - } else { - let slices: Vec<_> = views.iter().map(Bytes::as_ref).collect(); - parse_frame_views(&slices, framed.chunk_id).map(|frame| (frame.header, frame.physical_length)) - }; - flow_metrics - .frame_parse_wait_ns - .inc_by(u64::try_from(parse_started.elapsed().as_nanos()).unwrap_or(u64::MAX)); - let (header, physical_length) = - parsed.map_err(|error| ReadError::DataLoss(format!("invalid chunk frame: {error}")))?; - if !matches!(header.magic, FrameMagic::RepoSmallV1 | FrameMagic::RepoLargeV1) { - return Err(ReadError::DataLoss( - "framed location has unsupported frame kind".into(), - )); - } - match magic { - Some(previous) if previous != header.magic => { - return Err(ReadError::DataLoss("framed location mixes frame kinds".into())); - } - None => magic = Some(header.magic), - Some(_) => {} - } - if physical_length as u64 != frame_end - frame_start { - return Err(ReadError::DataLoss("frame length disagrees with location".into())); - } - let payload_start = usize::try_from(wanted_start - frame_logical_start) - .map_err(|_| ReadError::InvalidLocations("frame payload range overflows".into()))?; - let payload_end = usize::try_from(wanted_end - frame_logical_start) - .map_err(|_| ReadError::InvalidLocations("frame payload range overflows".into()))?; - let physical_payload_start = usize::from(header.payload_offset) + payload_start; - let physical_payload_end = usize::from(header.payload_offset) + payload_end; - let mut logical_cursor = output_start; - for view in slice_views(&views, physical_payload_start..physical_payload_end)? { - let view_end = logical_cursor + view.len() as u64; - result.ranges.push(ReadRangeData { - start: logical_cursor, - end: view_end, - data: view, - }); - logical_cursor = view_end; - } - if logical_cursor != output_end { - return Err(ReadError::InvalidLocations( - "frame payload views are incomplete".into(), - )); - } - } - Ok(result) -} - fn extract_physical_range(ranges: &[ReadRangeData], wanted: Range) -> ReadResult { let first = ranges.partition_point(|range| range.end <= wanted.start); if let Some(range) = ranges.get(first) { @@ -1045,66 +643,6 @@ fn extract_physical_range(ranges: &[ReadRangeData], wanted: Range) -> ReadR Ok(output.freeze()) } -fn extract_physical_views(ranges: &[ReadRangeData], wanted: Range) -> ReadResult> { - let first = ranges.partition_point(|range| range.end <= wanted.start); - let mut output = Vec::new(); - let mut cursor = wanted.start; - for range in &ranges[first..] { - if range.start >= wanted.end { - break; - } - if range.end <= cursor { - continue; - } - if range.start > cursor { - return Err(ReadError::InvalidLocations( - "physical frame data has a gap".into(), - )); - } - let end = wanted.end.min(range.end); - let offset = usize::try_from(cursor - range.start) - .map_err(|_| ReadError::InvalidLocations("physical frame offset overflows".into()))?; - let length = usize::try_from(end - cursor) - .map_err(|_| ReadError::InvalidLocations("physical frame length overflows".into()))?; - output.push(range.data.slice(offset..offset + length)); - cursor = end; - if cursor == wanted.end { - break; - } - } - if cursor != wanted.end { - return Err(ReadError::InvalidLocations( - "physical frame data is incomplete".into(), - )); - } - Ok(output) -} - -fn slice_views(views: &[Bytes], wanted: Range) -> ReadResult> { - let mut output = Vec::new(); - let mut cursor = 0_usize; - let mut covered = wanted.start; - for view in views { - let view_end = cursor.saturating_add(view.len()); - if view_end > covered && cursor < wanted.end { - let start = covered.saturating_sub(cursor); - let end = (wanted.end - cursor).min(view.len()); - output.push(view.slice(start..end)); - covered += end - start; - } - cursor = view_end; - if covered == wanted.end { - break; - } - } - if covered != wanted.end { - return Err(ReadError::InvalidLocations( - "frame payload views are incomplete".into(), - )); - } - Ok(output) -} - struct StripFailureObservation { strip_sequence: u32, failed_segments: Vec, @@ -1151,213 +689,6 @@ pub struct ChunkReadStream { pending_ranges: std::vec::IntoIter, } -impl ChunkReadStream { - pub async fn next_chunk(&mut self) -> Option> { - loop { - if let Some(pending) = self.next_pending() { - return Some(pending); - } - if self.delivery_cursor >= self.end { - return None; - } - let budget = Arc::clone(&self.reader.stream_budget); - let _registration = budget.register(); - let notified = budget.wake.notified(); - tokio::pin!(notified); - notified.as_mut().enable(); - if let Err(error) = self.fill_slots().await { - self.abort(error); - continue; - } - if let Some((start, end, result, lease)) = self.completed.remove(&self.delivery_cursor) { - self.load_unit(start, end, result, &lease); - continue; - } - if self.reads.is_empty() { - let started = Instant::now(); - notified.await; - self.reader - .flow_metrics - .stream_credit_wait_ns - .inc_by(u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX)); - continue; - } - match self.reads.join_next().await { - Some(Ok(completed)) => { - self.reader.flow_metrics.stream_units_completed.inc(); - if completed.0 != self.delivery_cursor { - self.reader.flow_metrics.stream_out_of_order.inc(); - } - self.completed.insert(completed.0, completed); - } - Some(Err(error)) => self.abort(ReadError::DiskIo(error.to_string())), - None => {} - } - } - } - - async fn fill_slots(&mut self) -> ReadResult<()> { - while self.fetch_cursor < self.end { - let (next, physical_bytes) = self.next_unit(self.fetch_cursor)?; - let Some(lease) = self.slots.try_reserve(&self.reader.stream_budget, physical_bytes) else { - self.reader.flow_metrics.stream_credit_stalls.inc(); - break; - }; - let start = self.fetch_cursor; - let index = self.locations.partition_point(|location| { - location.logical_offset.saturating_add(location.logical_length) <= start - }); - let location = self.locations[index].clone(); - let layout = self - .layout_for( - location - .chunk_id - .ok_or_else(|| ReadError::InvalidLocations("location has no chunk ID".into()))?, - ) - .await?; - let reader = self.reader.clone(); - self.reader.flow_metrics.stream_windows.inc(); - self.reader.flow_metrics.range_locations_examined.inc(); - self.reads.spawn(async move { - let local_start = start - location.logical_offset; - let result = reader - .read_location_partial_cached(&location, local_start, next - start, start, Some(layout)) - .await; - (start, next, result, lease) - }); - self.fetch_cursor = next; - } - Ok(()) - } - - async fn layout_for(&mut self, chunk_id: ChunkId) -> ReadResult { - if let Some(layout) = self - .layouts - .iter() - .find(|layout| layout.chunk_id == chunk_id && layout.usable()) - { - return Ok(layout.clone()); - } - let reader = self.reader.clone(); - let layout = tokio::spawn(async move { reader.query_layout(chunk_id).await }) - .await - .map_err(|error| ReadError::Metadata(error.to_string()))??; - self.layouts.retain(|cached| cached.chunk_id != chunk_id); - if self.layouts.len() == 8 { - self.layouts.remove(0); - } - self.layouts.push(layout.clone()); - Ok(layout) - } - - fn next_unit(&self, start: u64) -> ReadResult<(u64, usize)> { - let first = self.locations.partition_point(|location| { - location.logical_offset.saturating_add(location.logical_length) <= start - }); - let location = self - .locations - .get(first) - .ok_or_else(|| ReadError::InvalidLocations("stream has no location for requested byte".into()))?; - let local = start - location.logical_offset; - let window = self.window_bytes.min(DEFAULT_STREAM_WINDOW as u64); - if location.length == location.logical_length { - let length = window.min(location.logical_length - local).min(self.end - start); - return Ok((start + length, length as usize)); - } - let frames = (window / MAX_FRAME_BYTES as u64).max(1); - let frame_index = local / MAX_FRAME_PAYLOAD_BYTES as u64; - let local_end = ((frame_index + frames) * MAX_FRAME_PAYLOAD_BYTES as u64) - .min(location.logical_length) - .min(self.end - location.logical_offset); - let frame_location = ChunkLocation { - chunk_id: location - .chunk_id - .ok_or_else(|| ReadError::InvalidLocations("location has no chunk ID".into()))?, - frame_offset: location.offset, - logical_length: location.logical_length, - }; - let physical = frame_location - .physical_range_for_subrange(local..local_end) - .map_err(|error| ReadError::InvalidLocations(error.to_string()))?; - let bytes = usize::try_from(physical.end - physical.start) - .map_err(|_| ReadError::InvalidLocations("stream unit exceeds address space".into()))?; - Ok((location.logical_offset + local_end, bytes)) - } - - fn load_unit( - &mut self, - start: u64, - end: u64, - result: ReadResult, - lease: &Arc, - ) { - self.delivery_cursor = end; - let partial = match result { - Ok(partial) => partial, - Err(error) => { - self.abort(error); - return; - } - }; - let failure = partial.failures.into_iter().next(); - let verified_end = failure.as_ref().map_or(end, |failure| failure.start); - let mut ranges = partial.ranges; - let prefix_len = ranges.partition_point(|range| range.end <= verified_end); - let mut cursor = start; - for range in &ranges[..prefix_len] { - if range.start != cursor || range.end - range.start != range.data.len() as u64 { - self.abort(ReadError::InvalidLocations(format!( - "read ranges are not contiguous at byte {cursor}" - ))); - return; - } - cursor = range.end; - } - if cursor != verified_end { - self.abort(ReadError::InvalidLocations(format!( - "read ranges are not contiguous at byte {cursor}" - ))); - return; - } - ranges.truncate(prefix_len); - for range in &mut ranges { - range.data = retain(std::mem::take(&mut range.data), Arc::clone(lease)); - } - self.pending_ranges = ranges.into_iter(); - if let Some(failure) = failure { - self.reads.abort_all(); - self.completed.clear(); - self.fetch_cursor = self.end; - self.delivery_cursor = self.end; - self.pending_error = Some(ReadError::FailedRange { - start: failure.start, - end: failure.end, - message: failure.error.to_string(), - }); - } - } - - fn abort(&mut self, error: ReadError) { - self.reads.abort_all(); - self.completed.clear(); - self.fetch_cursor = self.end; - self.delivery_cursor = self.end; - self.pending_error = Some(error); - } - - fn next_pending(&mut self) -> Option> { - if let Some(range) = self.pending_ranges.next() { - self.cursor = range.end; - return Some(Ok(range.data)); - } - if let Some(error) = self.pending_error.take() { - self.cursor = self.end; - return Some(Err(error)); - } - None - } -} - fn complete_read(partial: PartialReadResult, start: u64, end: u64) -> ReadResult> { if let Some(failure) = partial.failures.into_iter().next() { return Err(ReadError::FailedRange { @@ -1437,65 +768,6 @@ fn normalize_locations(locations: &[Location]) -> ReadResult<(Vec, u64 Ok((locations, cursor)) } -fn validate_strip_layout(chunk: &Chunk) -> ReadResult<()> { - let mut previous_end = 0; - for (index, strip) in chunk.strips.iter().enumerate() { - let start = u64::from(strip.chunk_offset) * KIB; - let end = strip_end(strip)?; - if strip.capacity == 0 || strip.unit_kb == 0 || end <= start || (index > 0 && start < previous_end) { - return Err(ReadError::InvalidLocations(format!( - "chunk strip {} has invalid or overlapping geometry", - strip.strip_sequence - ))); - } - previous_end = end; - } - Ok(()) -} - -fn strip_end(strip: &crowdb_protocol::chunkdb::rpc::ChunkStrip) -> ReadResult { - u64::from(strip.chunk_offset) - .checked_add(u64::from(strip.capacity)) - .and_then(|kib| kib.checked_mul(KIB)) - .ok_or_else(|| ReadError::InvalidLocations("strip interval overflows".into())) -} - -fn strip_read_parts( - strip: &crowdb_protocol::chunkdb::rpc::ChunkStrip, - start: u64, - end: u64, -) -> ReadResult> { - let Some(crowdb_protocol::chunkdb::rpc::Strip::EcStrip(ec)) = strip.strip.as_ref() else { - return Ok(vec![(start, end)]); - }; - let first = ec - .segments - .first() - .ok_or_else(|| ReadError::InvalidLocations("EC strip has no segments".into()))?; - let shard_bytes = u64::from(first.unit_count) - .checked_mul(u64::from(strip.unit_kb)) - .and_then(|value| value.checked_mul(KIB)) - .ok_or_else(|| ReadError::InvalidLocations("EC shard size overflows".into()))?; - if shard_bytes == 0 { - return Err(ReadError::InvalidLocations("EC shard size is zero".into())); - } - let strip_start = u64::from(strip.chunk_offset) * KIB; - let mut parts = Vec::new(); - let mut cursor = start; - while cursor < end { - let local = cursor - strip_start; - let next_boundary = strip_start.saturating_add( - (local / shard_bytes) - .saturating_add(1) - .saturating_mul(shard_bytes), - ); - let part_end = end.min(next_boundary); - parts.push((cursor, part_end)); - cursor = part_end; - } - Ok(parts) -} - fn map_metadata_error(error: IoError) -> ReadError { match error { IoError::ChunkNotFound(message) => ReadError::ChunkDeleted(message), diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/framing.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/framing.rs new file mode 100644 index 000000000..579331739 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/framing.rs @@ -0,0 +1,315 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + mark_served_segments_corrupt, ChunkReader, FailedReadRange, LayoutSnapshot, PartialReadResult, + ReadRangeData, +}; +use crate::metrics::ReadFlowMetrics; +use crate::{ReadError, ReadResult}; +use bytes::Bytes; +use crowdb_protocol::chunkdb::rpc::Location; +use crowdb_protocol::frame::{ + parse_frame, parse_frame_views, ChunkLocation, FrameLayout, FrameMagic, FRAME_FOOTER_BYTES, + MAX_FRAME_BYTES, +}; +use std::ops::Range; +use std::time::Instant; + +#[derive(Clone, Copy)] +struct FramedLocation { + location: ChunkLocation, + geometry: FrameLayout, +} + +pub(super) fn object_layout(location: &Location) -> ReadResult { + if FrameLayout::OrdinaryV2 + .physical_length(location.logical_length) + .ok() + == Some(location.length) + { + return Ok(FrameLayout::OrdinaryV2); + } + Err(ReadError::InvalidLocations( + "framed location length is inconsistent".into(), + )) +} + +impl ChunkReader { + pub(super) async fn read_framed_location( + &self, + location: &Location, + local_start: u64, + length: u64, + logical_start: u64, + first_layout: Option, + ) -> ReadResult { + let chunk_id = location + .chunk_id + .ok_or_else(|| ReadError::InvalidLocations("location has no chunk ID".into()))?; + if local_start + .checked_add(length) + .map_or(true, |end| end > location.logical_length) + { + return Err(ReadError::InvalidLocations( + "framed location logical range is invalid".into(), + )); + } + let geometry = object_layout(location)?; + let framed = FramedLocation { + location: ChunkLocation { + chunk_id, + frame_offset: location.offset, + logical_length: location.logical_length, + }, + geometry, + }; + let physical_range = geometry + .physical_range(framed.location, local_start..local_start + length) + .map_err(|error| ReadError::InvalidLocations(error.to_string()))?; + let capacity = geometry.payload_capacity(); + let selected_logical_start = (local_start / capacity) * capacity; + self.read_verified_framed_range( + framed, + physical_range, + selected_logical_start, + local_start..local_start + length, + logical_start, + first_layout, + ) + .await + } + + async fn read_verified_framed_range( + &self, + framed: FramedLocation, + physical_range: Range, + selected_logical_start: u64, + requested: Range, + output_logical_start: u64, + mut first_layout: Option, + ) -> ReadResult { + let chunk_id = framed.location.chunk_id; + for _ in 0..self.policy.max_layout_retries { + let layout = match first_layout.take().filter(LayoutSnapshot::usable) { + Some(layout) => layout, + None => self.query_layout(chunk_id).await?, + }; + let (physical, observations) = self + .read_chunk_range_partial( + &layout.chunk, + physical_range.start, + physical_range.end - physical_range.start, + physical_range.start, + ) + .await?; + let decode_started = Instant::now(); + let decoded = decode_framed_partial( + &physical, + framed, + selected_logical_start, + requested.clone(), + output_logical_start, + &self.flow_metrics, + ); + self.flow_metrics + .frame_decode_wait_ns + .inc_by(u64::try_from(decode_started.elapsed().as_nanos()).unwrap_or(u64::MAX)); + let parsed = match decoded { + Ok(parsed) => parsed, + Err(error) => { + layout.invalidate(); + let corrupt = mark_served_segments_corrupt(observations); + if corrupt.is_empty() || Instant::now() >= layout.deadline { + return Err(error); + } + let mut chunk = (*layout.chunk).clone(); + if self.mark_observed_failures(&mut chunk, corrupt).await.is_err() { + return Err(error); + } + continue; + } + }; + if Instant::now() >= layout.deadline { + layout.invalidate(); + continue; + } + if observations + .iter() + .any(|observation| !observation.failed_segments.is_empty()) + { + layout.invalidate(); + let mut chunk = (*layout.chunk).clone(); + if self + .mark_observed_failures(&mut chunk, observations) + .await + .is_err() + { + continue; + } + } + return Ok(parsed); + } + Err(ReadError::LayoutExpired) + } +} + +fn decode_framed_partial( + physical: &PartialReadResult, + framed: FramedLocation, + selected_logical_start: u64, + requested: Range, + output_logical_start: u64, + flow_metrics: &ReadFlowMetrics, +) -> ReadResult { + let first_frame = selected_logical_start / framed.geometry.payload_capacity(); + let last_frame = (requested.end - 1) / framed.geometry.payload_capacity(); + let mut result = PartialReadResult::default(); + let mut magic = None; + for frame_index in first_frame..=last_frame { + let frame_logical_start = frame_index * framed.geometry.payload_capacity(); + let payload_len = + (framed.location.logical_length - frame_logical_start).min(framed.geometry.payload_capacity()); + let frame_start = framed + .location + .frame_offset + .checked_add(frame_index * MAX_FRAME_BYTES as u64) + .ok_or_else(|| ReadError::InvalidLocations("frame offset overflows".into()))?; + let frame_end = frame_start + .checked_add(framed.geometry.header_bytes() as u64 + payload_len + FRAME_FOOTER_BYTES as u64) + .ok_or_else(|| ReadError::InvalidLocations("frame end overflows".into()))?; + let wanted_start = requested.start.max(frame_logical_start); + let wanted_end = requested.end.min(frame_logical_start + payload_len); + let output_start = output_logical_start + wanted_start - requested.start; + let output_end = output_logical_start + wanted_end - requested.start; + if physical + .failures + .iter() + .any(|failure| failure.start < frame_end && failure.end > frame_start) + { + result.failures.push(FailedReadRange { + start: output_start, + end: output_end, + error: ReadError::DataLoss("frame bytes could not be reconstructed".into()), + }); + continue; + } + let views = extract_physical_views(&physical.ranges, frame_start..frame_end)?; + let parse_started = Instant::now(); + let parsed = if views.len() == 1 { + parse_frame(&views[0], framed.location.chunk_id) + .map(|frame| (frame.header, frame.physical_length)) + } else { + let slices: Vec<_> = views.iter().map(Bytes::as_ref).collect(); + parse_frame_views(&slices, framed.location.chunk_id) + .map(|frame| (frame.header, frame.physical_length)) + }; + flow_metrics + .frame_parse_wait_ns + .inc_by(u64::try_from(parse_started.elapsed().as_nanos()).unwrap_or(u64::MAX)); + let (header, physical_length) = + parsed.map_err(|error| ReadError::DataLoss(format!("invalid chunk frame: {error}")))?; + if !matches!(header.magic, FrameMagic::RepoSmallV2 | FrameMagic::RepoLargeV2) { + return Err(ReadError::DataLoss( + "framed location has unsupported frame kind".into(), + )); + } + if usize::from(header.payload_offset) != framed.geometry.header_bytes() { + return Err(ReadError::DataLoss( + "frame revision disagrees with location geometry".into(), + )); + } + match magic { + Some(previous) if previous != header.magic => { + return Err(ReadError::DataLoss("framed location mixes frame kinds".into())); + } + None => magic = Some(header.magic), + Some(_) => {} + } + if physical_length as u64 != frame_end - frame_start { + return Err(ReadError::DataLoss("frame length disagrees with location".into())); + } + let payload_start = usize::try_from(wanted_start - frame_logical_start) + .map_err(|_| ReadError::InvalidLocations("frame payload range overflows".into()))?; + let payload_end = usize::try_from(wanted_end - frame_logical_start) + .map_err(|_| ReadError::InvalidLocations("frame payload range overflows".into()))?; + let physical_payload_start = usize::from(header.payload_offset) + payload_start; + let physical_payload_end = usize::from(header.payload_offset) + payload_end; + let mut logical_cursor = output_start; + for view in slice_views(&views, physical_payload_start..physical_payload_end)? { + let view_end = logical_cursor + view.len() as u64; + result.ranges.push(ReadRangeData { + start: logical_cursor, + end: view_end, + data: view, + }); + logical_cursor = view_end; + } + if logical_cursor != output_end { + return Err(ReadError::InvalidLocations( + "frame payload views are incomplete".into(), + )); + } + } + Ok(result) +} + +fn extract_physical_views(ranges: &[ReadRangeData], wanted: Range) -> ReadResult> { + let first = ranges.partition_point(|range| range.end <= wanted.start); + let mut output = Vec::new(); + let mut cursor = wanted.start; + for range in &ranges[first..] { + if range.start >= wanted.end { + break; + } + if range.end <= cursor { + continue; + } + if range.start > cursor { + return Err(ReadError::InvalidLocations( + "physical frame data has a gap".into(), + )); + } + let end = wanted.end.min(range.end); + let offset = usize::try_from(cursor - range.start) + .map_err(|_| ReadError::InvalidLocations("physical frame offset overflows".into()))?; + let length = usize::try_from(end - cursor) + .map_err(|_| ReadError::InvalidLocations("physical frame length overflows".into()))?; + output.push(range.data.slice(offset..offset + length)); + cursor = end; + if cursor == wanted.end { + break; + } + } + if cursor != wanted.end { + return Err(ReadError::InvalidLocations( + "physical frame data is incomplete".into(), + )); + } + Ok(output) +} + +fn slice_views(views: &[Bytes], wanted: Range) -> ReadResult> { + let mut output = Vec::new(); + let mut cursor = 0_usize; + let mut covered = wanted.start; + for view in views { + let view_end = cursor.saturating_add(view.len()); + if view_end > covered && cursor < wanted.end { + let start = covered.saturating_sub(cursor); + let end = (wanted.end - cursor).min(view.len()); + output.push(view.slice(start..end)); + covered += end - start; + } + cursor = view_end; + if covered == wanted.end { + break; + } + } + if covered != wanted.end { + return Err(ReadError::InvalidLocations( + "frame payload views are incomplete".into(), + )); + } + Ok(output) +} diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout.rs new file mode 100644 index 000000000..51d2b6aac --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout.rs @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::KIB; +use crate::{ReadError, ReadResult}; +use crowdb_protocol::chunkdb::rpc::Chunk; + +pub(super) fn validate_strip_layout(chunk: &Chunk) -> ReadResult<()> { + let mut previous_end = 0; + for (index, strip) in chunk.strips.iter().enumerate() { + let start = u64::from(strip.chunk_offset) * KIB; + let end = strip_end(strip)?; + if strip.capacity == 0 || strip.unit_kb == 0 || end <= start || (index > 0 && start < previous_end) { + return Err(ReadError::InvalidLocations(format!( + "chunk strip {} has invalid or overlapping geometry", + strip.strip_sequence + ))); + } + previous_end = end; + } + Ok(()) +} + +pub(super) fn strip_end(strip: &crowdb_protocol::chunkdb::rpc::ChunkStrip) -> ReadResult { + u64::from(strip.chunk_offset) + .checked_add(u64::from(strip.capacity)) + .and_then(|kib| kib.checked_mul(KIB)) + .ok_or_else(|| ReadError::InvalidLocations("strip interval overflows".into())) +} + +pub(super) fn strip_read_parts( + strip: &crowdb_protocol::chunkdb::rpc::ChunkStrip, + start: u64, + end: u64, +) -> ReadResult> { + let Some(crowdb_protocol::chunkdb::rpc::Strip::EcStrip(ec)) = strip.strip.as_ref() else { + return Ok(vec![(start, end)]); + }; + let first = ec + .segments + .first() + .ok_or_else(|| ReadError::InvalidLocations("EC strip has no segments".into()))?; + let shard_bytes = u64::from(first.unit_count) + .checked_mul(u64::from(strip.unit_kb)) + .and_then(|value| value.checked_mul(KIB)) + .ok_or_else(|| ReadError::InvalidLocations("EC shard size overflows".into()))?; + if shard_bytes == 0 { + return Err(ReadError::InvalidLocations("EC shard size is zero".into())); + } + let strip_start = u64::from(strip.chunk_offset) * KIB; + let mut parts = Vec::new(); + let mut cursor = start; + while cursor < end { + let local = cursor - strip_start; + let next_boundary = strip_start.saturating_add( + (local / shard_bytes) + .saturating_add(1) + .saturating_mul(shard_bytes), + ); + let part_end = end.min(next_boundary); + parts.push((cursor, part_end)); + cursor = part_end; + } + Ok(parts) +} diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout_cache.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout_cache.rs new file mode 100644 index 000000000..2bb52272f --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/layout_cache.rs @@ -0,0 +1,89 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded frame-read reuse of the original server-granted layout deadline. + +use arc_swap::ArcSwapOption; +use crowdb_protocol::common::ChunkId; +use std::sync::Arc; + +use super::{ChunkReader, LayoutSnapshot, StripFailureObservation}; +use crate::ReadResult; + +const FRAME_LAYOUT_CACHE_SLOTS: usize = 64; + +pub(super) struct FrameLayoutCache { + slots: [ArcSwapOption; FRAME_LAYOUT_CACHE_SLOTS], +} + +impl Default for FrameLayoutCache { + fn default() -> Self { + Self { + slots: std::array::from_fn(|_| ArcSwapOption::empty()), + } + } +} + +impl FrameLayoutCache { + fn slot(&self, id: ChunkId) -> &ArcSwapOption { + let hash = id.high ^ id.low.rotate_left(17) ^ id.low; + &self.slots[(hash % FRAME_LAYOUT_CACHE_SLOTS as u64) as usize] + } + + fn get(&self, id: ChunkId, required_end: u64, synchronized: bool) -> Option { + self.slot(id) + .load_full() + .filter(|layout| { + layout.chunk_id == id + && layout.usable() + && if synchronized { + layout.chunk.strips.last().is_some_and(|strip| { + u64::from(strip.chunk_offset + strip.capacity) * 1024 >= required_end + }) + } else { + layout.chunk.acknowledged_cursor >= required_end + } + }) + .map(|layout| (*layout).clone()) + } + + fn insert(&self, layout: &LayoutSnapshot) { + self.slot(layout.chunk_id).store(Some(Arc::new(layout.clone()))); + } +} + +impl LayoutSnapshot { + pub(super) fn has_new_failures(&self, observations: &[StripFailureObservation]) -> bool { + observations.iter().any(|observation| { + !observation.failed_segments.is_empty() + && self + .chunk + .strips + .iter() + .find(|strip| strip.strip_sequence == observation.strip_sequence) + .map_or(true, |strip| { + observation + .failed_segments + .iter() + .any(|segment| !strip.unavailable_segments.contains(segment)) + }) + }) + } +} + +impl ChunkReader { + pub(super) async fn frame_layout( + &self, + id: ChunkId, + required_end: u64, + synchronized: bool, + ) -> ReadResult { + if let Some(layout) = self.frame_layouts.get(id, required_end, synchronized) { + self.flow_metrics.layout_cache_hits.inc(); + return Ok(layout); + } + let layout = self.query_layout(id).await?; + self.frame_layouts.insert(&layout); + Ok(layout) + } +} diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/streaming.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/streaming.rs new file mode 100644 index 000000000..7066ee425 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/streaming.rs @@ -0,0 +1,221 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::super::read_credit::{retain, ReadLease}; +use super::framing::object_layout; +use super::{ChunkReadStream, LayoutSnapshot, PartialReadResult, DEFAULT_STREAM_WINDOW}; +use crate::{ReadError, ReadResult}; +use bytes::Bytes; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::frame::{ChunkLocation, MAX_FRAME_BYTES}; +use std::sync::Arc; +use std::time::Instant; + +impl ChunkReadStream { + pub async fn next_chunk(&mut self) -> Option> { + loop { + if let Some(pending) = self.next_pending() { + return Some(pending); + } + if self.delivery_cursor >= self.end { + return None; + } + let budget = Arc::clone(&self.reader.stream_budget); + let _registration = budget.register(); + let notified = budget.wake.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if let Err(error) = self.fill_slots().await { + self.abort(error); + continue; + } + if let Some((start, end, result, lease)) = self.completed.remove(&self.delivery_cursor) { + self.load_unit(start, end, result, &lease); + continue; + } + if self.reads.is_empty() { + let started = Instant::now(); + notified.await; + self.reader + .flow_metrics + .stream_credit_wait_ns + .inc_by(u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX)); + continue; + } + match self.reads.join_next().await { + Some(Ok(completed)) => { + self.reader.flow_metrics.stream_units_completed.inc(); + if completed.0 != self.delivery_cursor { + self.reader.flow_metrics.stream_out_of_order.inc(); + } + self.completed.insert(completed.0, completed); + } + Some(Err(error)) => self.abort(ReadError::DiskIo(error.to_string())), + None => {} + } + } + } + + async fn fill_slots(&mut self) -> ReadResult<()> { + while self.fetch_cursor < self.end { + let (next, physical_bytes) = self.next_unit(self.fetch_cursor)?; + let Some(lease) = self.slots.try_reserve(&self.reader.stream_budget, physical_bytes) else { + self.reader.flow_metrics.stream_credit_stalls.inc(); + break; + }; + let start = self.fetch_cursor; + let index = self.locations.partition_point(|location| { + location.logical_offset.saturating_add(location.logical_length) <= start + }); + let location = self.locations[index].clone(); + let layout = self + .layout_for( + location + .chunk_id + .ok_or_else(|| ReadError::InvalidLocations("location has no chunk ID".into()))?, + ) + .await?; + let reader = self.reader.clone(); + self.reader.flow_metrics.stream_windows.inc(); + self.reader.flow_metrics.range_locations_examined.inc(); + self.reads.spawn(async move { + let local_start = start - location.logical_offset; + let result = reader + .read_location_partial_cached(&location, local_start, next - start, start, Some(layout)) + .await; + (start, next, result, lease) + }); + self.fetch_cursor = next; + } + Ok(()) + } + + async fn layout_for(&mut self, chunk_id: ChunkId) -> ReadResult { + if let Some(layout) = self + .layouts + .iter() + .find(|layout| layout.chunk_id == chunk_id && layout.usable()) + { + return Ok(layout.clone()); + } + let reader = self.reader.clone(); + let layout = tokio::spawn(async move { reader.query_layout(chunk_id).await }) + .await + .map_err(|error| ReadError::Metadata(error.to_string()))??; + self.layouts.retain(|cached| cached.chunk_id != chunk_id); + if self.layouts.len() == 8 { + self.layouts.remove(0); + } + self.layouts.push(layout.clone()); + Ok(layout) + } + + fn next_unit(&self, start: u64) -> ReadResult<(u64, usize)> { + let first = self.locations.partition_point(|location| { + location.logical_offset.saturating_add(location.logical_length) <= start + }); + let location = self + .locations + .get(first) + .ok_or_else(|| ReadError::InvalidLocations("stream has no location for requested byte".into()))?; + let local = start - location.logical_offset; + let window = self.window_bytes.min(DEFAULT_STREAM_WINDOW as u64); + if location.length == location.logical_length { + let length = window.min(location.logical_length - local).min(self.end - start); + return Ok((start + length, length as usize)); + } + let frames = (window / MAX_FRAME_BYTES as u64).max(1); + let geometry = object_layout(location)?; + let capacity = geometry.payload_capacity(); + let frame_index = local / capacity; + let local_end = ((frame_index + frames) * capacity) + .min(location.logical_length) + .min(self.end - location.logical_offset); + let frame_location = ChunkLocation { + chunk_id: location + .chunk_id + .ok_or_else(|| ReadError::InvalidLocations("location has no chunk ID".into()))?, + frame_offset: location.offset, + logical_length: location.logical_length, + }; + let physical = geometry + .physical_range(frame_location, local..local_end) + .map_err(|error| ReadError::InvalidLocations(error.to_string()))?; + let bytes = usize::try_from(physical.end - physical.start) + .map_err(|_| ReadError::InvalidLocations("stream unit exceeds address space".into()))?; + Ok((location.logical_offset + local_end, bytes)) + } + + fn load_unit( + &mut self, + start: u64, + end: u64, + result: ReadResult, + lease: &Arc, + ) { + self.delivery_cursor = end; + let partial = match result { + Ok(partial) => partial, + Err(error) => { + self.abort(error); + return; + } + }; + let failure = partial.failures.into_iter().next(); + let verified_end = failure.as_ref().map_or(end, |failure| failure.start); + let mut ranges = partial.ranges; + let prefix_len = ranges.partition_point(|range| range.end <= verified_end); + let mut cursor = start; + for range in &ranges[..prefix_len] { + if range.start != cursor || range.end - range.start != range.data.len() as u64 { + self.abort(ReadError::InvalidLocations(format!( + "read ranges are not contiguous at byte {cursor}" + ))); + return; + } + cursor = range.end; + } + if cursor != verified_end { + self.abort(ReadError::InvalidLocations(format!( + "read ranges are not contiguous at byte {cursor}" + ))); + return; + } + ranges.truncate(prefix_len); + for range in &mut ranges { + range.data = retain(std::mem::take(&mut range.data), Arc::clone(lease)); + } + self.pending_ranges = ranges.into_iter(); + if let Some(failure) = failure { + self.reads.abort_all(); + self.completed.clear(); + self.fetch_cursor = self.end; + self.delivery_cursor = self.end; + self.pending_error = Some(ReadError::FailedRange { + start: failure.start, + end: failure.end, + message: failure.error.to_string(), + }); + } + } + + fn abort(&mut self, error: ReadError) { + self.reads.abort_all(); + self.completed.clear(); + self.fetch_cursor = self.end; + self.delivery_cursor = self.end; + self.pending_error = Some(error); + } + + fn next_pending(&mut self) -> Option> { + if let Some(range) = self.pending_ranges.next() { + self.cursor = range.end; + return Some(Ok(range.data)); + } + if let Some(error) = self.pending_error.take() { + self.cursor = self.end; + return Some(Err(error)); + } + None + } +} diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_reader/verified.rs b/lib/crowdb-chunk-client/src/chunk/chunk_reader/verified.rs new file mode 100644 index 000000000..d9a717253 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/chunk_reader/verified.rs @@ -0,0 +1,263 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Verified frame reads under a persisted or owner-synchronized prefix. + +use super::{ + extract_physical_range, mark_served_segments_corrupt, Bytes, ChunkId, ChunkReader, ChunkState, Instant, + ReadError, ReadResult, Segment, +}; +use crowdb_protocol::frame::{ + parse_frame, FrameMagic, FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, MAX_FRAME_BYTES, +}; + +impl ChunkReader { + /// Read and verify a frame under the ordinary persisted chunk boundary. + /// # Errors + /// Returns invalid geometry, unavailable IO or failed frame verification. + pub async fn read_verified_frame( + &self, + chunk_id: ChunkId, + offset: u64, + length: u64, + magic: FrameMagic, + ) -> ReadResult { + self.read_verified_frame_bounded(chunk_id, offset, length, magic, None) + .await + } + + /// Read a frame from an already synchronized Stream/Wal prefix supplied by + /// its owning stream. This bound is local to this call and never enters the + /// ordinary reader's layout cache or persisted metadata. + /// # Errors + /// Rejects a non-stream chunk, an invalid bound or failed frame verification. + pub async fn read_synchronized_stream_frame( + &self, + chunk_id: ChunkId, + offset: u64, + length: u64, + synchronized_end: u64, + ) -> ReadResult { + use crowdb_protocol::chunk_stream::StreamPurpose; + if !StreamPurpose::Stream.matches(chunk_id) && !StreamPurpose::Wal.matches(chunk_id) { + return Err(ReadError::InvalidLocations( + "local stream boundary has a different chunk purpose".into(), + )); + } + self.read_verified_frame_bounded( + chunk_id, + offset, + length, + FrameMagic::StreamV2, + Some(synchronized_end), + ) + .await + } + + #[allow( + clippy::too_many_lines, + reason = "layout acquisition, validation, failure observation, and retries share one frame-read state machine" + )] + async fn read_verified_frame_bounded( + &self, + chunk_id: ChunkId, + frame_offset: u64, + frame_length: u64, + expected_magic: FrameMagic, + synchronized_end: Option, + ) -> ReadResult { + if frame_length < (FRAME_HEADER_PREFIX_BYTES + FRAME_FOOTER_BYTES) as u64 + || frame_length > MAX_FRAME_BYTES as u64 + { + return Err(ReadError::InvalidLocations( + "frame length is outside the public frame bounds".into(), + )); + } + let frame_end = frame_offset + .checked_add(frame_length) + .ok_or_else(|| ReadError::InvalidLocations("frame end overflows".into()))?; + for attempt in 1..=self.policy.max_layout_retries { + let query_started = Instant::now(); + let layout = match self + .frame_layout(chunk_id, frame_end, synchronized_end.is_some()) + .await + { + Ok(layout) => layout, + Err(error) => { + tracing::warn!( + chunk_high = chunk_id.high, + chunk_low = chunk_id.low, + frame_offset, + frame_length, + attempt, + error = %error, + "verified frame layout query failed" + ); + return Err(error); + } + }; + let deadline = layout.deadline; + let layout_validity_ms = deadline.saturating_duration_since(Instant::now()).as_millis(); + let mut chunk = (*layout.chunk).clone(); + if let Some(end) = synchronized_end { + if frame_end > end + || (chunk.state == ChunkState::Sealed as i32 && end > chunk.acknowledged_cursor) + { + return Err(ReadError::InvalidLocations( + "frame exceeds synchronized stream prefix".into(), + )); + } + chunk.acknowledged_cursor = chunk.acknowledged_cursor.max(end); + } + tracing::debug!( + chunk_high = chunk_id.high, + chunk_low = chunk_id.low, + frame_offset, + frame_length, + attempt, + layout_validity_ms, + strip_count = chunk.strips.len(), + "verified frame layout acquired" + ); + let (physical, observations) = match self + .read_chunk_range_partial(&chunk, frame_offset, frame_length, frame_offset) + .await + { + Ok(result) => result, + Err(error) => { + layout.invalidate(); + tracing::warn!( + chunk_high = chunk_id.high, + chunk_low = chunk_id.low, + frame_offset, + frame_length, + attempt, + error = %error, + "verified frame physical read failed" + ); + return Err(error); + } + }; + let bytes = match extract_physical_range(&physical.ranges, frame_offset..frame_end) { + Ok(bytes) if physical.failures.is_empty() => bytes, + Ok(_) | Err(_) => { + layout.invalidate(); + let failed_start = physical.failures.first().map(|failure| failure.start); + let failed_end = physical.failures.first().map(|failure| failure.end); + let failed_error = physical.failures.first().map(|failure| failure.error.to_string()); + let expired = Instant::now() >= deadline; + let observation_error = if expired { + None + } else { + self.mark_observed_failures(&mut chunk, observations).await.err() + }; + tracing::warn!( + chunk_high = chunk_id.high, + chunk_low = chunk_id.low, + frame_offset, + frame_length, + attempt, + completed_ranges = physical.ranges.len(), + failed_ranges = physical.failures.len(), + failed_start = ?failed_start, + failed_end = ?failed_end, + failed_error = ?failed_error, + expired, + observation_error = ?observation_error, + "verified frame read did not produce a complete range" + ); + if expired || observation_error.is_some() { + return Err(ReadError::DataLoss( + "frame bytes could not be reconstructed".into(), + )); + } + continue; + } + }; + let decode_started = Instant::now(); + let parsed_frame = parse_frame(&bytes, chunk_id); + let parse_ns = u64::try_from(decode_started.elapsed().as_nanos()).unwrap_or(u64::MAX); + self.flow_metrics.frame_decode_wait_ns.inc_by(parse_ns); + self.flow_metrics.frame_parse_wait_ns.inc_by(parse_ns); + match parsed_frame { + Ok(frame) if frame.header.magic == expected_magic => { + if !layout.usable() { + layout.invalidate(); + tracing::warn!( + chunk_high = chunk_id.high, + chunk_low = chunk_id.low, + frame_offset, + frame_length, + attempt, + elapsed_ms = query_started.elapsed().as_millis(), + layout_validity_ms, + "verified frame read exceeded layout validity" + ); + continue; + } + if layout.has_new_failures(&observations) { + layout.invalidate(); + self.mark_observed_failures(&mut chunk, observations).await?; + } + return Ok(bytes); + } + Ok(frame) => { + layout.invalidate(); + tracing::warn!( + chunk_high = chunk_id.high, + chunk_low = chunk_id.low, + frame_offset, + frame_length, + attempt, + expected_magic = ?expected_magic, + actual_magic = ?frame.header.magic, + "verified frame kind disagrees with its location" + ); + return Err(ReadError::DataLoss( + "frame kind disagrees with its location".into(), + )); + } + Err(error) => { + layout.invalidate(); + let served_segments: Vec = observations + .iter() + .flat_map(|observation| observation.served_segments.iter().copied()) + .collect(); + let corrupt = mark_served_segments_corrupt(observations); + let corrupt_segments = corrupt.len(); + let expired = Instant::now() >= deadline; + let observation_error = if corrupt_segments == 0 || expired { + None + } else { + self.mark_observed_failures(&mut chunk, corrupt).await.err() + }; + tracing::warn!( + chunk_high = chunk_id.high, + chunk_low = chunk_id.low, + frame_offset, + frame_length, + attempt, + error = %error, + corrupt_segments, + served_segments = ?served_segments, + expired, + observation_error = ?observation_error, + "verified frame parsing failed" + ); + if corrupt_segments == 0 || expired || observation_error.is_some() { + return Err(ReadError::DataLoss(format!("invalid chunk frame: {error}"))); + } + } + } + } + tracing::warn!( + chunk_high = chunk_id.high, + chunk_low = chunk_id.low, + frame_offset, + frame_length, + attempts = self.policy.max_layout_retries, + "verified frame read exhausted valid chunk layouts" + ); + Err(ReadError::LayoutExpired) + } +} diff --git a/lib/crowdb-chunk-client/src/chunk/chunk_writer.rs b/lib/crowdb-chunk-client/src/chunk/chunk_writer.rs index a30c6a96b..08f278605 100644 --- a/lib/crowdb-chunk-client/src/chunk/chunk_writer.rs +++ b/lib/crowdb-chunk-client/src/chunk/chunk_writer.rs @@ -792,11 +792,9 @@ impl ChunkWriter { self.finish_completion(handle).await?; } self.completion_wait_time += wait_started.elapsed(); - let sealed_length_kb = bytes_in_chunk.div_ceil(1024) as u32; self.allocator .seal_chunk(SealChunkRequest { chunk_id: Some(cid), - seal_length: sealed_length_kb, seal_bytes: bytes_in_chunk, }) .await?; diff --git a/lib/crowdb-chunk-client/src/chunk/frame_observation.rs b/lib/crowdb-chunk-client/src/chunk/frame_observation.rs new file mode 100644 index 000000000..1981df6e8 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/frame_observation.rs @@ -0,0 +1,96 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Physical prefix observations for an exclusively owned/excluded last chunk. +//! Ordinary object reads retain their acknowledged-cursor bound. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_protocol::frame::{FrameScanConfig, FrameScanError, FrameScanSummary, ScannedFrameExtent}; +use thiserror::Error; + +use crate::IoError; + +mod mirrored; +mod prefix; +pub use mirrored::MirroredFrameReaders; +mod replicas; +pub use prefix::{observe_frame_prefix, replay_frame_prefix}; +pub use replicas::{select_frame_prefix, FrameReplicaSelection}; + +pub const MAX_FRAME_OBSERVATION_REPLICAS: usize = 5; + +/// Supplies exact physical windows from one stable layout observation. Callers +/// establish owner exclusion and layout/read deadlines before constructing it. +/// IO failures and short reads must not be replaced with fabricated zero bytes. +#[async_trait] +pub trait FrameWindowReader: Send + Sync { + async fn read_window(&self, offset: u64, length: u32) -> crate::Result; +} + +/// Receives bounded mapping batches only during replay of a validated source. +/// A failed replay must never publish its partial mappings as a stable root. +#[async_trait] +pub trait FrameExtentSink: Send { + async fn append_extents(&mut self, extents: &[ScannedFrameExtent]) -> crate::Result<()>; +} + +#[derive(Clone, Copy, Debug)] +pub struct FrameObservation { + config: FrameScanConfig, + window_bytes: u32, + summary: FrameScanSummary, + reads: u64, + read_bytes: u64, +} + +impl FrameObservation { + #[must_use] + pub const fn summary(&self) -> &FrameScanSummary { + &self.summary + } + #[must_use] + pub const fn reads(&self) -> u64 { + self.reads + } + #[must_use] + pub const fn read_bytes(&self) -> u64 { + self.read_bytes + } +} + +#[derive(Debug, Error)] +pub enum FrameObservationError { + #[error("invalid physical frame layout: {0}")] + InvalidLayout(String), + #[error("invalid physical prefix observation bounds or replica budget")] + InvalidRequest, + #[error("physical prefix observation deadline expired")] + DeadlineExpired, + #[error("physical read at {offset} failed: {source}")] + Read { offset: u64, source: IoError }, + #[error("short physical read at {offset}: expected {expected} bytes, received {actual}")] + ShortRead { + offset: u64, + expected: u32, + actual: usize, + }, + #[error(transparent)] + InvalidPrefix(#[from] FrameScanError), + #[error("mapping sink failed: {0}")] + Sink(IoError), + #[error("no healthy completed replica observation: {failures:?}")] + NoHealthyReplica { failures: Vec }, + #[error("valid replicas {first} and {second} conflict through complete boundary {physical_end}")] + ConflictingPrefixes { + first: usize, + second: usize, + physical_end: u64, + }, + #[error("replica observation task failed: {0}")] + Task(String), + #[error("replayed prefix differs from its validated observation")] + ChangedPrefix, +} + +pub type FrameObservationResult = Result; diff --git a/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs new file mode 100644 index 000000000..22fb4183a --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored.rs @@ -0,0 +1,171 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Fixed current-layout physical readers for an excluded Stream/Wal owner. + +use crowdb_protocol::chunk_stream::{StreamName, StreamPurpose}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, Strip}; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::frame::{FrameScanConfig, FrameScanMode}; +use std::sync::Arc; +use tokio::time::Instant; + +use super::{ + FrameObservationError, FrameObservationResult, FrameWindowReader, MAX_FRAME_OBSERVATION_REPLICAS, +}; +use crate::{DiskWriter, STREAM_CHUNK_BYTES}; + +mod read; +use read::MirroredWindowReader; + +struct MirroredLayout { + chunk: Arc, + disk_io: Arc, + chunk_id: ChunkId, + capacity: u64, + deadline: Instant, +} + +/// Owns one immutable layout snapshot. Construction requires caller-proven owner +/// exclusion; clones of its readers retain the same conservative local deadline. +pub struct MirroredFrameReaders { + layout: Arc, + readers: Vec>, +} + +impl MirroredFrameReaders { + pub fn new( + chunk: Arc, + stream_name: StreamName, + disk_io: Arc, + deadline: Instant, + ) -> FrameObservationResult { + if deadline <= Instant::now() { + return Err(FrameObservationError::DeadlineExpired); + } + let chunk_id = chunk.id.ok_or_else(|| invalid_layout("chunk has no identity"))?; + let purpose = if StreamPurpose::Wal.matches(chunk_id) { + StreamPurpose::Wal + } else if StreamPurpose::Stream.matches(chunk_id) { + StreamPurpose::Stream + } else { + return Err(invalid_layout("physical tail reader requires a Stream/Wal chunk")); + }; + if chunk.chunk_type != purpose.chunk_type() as i32 + || chunk.owner_key != stream_name.chunk_owner_key() + || chunk.writer_epoch == 0 + || ![ChunkState::Active as i32, ChunkState::Sealed as i32].contains(&chunk.state) + { + return Err(invalid_layout("chunk purpose, owner, epoch or state is invalid")); + } + let (capacity, replicas) = validate_layout(&chunk, chunk_id)?; + let layout = Arc::new(MirroredLayout { + chunk, + disk_io, + chunk_id, + capacity, + deadline, + }); + let readers = (0..replicas) + .map(|replica| { + Arc::new(MirroredWindowReader { + layout: Arc::clone(&layout), + replica, + }) as Arc + }) + .collect(); + Ok(Self { layout, readers }) + } + + #[must_use] + pub fn readers(&self) -> &[Arc] { + &self.readers + } + #[must_use] + pub fn capacity(&self) -> u64 { + self.layout.capacity + } + #[must_use] + pub fn deadline(&self) -> Instant { + self.layout.deadline + } + + pub fn scan_config( + &self, + physical_start: u64, + logical_start: u64, + required_end: u64, + max_append_bytes: u64, + ) -> FrameObservationResult { + if physical_start > required_end || required_end > self.layout.capacity || max_append_bytes == 0 { + return Err(FrameObservationError::InvalidRequest); + } + Ok(FrameScanConfig { + chunk_id: self.layout.chunk_id, + mode: FrameScanMode::Grouped, + physical_start, + physical_limit: self.layout.capacity, + required_end, + logical_start, + max_append_bytes, + }) + } +} + +fn validate_layout(chunk: &Chunk, chunk_id: ChunkId) -> FrameObservationResult<(u64, usize)> { + if chunk.strips.is_empty() { + return Err(invalid_layout("chunk has no allocated strips")); + } + let mut end = 0_u64; + let mut replicas = 0; + for strip in &chunk.strips { + let start = u64::from(strip.chunk_offset) * 1024; + let length = u64::from(strip.capacity) * 1024; + let unit_bytes = u64::from(strip.unit_kb) * 1024; + if start != end || length == 0 || unit_bytes == 0 || start + length > STREAM_CHUNK_BYTES { + return Err(invalid_layout( + "chunk strips have gaps, overlaps or invalid capacity", + )); + } + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(invalid_layout("tail layout is not mirrored")); + }; + if mirror.segments.is_empty() || mirror.segments.len() > MAX_FRAME_OBSERVATION_REPLICAS { + return Err(invalid_layout("mirror replica count is invalid")); + } + for (index, segment) in mirror.segments.iter().enumerate() { + if segment.disk_id.is_none() + || segment.owner_chunk != Some(chunk_id) + || segment.allocation_ts == 0 + || u64::from(segment.unit_count) + .checked_mul(unit_bytes) + .map_or(true, |capacity| capacity < length) + || segment + .unit_offset + .checked_add(u64::from(segment.unit_count)) + .and_then(|end| end.checked_mul(unit_bytes)) + .is_none() + || mirror.segments[..index].iter().any(|previous| { + previous.disk_id == segment.disk_id + && previous.zone_index == segment.zone_index + && previous.unit_offset < segment.unit_offset + u64::from(segment.unit_count) + && segment.unit_offset < previous.unit_offset + u64::from(previous.unit_count) + }) + { + return Err(invalid_layout( + "mirror allocation identity or capacity is invalid", + )); + } + } + replicas = replicas.max(mirror.segments.len()); + end = start + length; + } + if chunk.acknowledged_cursor > end { + return Err(invalid_layout("checkpoint exceeds allocated layout")); + } + Ok((end, replicas)) +} + +fn invalid_layout(message: &str) -> FrameObservationError { + FrameObservationError::InvalidLayout(message.into()) +} diff --git a/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/read.rs b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/read.rs new file mode 100644 index 000000000..1d7b0d124 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/frame_observation/mirrored/read.rs @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::{Bytes, BytesMut}; +use crowdb_protocol::{chunkdb::rpc::Strip, frame::MAX_FRAME_SCAN_INPUT_BYTES}; +use std::sync::Arc; +use tokio::time::{timeout_at, Instant}; + +use super::MirroredLayout; +use crate::{FrameWindowReader, IoError, Result}; + +pub(super) struct MirroredWindowReader { + pub(super) layout: Arc, + pub(super) replica: usize, +} + +#[async_trait] +impl FrameWindowReader for MirroredWindowReader { + async fn read_window(&self, offset: u64, length: u32) -> Result { + self.check_deadline()?; + let end = offset + .checked_add(u64::from(length)) + .ok_or_else(|| invalid("physical range overflows"))?; + if length == 0 || length as usize > MAX_FRAME_SCAN_INPUT_BYTES || end > self.layout.capacity { + return Err(invalid("physical window exceeds its allocated scope")); + } + let mut index = self + .layout + .chunk + .strips + .partition_point(|strip| u64::from(strip.chunk_offset) * 1024 <= offset) + .checked_sub(1) + .ok_or_else(|| invalid("window precedes allocated strips"))?; + let mut cursor = offset; + let mut output: Option = None; + while cursor < end { + self.check_deadline()?; + let strip = self + .layout + .chunk + .strips + .get(index) + .ok_or_else(|| invalid("window exceeds allocated strips"))?; + let start = u64::from(strip.chunk_offset) * 1024; + let strip_end = start + u64::from(strip.capacity) * 1024; + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(invalid("window has no mirror layout")); + }; + let segment = mirror + .segments + .get(self.replica) + .ok_or_else(|| invalid("replica is absent from an allocated strip"))?; + if strip.unavailable_segments.contains(segment) { + return Err(invalid("replica is marked unavailable in this layout")); + } + let take = u32::try_from((end.min(strip_end)) - cursor).expect("bounded window"); + let data = timeout_at( + self.layout.deadline, + self.layout + .disk_io + .read(segment, u64::from(strip.unit_kb) * 1024, cursor - start, take), + ) + .await + .map_err(|_| invalid("physical layout read deadline expired"))??; + self.check_deadline()?; + if data.len() != take as usize { + return Err(invalid("short physical segment read")); + } + if cursor == offset && take == length { + return Ok(data); + } + output + .get_or_insert_with(|| BytesMut::with_capacity(length as usize)) + .extend_from_slice(&data); + cursor += u64::from(take); + index += 1; + } + Ok(output.expect("cross-strip window has data").freeze()) + } +} + +impl MirroredWindowReader { + fn check_deadline(&self) -> Result<()> { + if Instant::now() >= self.layout.deadline { + Err(invalid("physical layout read deadline expired")) + } else { + Ok(()) + } + } +} + +fn invalid(message: &str) -> IoError { + IoError::ReadFailed(message.into()) +} diff --git a/lib/crowdb-chunk-client/src/chunk/frame_observation/prefix.rs b/lib/crowdb-chunk-client/src/chunk/frame_observation/prefix.rs new file mode 100644 index 000000000..07a7f7354 --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/frame_observation/prefix.rs @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::frame::{ + FramePrefixScanner, FrameScanConfig, FrameScanStop, MAX_FRAME_SCAN_EXTENTS, MAX_FRAME_SCAN_INPUT_BYTES, +}; +use tokio::time::{timeout_at, Instant}; + +use super::{ + FrameExtentSink, FrameObservation, FrameObservationError, FrameObservationResult, FrameWindowReader, +}; + +/// Observes one complete physical prefix within the caller's original deadline. +pub async fn observe_frame_prefix( + reader: &dyn FrameWindowReader, + config: FrameScanConfig, + window_bytes: u32, + deadline: Instant, +) -> FrameObservationResult { + scan_prefix(reader, config, window_bytes, deadline, None).await +} + +/// Rebuilds bounded mappings from an observed source and verifies the exact +/// complete-prefix digest before returning success. The original deadline applies. +pub async fn replay_frame_prefix( + reader: &dyn FrameWindowReader, + expected: &FrameObservation, + deadline: Instant, + sink: &mut dyn FrameExtentSink, +) -> FrameObservationResult { + let mut config = expected.config; + config.physical_limit = expected.summary.physical_end; + config.required_end = config.physical_limit; + let actual = scan_prefix(reader, config, expected.window_bytes, deadline, Some(sink)).await?; + if actual.summary.physical_end != expected.summary.physical_end + || actual.summary.logical_end != expected.summary.logical_end + || actual.summary.digest != expected.summary.digest + { + return Err(FrameObservationError::ChangedPrefix); + } + Ok(actual) +} + +async fn scan_prefix( + reader: &dyn FrameWindowReader, + config: FrameScanConfig, + window_bytes: u32, + deadline: Instant, + mut sink: Option<&mut dyn FrameExtentSink>, +) -> FrameObservationResult { + if window_bytes == 0 || window_bytes as usize > MAX_FRAME_SCAN_INPUT_BYTES { + return Err(FrameObservationError::InvalidRequest); + } + let mut scanner = FramePrefixScanner::new(config)?; + let mut offset = config.physical_start; + let mut reads = 0; + let mut read_bytes = 0; + while offset < config.physical_limit && scanner.stop() == FrameScanStop::Running { + check_deadline(deadline)?; + let length = u32::try_from((config.physical_limit - offset).min(u64::from(window_bytes))) + .expect("bounded window"); + let bytes = timeout_at(deadline, reader.read_window(offset, length)) + .await + .map_err(|_| FrameObservationError::DeadlineExpired)? + .map_err(|source| FrameObservationError::Read { offset, source })?; + reads += 1; + read_bytes += bytes.len() as u64; + if bytes.len() != length as usize { + return Err(FrameObservationError::ShortRead { + offset, + expected: length, + actual: bytes.len(), + }); + } + let mut consumed = 0; + while consumed < bytes.len() && scanner.stop() == FrameScanStop::Running { + check_deadline(deadline)?; + let batch = scanner.push( + offset + consumed as u64, + &bytes[consumed..], + MAX_FRAME_SCAN_EXTENTS, + )?; + if let Some(sink) = &mut sink { + if !batch.extents.is_empty() { + timeout_at(deadline, sink.append_extents(&batch.extents)) + .await + .map_err(|_| FrameObservationError::DeadlineExpired)? + .map_err(FrameObservationError::Sink)?; + } + } + consumed += batch.consumed; + } + offset += consumed as u64; + tokio::task::yield_now().await; + } + check_deadline(deadline)?; + Ok(FrameObservation { + config, + window_bytes, + summary: scanner.finish()?, + reads, + read_bytes, + }) +} + +fn check_deadline(deadline: Instant) -> FrameObservationResult<()> { + if Instant::now() >= deadline { + Err(FrameObservationError::DeadlineExpired) + } else { + Ok(()) + } +} diff --git a/lib/crowdb-chunk-client/src/chunk/frame_observation/replicas.rs b/lib/crowdb-chunk-client/src/chunk/frame_observation/replicas.rs new file mode 100644 index 000000000..ba1aeed1e --- /dev/null +++ b/lib/crowdb-chunk-client/src/chunk/frame_observation/replicas.rs @@ -0,0 +1,149 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::frame::FrameScanConfig; +use std::sync::Arc; +use tokio::{task::JoinSet, time::Instant}; + +use super::{ + observe_frame_prefix, FrameObservation, FrameObservationError, FrameObservationResult, FrameWindowReader, + MAX_FRAME_OBSERVATION_REPLICAS, +}; + +#[derive(Debug)] +pub struct FrameReplicaSelection { + selected_index: usize, + selected: FrameObservation, + healthy_replicas: usize, + failures: Vec, +} + +impl FrameReplicaSelection { + #[must_use] + pub const fn selected_index(&self) -> usize { + self.selected_index + } + #[must_use] + pub const fn observation(&self) -> &FrameObservation { + &self.selected + } + #[must_use] + pub const fn healthy_replicas(&self) -> usize { + self.healthy_replicas + } + #[must_use] + pub fn failures(&self) -> &[String] { + &self.failures + } +} + +/// Observes available replicas concurrently, discards incomplete/failed +/// observations, and compares their shortest complete prefix without arbitration. +/// The first half of the remaining budget bounds observation; comparison ends +/// after three quarters, reserving the rest for replay under the same deadline. +pub async fn select_frame_prefix( + readers: &[Arc], + config: FrameScanConfig, + window_bytes: u32, + deadline: Instant, +) -> FrameObservationResult { + if readers.is_empty() || readers.len() > MAX_FRAME_OBSERVATION_REPLICAS { + return Err(FrameObservationError::InvalidRequest); + } + let start = Instant::now(); + if start >= deadline { + return Err(FrameObservationError::DeadlineExpired); + } + let remaining = deadline - start; + let observation_deadline = start + remaining / 2; + let comparison_deadline = deadline - remaining / 4; + let mut tasks = JoinSet::new(); + for (index, reader) in readers.iter().enumerate() { + let reader = Arc::clone(reader); + tasks.spawn(async move { + ( + index, + observe_frame_prefix(reader.as_ref(), config, window_bytes, observation_deadline).await, + ) + }); + } + let mut healthy = Vec::new(); + let mut failures = Vec::new(); + while let Some(result) = tasks.join_next().await { + let (index, observation) = result.map_err(|error| FrameObservationError::Task(error.to_string()))?; + match observation { + Ok(observation) => healthy.push((index, observation)), + Err(error) => failures.push(format!("replica {index}: {error}")), + } + } + let Some((selected_index, selected)) = healthy + .iter() + .min_by_key(|(index, observation)| (observation.summary.physical_end, *index)) + .copied() + else { + return Err(FrameObservationError::NoHealthyReplica { failures }); + }; + let comparison_failures = + compare_prefixes(readers, &healthy, selected_index, selected, comparison_deadline).await?; + let healthy_replicas = healthy.len() - comparison_failures.len(); + failures.extend(comparison_failures); + Ok(FrameReplicaSelection { + selected_index, + selected, + healthy_replicas, + failures, + }) +} + +async fn compare_prefixes( + readers: &[Arc], + healthy: &[(usize, FrameObservation)], + selected_index: usize, + selected: FrameObservation, + deadline: Instant, +) -> FrameObservationResult> { + let mut tasks = JoinSet::new(); + for &(index, observation) in healthy { + if observation.summary.physical_end == selected.summary.physical_end { + check_match(selected_index, selected, index, observation)?; + continue; + } + let reader = Arc::clone(&readers[index]); + let mut config = observation.config; + config.physical_limit = selected.summary.physical_end; + config.required_end = config.physical_limit; + tasks.spawn(async move { + ( + index, + observe_frame_prefix(reader.as_ref(), config, observation.window_bytes, deadline).await, + ) + }); + } + let mut failures = Vec::new(); + while let Some(result) = tasks.join_next().await { + let (index, observation) = result.map_err(|error| FrameObservationError::Task(error.to_string()))?; + match observation { + Ok(observation) => check_match(selected_index, selected, index, observation)?, + Err(error) => failures.push(format!("replica {index} comparison: {error}")), + } + } + Ok(failures) +} + +fn check_match( + first: usize, + expected: FrameObservation, + second: usize, + actual: FrameObservation, +) -> FrameObservationResult<()> { + if expected.summary.digest != actual.summary.digest + || expected.summary.logical_end != actual.summary.logical_end + { + return Err(FrameObservationError::ConflictingPrefixes { + first, + second, + physical_end: expected.summary.physical_end, + }); + } + Ok(()) +} diff --git a/lib/crowdb-chunk-client/src/chunk/mirror_chunk_writer.rs b/lib/crowdb-chunk-client/src/chunk/mirror_chunk_writer.rs deleted file mode 100644 index 4d74e109a..000000000 --- a/lib/crowdb-chunk-client/src/chunk/mirror_chunk_writer.rs +++ /dev/null @@ -1,440 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Direct, one-chunk mirror writer for ordered byte streams. - -use std::sync::Arc; - -use bytes::Bytes; -use crowdb_protocol::chunk_stream::{StreamName, StreamPurpose}; -use crowdb_protocol::chunkdb::rpc::{ - AdvanceChunkWriteRequest, AllocateChunkRequest, AppendChunkRequest, Chunk, ChunkState, SealChunkRequest, - Strip, StripType, -}; -use crowdb_protocol::common::ChunkId; -use tokio::task::JoinSet; - -use crate::{ChunkAllocator, DiskWriter, IoError, Result}; - -/// Default hard logical capacity of one stream chunk. -pub const STREAM_CHUNK_BYTES: u64 = 256 * 1024 * 1024; -/// Allocation granularity for a growing stream chunk. -pub const STREAM_STRIP_BYTES: u64 = 1024 * 1024; - -/// A single-owner direct writer that replicates each append to every mirror -/// before advancing chunkdb's fenced acknowledged cursor. -pub struct MirrorChunkWriter { - allocator: Arc, - disk_writer: Arc, - chunk: Chunk, - chunk_id: ChunkId, - writer_epoch: u64, - writer_lease_ms: u64, - capacity: u64, -} - -impl MirrorChunkWriter { - /// Allocates one stream chunk with the default mirror policy. - /// - /// # Errors - /// - /// Returns an allocation or layout error when chunkdb cannot supply an - /// active mirror strip. - pub async fn allocate( - allocator: Arc, - disk_writer: Arc, - stream_name: StreamName, - writer_epoch: u64, - writer_lease_ms: u64, - purpose: StreamPurpose, - ) -> Result { - Self::allocate_with_copy_count( - allocator, - disk_writer, - stream_name, - writer_epoch, - writer_lease_ms, - 2, - purpose, - ) - .await - } - - /// Allocates one stream chunk with an explicit mirror count. - /// - /// # Errors - /// - /// Returns an allocation or layout error for a zero count or unavailable placement. - pub async fn allocate_with_copy_count( - allocator: Arc, - disk_writer: Arc, - stream_name: StreamName, - writer_epoch: u64, - writer_lease_ms: u64, - copy_count: u32, - purpose: StreamPurpose, - ) -> Result { - if writer_epoch == 0 || writer_lease_ms == 0 || !(1..=5).contains(©_count) { - return Err(IoError::AllocationFailed( - "stream mirror writer requires a nonzero epoch and lease".into(), - )); - } - let response = allocator - .allocate_chunk(AllocateChunkRequest { - chunk_id: None, - write_granularity: u32::try_from(STREAM_STRIP_BYTES / 1024).unwrap_or(u32::MAX), - strip_count: 1, - strip_type: StripType::Mirror as i32, - data_num: 0, - code_num: 0, - copy_count, - chunk_type: purpose.chunk_type() as i32, - writer_epoch, - writer_lease_ms, - owner_key: stream_name.chunk_owner_key(), - }) - .await?; - let chunk = response - .chunk - .ok_or_else(|| IoError::AllocationFailed("stream chunk allocation returned no chunk".into()))?; - Self::open_with_copy_count( - allocator, - disk_writer, - chunk, - stream_name, - writer_epoch, - writer_lease_ms, - copy_count, - purpose, - ) - } - - /// Opens an allocated or recovered direct mirror chunk. - /// - /// # Errors - /// - /// Returns an error for a mismatched epoch, state, capacity, or layout. - pub fn open( - allocator: Arc, - disk_writer: Arc, - chunk: Chunk, - stream_name: StreamName, - writer_epoch: u64, - writer_lease_ms: u64, - purpose: StreamPurpose, - ) -> Result { - Self::open_with_copy_count( - allocator, - disk_writer, - chunk, - stream_name, - writer_epoch, - writer_lease_ms, - 2, - purpose, - ) - } - - /// Opens a direct mirror chunk with the configured layout width. - /// - /// # Errors - /// - /// Returns an error for a mismatched epoch, state, capacity, or mirror count. - #[allow(clippy::too_many_arguments)] - pub fn open_with_copy_count( - allocator: Arc, - disk_writer: Arc, - chunk: Chunk, - stream_name: StreamName, - writer_epoch: u64, - writer_lease_ms: u64, - copy_count: u32, - purpose: StreamPurpose, - ) -> Result { - let chunk_id = chunk - .id - .ok_or_else(|| IoError::AllocationFailed("stream chunk has no identity".into()))?; - if chunk.writer_epoch != writer_epoch - || chunk.state != ChunkState::Active as i32 - || chunk.chunk_type != purpose.chunk_type() as i32 - || !purpose.matches(chunk_id) - || chunk.owner_key != stream_name.chunk_owner_key() - || chunk.strips.is_empty() - { - return Err(IoError::MetadataConflict( - "stream chunk ownership, state, type, or strip count is invalid".into(), - )); - } - let capacity = chunk - .strips - .iter() - .try_fold(0_u64, |total, strip| { - let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { - return Err(IoError::MetadataConflict( - "stream chunk does not contain a mirror strip".into(), - )); - }; - let actual_copies = mirror.segments.len(); - let protected = if copy_count == 1 { - actual_copies == 1 - } else { - (2..=copy_count as usize).contains(&actual_copies) - }; - if !protected || strip.unit_kb == 0 || strip.capacity == 0 { - return Err(IoError::MetadataConflict( - "stream chunk mirror geometry is invalid".into(), - )); - } - total - .checked_add(u64::from(strip.capacity) * 1024) - .ok_or_else(|| IoError::MetadataConflict("stream chunk capacity overflows".into())) - })? - .min(STREAM_CHUNK_BYTES); - if chunk.acknowledged_cursor > capacity { - return Err(IoError::MetadataConflict( - "stream chunk cursor exceeds its capacity".into(), - )); - } - Ok(Self { - allocator, - disk_writer, - chunk, - chunk_id, - writer_epoch, - writer_lease_ms, - capacity, - }) - } - - #[must_use] - pub fn chunk_id(&self) -> ChunkId { - self.chunk_id - } - - #[must_use] - pub fn cursor(&self) -> u64 { - self.chunk.acknowledged_cursor - } - - #[must_use] - pub fn capacity(&self) -> u64 { - self.capacity - } - - /// Returns the current authoritative chunk metadata snapshot. - #[must_use] - pub fn chunk(&self) -> &Chunk { - &self.chunk - } - - /// Appends mirror strips until the requested capacity is addressable. - /// - /// # Errors - /// - /// Returns an allocation error when chunkdb cannot extend this active - /// chunk, or when the logical stream-chunk limit would be exceeded. - pub async fn grow_to(&mut self, required_capacity: u64) -> Result { - if required_capacity <= self.capacity { - return Ok(true); - } - if required_capacity > STREAM_CHUNK_BYTES { - return Ok(false); - } - let first = self - .chunk - .strips - .first() - .ok_or_else(|| IoError::MetadataConflict("stream chunk has no mirror strip".into()))?; - let strip_bytes = u64::from(first.capacity) * 1024; - let strip_count = u32::try_from((required_capacity - self.capacity).div_ceil(strip_bytes)) - .map_err(|_| IoError::AllocationFailed("stream chunk extension is too large".into()))?; - let unit_count = first - .strip - .as_ref() - .and_then(|strip| match strip { - Strip::MirrorStrip(mirror) => mirror.segments.first(), - Strip::EcStrip(_) => None, - }) - .map(|segment| segment.unit_count) - .filter(|count| *count > 0) - .ok_or_else(|| IoError::MetadataConflict("stream chunk mirror geometry changed".into()))?; - for attempt in 0..2 { - let response = self - .allocator - .append_chunk(AppendChunkRequest { - chunk_id: Some(self.chunk_id), - modify_ts: self.chunk.modify_ts, - strip_size: unit_count, - strip_count, - strip_type: StripType::Mirror as i32, - data_num: 0, - code_num: 0, - copy_count: self.copy_count()?, - }) - .await?; - if let Some(current) = response.chunk { - self.chunk = current; - self.capacity = chunk_capacity(&self.chunk)?; - if attempt == 0 { - continue; - } - return Err(IoError::MetadataConflict( - "stream chunk revision changed twice".into(), - )); - } - if response.strips.is_empty() { - return Err(IoError::AllocationFailed( - "stream chunk extension returned no strips".into(), - )); - } - self.chunk.strips.extend(response.strips); - self.chunk.modify_ts = response.modify_ts; - self.chunk.capacity = self.chunk.strips.iter().map(|strip| strip.capacity).sum(); - self.capacity = chunk_capacity(&self.chunk)?; - return Ok(self.capacity >= required_capacity); - } - unreachable!("two append attempts either return or fail") - } - - fn copy_count(&self) -> Result { - self.chunk - .strips - .first() - .and_then(|strip| match &strip.strip { - Some(Strip::MirrorStrip(mirror)) => u32::try_from(mirror.segments.len()).ok(), - _ => None, - }) - .filter(|count| *count > 0) - .ok_or_else(|| IoError::MetadataConflict("stream chunk mirror layout changed".into())) - } - - /// Replicates an owned buffer and durably advances the fenced cursor. - /// - /// # Errors - /// - /// Returns a typed write or metadata error without retrying bytes at a - /// different cursor. - pub async fn append(&mut self, data: Bytes) -> Result<(u64, u64)> { - if data.is_empty() { - return Ok((self.cursor(), self.cursor())); - } - let begin = self.cursor(); - let end = - begin - .checked_add(u64::try_from(data.len()).map_err(|_| { - IoError::WriteFailed("stream append length exceeds addressable range".into()) - })?) - .ok_or_else(|| IoError::WriteFailed("stream append cursor overflows".into()))?; - if end > self.capacity { - return Err(IoError::WriteFailed( - "stream append exceeds the direct chunk capacity".into(), - )); - } - let mut writes = JoinSet::new(); - let mut copied = 0_usize; - for strip in &self.chunk.strips { - let strip_start = u64::from(strip.chunk_offset) * 1024; - let strip_end = strip_start.saturating_add(u64::from(strip.capacity) * 1024); - let write_start = begin.max(strip_start); - let write_end = end.min(strip_end); - if write_start >= write_end { - continue; - } - let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { - return Err(IoError::MetadataConflict( - "stream chunk mirror layout changed".into(), - )); - }; - let byte_count = usize::try_from(write_end - write_start) - .map_err(|_| IoError::WriteFailed("stream append slice is too large".into()))?; - let view = data.slice(copied..copied + byte_count); - copied += byte_count; - let unit_bytes = u64::from(strip.unit_kb) * 1024; - for segment in &mirror.segments { - let segment = *segment; - let disk_writer = Arc::clone(&self.disk_writer); - let view = view.clone(); - let offset = write_start - strip_start; - writes.spawn(async move { - disk_writer - .write_at_byte_offset(&segment, unit_bytes, offset, view) - .await - }); - } - } - if copied != data.len() { - return Err(IoError::MetadataConflict( - "stream chunk strips do not cover the append range".into(), - )); - } - while let Some(result) = writes.join_next().await { - result.map_err(|error| IoError::WriteFailed(format!("mirror write task failed: {error}")))??; - } - let response = self - .allocator - .advance_chunk_write(AdvanceChunkWriteRequest { - chunk_id: Some(self.chunk_id), - writer_epoch: self.writer_epoch, - expected_modify_ts: self.chunk.modify_ts, - acknowledged_cursor: end, - closed_strip_sequence: None, - writer_lease_ms: self.writer_lease_ms, - }) - .await?; - let updated = response - .chunk - .ok_or_else(|| IoError::MetadataConflict("cursor advance returned no chunk".into()))?; - if updated.id != Some(self.chunk_id) - || updated.writer_epoch != self.writer_epoch - || updated.acknowledged_cursor != end - { - return Err(IoError::MetadataConflict( - "cursor advance returned an inconsistent chunk".into(), - )); - } - self.chunk = updated; - Ok((begin, end)) - } - - /// Seals this chunk at its acknowledged cursor. - /// - /// # Errors - /// - /// Returns a metadata error if chunkdb cannot durably seal the chunk. - pub async fn seal(&mut self) -> Result<()> { - let seal_length = u32::try_from(self.cursor().div_ceil(1024)).unwrap_or(u32::MAX); - let response = self - .allocator - .seal_chunk(SealChunkRequest { - chunk_id: Some(self.chunk_id), - seal_length, - seal_bytes: self.cursor(), - }) - .await?; - let sealed = response - .chunk - .ok_or_else(|| IoError::MetadataConflict("seal returned no chunk".into()))?; - if sealed.id != Some(self.chunk_id) - || sealed.state != ChunkState::Sealed as i32 - || sealed.acknowledged_cursor < self.cursor() - { - return Err(IoError::MetadataConflict( - "seal returned an inconsistent stream chunk".into(), - )); - } - self.chunk = sealed; - Ok(()) - } -} - -fn chunk_capacity(chunk: &Chunk) -> Result { - chunk - .strips - .iter() - .try_fold(0_u64, |total, strip| { - total - .checked_add(u64::from(strip.capacity) * 1024) - .ok_or_else(|| IoError::MetadataConflict("stream chunk capacity overflows".into())) - }) - .map(|capacity| capacity.min(STREAM_CHUNK_BYTES)) -} diff --git a/lib/crowdb-chunk-client/src/chunk/mirror_flow.rs b/lib/crowdb-chunk-client/src/chunk/mirror_flow.rs index db2de74fe..f277d5927 100644 --- a/lib/crowdb-chunk-client/src/chunk/mirror_flow.rs +++ b/lib/crowdb-chunk-client/src/chunk/mirror_flow.rs @@ -85,7 +85,6 @@ impl MirrorStripFlow { pub async fn write( &self, chunk: &mut Chunk, - committed_cursor: u64, strip_sequence: u32, block_offset: u64, data: Bytes, @@ -94,7 +93,6 @@ impl MirrorStripFlow { ) -> Result<()> { self.write_views( chunk, - committed_cursor, strip_sequence, block_offset, vec![data], @@ -108,7 +106,6 @@ impl MirrorStripFlow { pub async fn write_views( &self, chunk: &mut Chunk, - committed_cursor: u64, strip_sequence: u32, block_offset: u64, data: Vec, @@ -177,15 +174,8 @@ impl MirrorStripFlow { image.freeze() }; for segment in failed { - self.repair( - chunk, - committed_cursor, - strip_sequence, - segment, - repair_image.clone(), - unit_bytes, - ) - .await?; + self.repair(chunk, strip_sequence, segment, repair_image.clone(), unit_bytes) + .await?; } Ok(()) } @@ -205,10 +195,17 @@ impl MirrorStripFlow { let segment = *segment; let disk_writer = Arc::clone(&self.disk_writer); let data = data.clone(); + let sync = self.sync; writes.spawn(async move { - let result = disk_writer - .write_views_at_byte_offset(&segment, unit_bytes, block_offset, data) - .await; + let result = if sync { + disk_writer + .write_synchronized_views_at_byte_offset(&segment, unit_bytes, block_offset, data) + .await + } else { + disk_writer + .write_views_at_byte_offset(&segment, unit_bytes, block_offset, data) + .await + }; (segment, result) }); } @@ -221,24 +218,6 @@ impl MirrorStripFlow { failed.push(segment); } } - if self.sync { - let mut syncs = JoinSet::new(); - for segment in segments { - if !failed.contains(segment) { - let segment = *segment; - let disk_writer = Arc::clone(&self.disk_writer); - syncs.spawn(async move { (segment, disk_writer.fsync(&segment).await) }); - } - } - while let Some(result) = syncs.join_next().await { - let (segment, result) = result - .map_err(|error| IoError::WriteFailed(format!("mirror fsync task failed: {error}")))?; - if let Err(error) = result { - tracing::warn!(?chunk_id, strip_sequence, ?segment, %error, "mirror strip fsync failed"); - failed.push(segment); - } - } - } Ok(failed) } } diff --git a/lib/crowdb-chunk-client/src/chunk/mirror_flow/repair.rs b/lib/crowdb-chunk-client/src/chunk/mirror_flow/repair.rs index 81b071eeb..9e2e7dc25 100644 --- a/lib/crowdb-chunk-client/src/chunk/mirror_flow/repair.rs +++ b/lib/crowdb-chunk-client/src/chunk/mirror_flow/repair.rs @@ -2,10 +2,10 @@ use std::sync::atomic::Ordering; use std::time::Duration; use std::time::Instant; -use bytes::Bytes; +use bytes::{Bytes, BytesMut}; use crowdb_protocol::chunkdb::rpc::{ - AllocateReplacementSegmentRequest, Chunk, DiscardReplacementSegmentRequest, QueryChunkRequest, - ReplaceChunkStripRangeRequest, Strip, + AllocateReplacementSegmentRequest, Chunk, ChunkState, ChunkType, DiscardReplacementSegmentRequest, + QueryChunkRequest, ReplaceChunkStripRangeRequest, Strip, }; use crowdb_protocol::common::ChunkId; use crowdb_protocol::diskdb::rpc::Segment; @@ -18,7 +18,6 @@ impl MirrorStripFlow { pub(super) async fn repair( &self, chunk: &mut Chunk, - committed_cursor: u64, strip_sequence: u32, failed: Segment, image: Bytes, @@ -29,7 +28,7 @@ impl MirrorStripFlow { metrics.active_repairs.fetch_add(1, Ordering::Relaxed); } let result = self - .try_repair(chunk, committed_cursor, strip_sequence, failed, image, unit_bytes) + .try_repair(chunk, strip_sequence, failed, image, unit_bytes) .await; if let Some(metrics) = &self.metrics { let elapsed = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX); @@ -46,7 +45,6 @@ impl MirrorStripFlow { async fn try_repair( &self, chunk: &mut Chunk, - committed_cursor: u64, strip_sequence: u32, failed: Segment, image: Bytes, @@ -110,21 +108,40 @@ impl MirrorStripFlow { last_error = "replacement allocation returned no segment".into(); continue; }; - let write = self - .disk_writer - .write_at_byte_offset(&replacement, unit_bytes, 0, image.clone()) - .await; - let sync = if write.is_ok() && self.sync { - self.disk_writer.fsync(&replacement).await + let stream_initialization = matches!( + ChunkType::try_from(chunk.chunk_type), + Ok(ChunkType::Stream | ChunkType::Wal) + ); + let write = if stream_initialization { + let capacity = usize::try_from(u64::from(old_strip.capacity) * 1024) + .map_err(|_| IoError::WriteFailed("replacement capacity overflows".into()))?; + if image.len() > capacity + || capacity > usize::try_from(crate::STREAM_STRIP_BYTES).expect("bounded stream strip") + { + return Err(IoError::WriteFailed("invalid stream replacement image".into())); + } + let mut initialized = BytesMut::zeroed(capacity); + initialized[..image.len()].copy_from_slice(&image); + self.disk_writer + .initialize_synchronized_at_byte_offset(&replacement, unit_bytes, 0, initialized.freeze()) + .await + } else if self.sync { + self.disk_writer + .write_synchronized_at_byte_offset(&replacement, unit_bytes, 0, image.clone()) + .await } else { - Ok(()) + self.disk_writer + .write_at_byte_offset(&replacement, unit_bytes, 0, image.clone()) + .await }; - if let Err(error) = write.and(sync) { + if let Err(error) = write { last_error = error.to_string(); if let Some(disk) = replacement.disk_id { self.failed_disks.insert(disk); } - self.discard(chunk.id, replacement).await; + if !stream_initialization { + self.discard(chunk.id, replacement).await; + } continue; } let Some(slot) = mirror.segments.iter_mut().find(|segment| **segment == failed) else { @@ -182,12 +199,17 @@ impl MirrorStripFlow { if let Some(metrics) = &self.metrics { metrics.exhausted_repairs.fetch_add(1, Ordering::Relaxed); } - self.mark_unavailable(chunk, committed_cursor, strip_sequence, failed) - .await?; + // Transport/write uncertainty does not prove corruption of the already + // acknowledged prefix. Keep its locations readable; complete frame + // verification is the authority for durable corruption markers. Err(IoError::ReplicaRepairExhausted(last_error)) } - async fn publish(&self, request: ReplaceChunkStripRangeRequest, replacement: Segment) -> Result { + async fn publish( + &self, + mut request: ReplaceChunkStripRangeRequest, + replacement: Segment, + ) -> Result { let mut attempt = 0_usize; loop { if attempt > 0 { @@ -204,8 +226,9 @@ impl MirrorStripFlow { } Err(error @ IoError::MetadataConflict(_)) => { if self.resolve_ambiguity { - match self.resolve_replacement(&request).await { + match self.resolve_replacement(&mut request).await { Ok(Some(chunk)) => return Ok(chunk), + Ok(None) if attempt < self.attempts => continue, Ok(None) => {} Err(error) => return Err(error), } @@ -213,7 +236,7 @@ impl MirrorStripFlow { self.discard(request.chunk_id, replacement).await; return Err(error); } - Err(_) if self.resolve_ambiguity => match self.resolve_replacement(&request).await { + Err(_) if self.resolve_ambiguity => match self.resolve_replacement(&mut request).await { Ok(Some(chunk)) => return Ok(chunk), Ok(None) if attempt < self.attempts => {} Ok(None) => { @@ -234,7 +257,10 @@ impl MirrorStripFlow { } } - async fn resolve_replacement(&self, request: &ReplaceChunkStripRangeRequest) -> Result> { + async fn resolve_replacement( + &self, + request: &mut ReplaceChunkStripRangeRequest, + ) -> Result> { loop { match self.inspect_replacement(request).await { Ok(state) => return Ok(state), @@ -247,7 +273,10 @@ impl MirrorStripFlow { } } - async fn inspect_replacement(&self, request: &ReplaceChunkStripRangeRequest) -> Result> { + async fn inspect_replacement( + &self, + request: &mut ReplaceChunkStripRangeRequest, + ) -> Result> { let chunk = self .allocator .query_chunk(QueryChunkRequest { @@ -256,7 +285,7 @@ impl MirrorStripFlow { .await? .chunk .ok_or_else(|| IoError::MetadataConflict("replacement chunk disappeared".into()))?; - if chunk.writer_epoch != self.writer_epoch { + if chunk.id != request.chunk_id || chunk.writer_epoch != self.writer_epoch { return Err(IoError::MetadataConflict("mirror writer epoch changed".into())); } let start = usize::try_from(request.start_index) @@ -266,9 +295,14 @@ impl MirrorStripFlow { return Ok(Some(chunk)); } let old_end = start.saturating_add(request.old_strips.len()); - if chunk.modify_ts == request.expected_modify_ts + if chunk.state == ChunkState::Active as i32 + && chunk.modify_ts >= request.expected_modify_ts && chunk.strips.get(start..old_end) == Some(request.old_strips.as_slice()) { + // Independent strip preparation can advance the chunk revision. + // Reuse the already initialized replacement and stable operation ID + // only while this owner and the exact replaced range still match. + request.expected_modify_ts = chunk.modify_ts; return Ok(None); } Err(IoError::MetadataConflict( @@ -285,46 +319,4 @@ impl MirrorStripFlow { }) .await; } - - async fn mark_unavailable( - &self, - chunk: &mut Chunk, - committed_cursor: u64, - strip_sequence: u32, - failed: Segment, - ) -> Result<()> { - let strip_index = chunk - .strips - .iter() - .position(|strip| strip.strip_sequence == strip_sequence) - .ok_or_else(|| IoError::MetadataConflict("failed strip disappeared".into()))?; - let old = chunk.strips[strip_index].clone(); - let strip_start = u64::from(old.chunk_offset) * 1024; - if committed_cursor <= strip_start { - return Ok(()); - } - let mut degraded = old.clone(); - if !degraded.unavailable_segments.contains(&failed) { - degraded.unavailable_segments.push(failed); - } - let operation_id = ChunkId { - high: self.writer_epoch ^ chunk.modify_ts ^ u64::MAX, - low: u64::from(strip_sequence) ^ failed.unit_offset, - }; - let response = self - .allocator - .replace_chunk_strip_range(ReplaceChunkStripRangeRequest { - chunk_id: chunk.id, - expected_modify_ts: chunk.modify_ts, - start_index: u32::try_from(strip_index).unwrap_or(u32::MAX), - old_strips: vec![old], - replacement_strips: vec![degraded], - operation_id: Some(operation_id), - }) - .await?; - *chunk = response - .chunk - .ok_or_else(|| IoError::MetadataConflict("degraded marker returned no chunk".into()))?; - Ok(()) - } } diff --git a/lib/crowdb-chunk-client/src/client.rs b/lib/crowdb-chunk-client/src/client.rs index 134e081e8..78552bf8f 100644 --- a/lib/crowdb-chunk-client/src/client.rs +++ b/lib/crowdb-chunk-client/src/client.rs @@ -3,6 +3,13 @@ //! Application-facing chunk IO client and prepared large writes. +mod disk_metrics; +use disk_metrics::MetricsDiskWriter; + +mod readiness; +#[cfg(feature = "test-util")] +pub use readiness::observe_readiness_phase_for_tests; + use std::collections::VecDeque; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -20,10 +27,9 @@ use crowdb_protocol::chunkdb::rpc::{ AppendChunkResponse, DeleteChunkRequest, DeleteChunkResponse, DiscardReplacementSegmentRequest, DiscardReplacementSegmentResponse, Location, QueryChunkRequest, QueryChunkResponse, ReplaceChunkStripRangeRequest, ReplaceChunkStripRangeResponse, SealChunkRequest, SealChunkResponse, - UpdateChunkStripRequest, UpdateChunkStripResponse, }; use crowdb_protocol::diskdb::rpc::Segment; -use crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES; +use crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES; use crate::metrics::SmallWriteMetrics; use crate::negative_list::FailedDiskList; @@ -81,6 +87,7 @@ pub struct ChunkIoClient { disk_writer: Arc, topology: Option>, metrics: Option>, + readiness: Arc, small_pool: Arc, reader: ChunkReader, failed_disks: Arc, @@ -156,6 +163,7 @@ impl ChunkIoClient { disk_writer, })), metrics: None, + readiness: Arc::default(), small_pool, reader, failed_disks, @@ -229,6 +237,7 @@ impl ChunkIoClient { disk_writer, topology: None, metrics: None, + readiness: Arc::default(), small_pool, reader, failed_disks, @@ -240,6 +249,7 @@ impl ChunkIoClient { /// Attach aggregate write-path metrics registered by the embedding process. #[must_use] pub fn with_metrics(mut self, metrics: &Arc) -> Self { + self.readiness = Arc::clone(&metrics.readiness); self.allocator = Arc::new(MetricsChunkAllocator { inner: self.allocator, metrics: Arc::clone(metrics), @@ -326,10 +336,10 @@ impl ChunkIoClient { if object_size == 0 { return Ok(SharedObjectWriter::empty()); } - if object_size > MAX_FRAME_PAYLOAD_BYTES { + if object_size > MAX_FRAME_V2_PAYLOAD_BYTES { return Err(IoError::ObjectTooLarge { size: object_size, - limit: MAX_FRAME_PAYLOAD_BYTES, + limit: MAX_FRAME_V2_PAYLOAD_BYTES, }); } let runtime = self.small_pool.prepare(object_size).await?; @@ -351,10 +361,10 @@ impl ChunkIoClient { if object_size == 0 { return Ok(SharedObjectWriter::empty()); } - if object_size > MAX_FRAME_PAYLOAD_BYTES { + if object_size > MAX_FRAME_V2_PAYLOAD_BYTES { return Err(IoError::ObjectTooLarge { size: object_size, - limit: MAX_FRAME_PAYLOAD_BYTES, + limit: MAX_FRAME_V2_PAYLOAD_BYTES, }); } let runtime = self.small_pool.prepare(object_size).await?; @@ -597,10 +607,6 @@ impl ChunkAllocator for MetricsChunkAllocator { result } - async fn update_chunk_strip(&self, req: UpdateChunkStripRequest) -> Result { - self.inner.update_chunk_strip(req).await - } - async fn query_chunk(&self, req: QueryChunkRequest) -> Result { let mut operation = self.metrics.chunk_query.start(); let result = self.inner.query_chunk(req).await; @@ -639,118 +645,6 @@ impl ChunkAllocator for MetricsChunkAllocator { } } -struct MetricsDiskWriter { - inner: Arc, - metrics: Arc, -} - -#[async_trait] -impl DiskWriter for MetricsDiskWriter { - async fn write(&self, seg: &Segment, unit_bytes: u64, data: Bytes) -> Result<()> { - let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX); - let mut operation = self.metrics.diskio_write.start(); - let result = self.inner.write(seg, unit_bytes, data).await; - if result.is_ok() { - self.metrics.diskio_write_bytes.observe(bytes); - operation.mark_success(); - } - result - } - - async fn write_views(&self, seg: &Segment, unit_bytes: u64, data: Vec) -> Result<()> { - let bytes = data.iter().fold(0_u64, |total, view| { - total.saturating_add(u64::try_from(view.len()).unwrap_or(u64::MAX)) - }); - let mut operation = self.metrics.diskio_write.start(); - let result = self.inner.write_views(seg, unit_bytes, data).await; - if result.is_ok() { - self.metrics.diskio_write_bytes.observe(bytes); - operation.mark_success(); - } - result - } - - async fn fsync(&self, seg: &Segment) -> Result<()> { - self.inner.fsync(seg).await - } - - async fn write_views_at_byte_offset( - &self, - seg: &Segment, - unit_bytes: u64, - byte_offset: u64, - data: Vec, - ) -> Result<()> { - let bytes = data.iter().map(|view| view.len() as u64).sum(); - let mut operation = self.metrics.diskio_write.start(); - let result = self - .inner - .write_views_at_byte_offset(seg, unit_bytes, byte_offset, data) - .await; - if result.is_ok() { - self.metrics.diskio_write_bytes.observe(bytes); - operation.mark_success(); - } - result - } - - async fn write_priority_at_byte_offset( - &self, - seg: &Segment, - unit_bytes: u64, - byte_offset: u64, - data: Bytes, - ) -> Result<()> { - let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX); - let mut operation = self.metrics.diskio_write.start(); - let result = self - .inner - .write_priority_at_byte_offset(seg, unit_bytes, byte_offset, data) - .await; - if result.is_ok() { - self.metrics.diskio_write_bytes.observe(bytes); - operation.mark_success(); - } - result - } - - async fn fsync_priority(&self, seg: &Segment) -> Result<()> { - self.inner.fsync_priority(seg).await - } - - async fn read(&self, seg: &Segment, unit_bytes: u64, segment_offset: u64, length: u32) -> Result { - let mut operation = self.metrics.diskio_read.start(); - let result = self.inner.read(seg, unit_bytes, segment_offset, length).await; - if let Ok(data) = &result { - self.metrics - .diskio_read_bytes - .observe(u64::try_from(data.len()).unwrap_or(u64::MAX)); - operation.mark_success(); - } - result - } - - async fn write_at_byte_offset( - &self, - seg: &Segment, - unit_bytes: u64, - byte_offset: u64, - data: Bytes, - ) -> Result<()> { - let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX); - let mut operation = self.metrics.diskio_write.start(); - let result = self - .inner - .write_at_byte_offset(seg, unit_bytes, byte_offset, data) - .await; - if result.is_ok() { - self.metrics.diskio_write_bytes.observe(bytes); - operation.mark_success(); - } - result - } -} - async fn discover_current_range_bindings( service: &ServiceRegistryClient, kv: Arc, diff --git a/lib/crowdb-chunk-client/src/client/disk_metrics.rs b/lib/crowdb-chunk-client/src/client/disk_metrics.rs new file mode 100644 index 000000000..03814fb71 --- /dev/null +++ b/lib/crowdb-chunk-client/src/client/disk_metrics.rs @@ -0,0 +1,229 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Metrics forwarding preserves the underlying writer's completion contract. + +use super::{async_trait, Arc, Bytes, ChunkClientMetrics, DiskWriter, Result, Segment}; + +pub(super) struct MetricsDiskWriter { + pub(super) inner: Arc, + pub(super) metrics: Arc, +} + +#[async_trait] +impl DiskWriter for MetricsDiskWriter { + async fn write(&self, seg: &Segment, unit_bytes: u64, data: Bytes) -> Result<()> { + let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX); + let mut operation = self.metrics.diskio_write.start(); + let result = self.inner.write(seg, unit_bytes, data).await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } + + async fn write_views(&self, seg: &Segment, unit_bytes: u64, data: Vec) -> Result<()> { + let bytes = data.iter().fold(0_u64, |total, view| { + total.saturating_add(u64::try_from(view.len()).unwrap_or(u64::MAX)) + }); + let mut operation = self.metrics.diskio_write.start(); + let result = self.inner.write_views(seg, unit_bytes, data).await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } + + async fn fsync(&self, seg: &Segment) -> Result<()> { + self.inner.fsync(seg).await + } + + async fn drain_segment( + &self, + seg: &Segment, + unit_bytes: u64, + deadline: std::time::Instant, + ) -> Result<()> { + self.inner.drain_segment(seg, unit_bytes, deadline).await + } + + async fn write_views_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Vec, + ) -> Result<()> { + let bytes = data.iter().map(|view| view.len() as u64).sum(); + let mut operation = self.metrics.diskio_write.start(); + let result = self + .inner + .write_views_at_byte_offset(seg, unit_bytes, byte_offset, data) + .await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } + + async fn write_synchronized_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Bytes, + ) -> Result<()> { + let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX); + let mut operation = self.metrics.diskio_write.start(); + let result = self + .inner + .write_synchronized_at_byte_offset(seg, unit_bytes, byte_offset, data) + .await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } + + async fn write_synchronized_views_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Vec, + ) -> Result<()> { + let bytes = data.iter().map(|view| view.len() as u64).sum(); + let mut operation = self.metrics.diskio_write.start(); + let result = self + .inner + .write_synchronized_views_at_byte_offset(seg, unit_bytes, byte_offset, data) + .await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } + + async fn initialize_synchronized_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Bytes, + ) -> Result<()> { + let bytes = data.len() as u64; + let mut operation = self.metrics.diskio_write.start(); + let result = self + .inner + .initialize_synchronized_at_byte_offset(seg, unit_bytes, byte_offset, data) + .await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } + + async fn write_synchronized_views_until( + &self, + seg: &Segment, + unit_bytes: u64, + offset: u64, + data: Vec, + deadline: std::time::Instant, + ) -> Result<()> { + let bytes = data.iter().map(|view| view.len() as u64).sum(); + let mut operation = self.metrics.diskio_write.start(); + let result = self + .inner + .write_synchronized_views_until(seg, unit_bytes, offset, data, deadline) + .await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } + + async fn initialize_synchronized_until( + &self, + seg: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + deadline: std::time::Instant, + ) -> Result<()> { + let bytes = data.len() as u64; + let mut operation = self.metrics.diskio_write.start(); + let result = self + .inner + .initialize_synchronized_until(seg, unit_bytes, offset, data, deadline) + .await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } + + async fn write_priority_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Bytes, + ) -> Result<()> { + let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX); + let mut operation = self.metrics.diskio_write.start(); + let result = self + .inner + .write_priority_at_byte_offset(seg, unit_bytes, byte_offset, data) + .await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } + + async fn fsync_priority(&self, seg: &Segment) -> Result<()> { + self.inner.fsync_priority(seg).await + } + + async fn read(&self, seg: &Segment, unit_bytes: u64, segment_offset: u64, length: u32) -> Result { + let mut operation = self.metrics.diskio_read.start(); + let result = self.inner.read(seg, unit_bytes, segment_offset, length).await; + if let Ok(data) = &result { + self.metrics + .diskio_read_bytes + .observe(u64::try_from(data.len()).unwrap_or(u64::MAX)); + operation.mark_success(); + } + result + } + + async fn write_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Bytes, + ) -> Result<()> { + let bytes = u64::try_from(data.len()).unwrap_or(u64::MAX); + let mut operation = self.metrics.diskio_write.start(); + let result = self + .inner + .write_at_byte_offset(seg, unit_bytes, byte_offset, data) + .await; + if result.is_ok() { + self.metrics.diskio_write_bytes.observe(bytes); + operation.mark_success(); + } + result + } +} diff --git a/lib/crowdb-chunk-client/src/client/readiness.rs b/lib/crowdb-chunk-client/src/client/readiness.rs new file mode 100644 index 000000000..04b550889 --- /dev/null +++ b/lib/crowdb-chunk-client/src/client/readiness.rs @@ -0,0 +1,120 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Read-only ChunkDB and physical DiskIO evidence through current client routes. + +use crowdb_protocol::{common::HwStatus, diskdb::rpc::Segment}; +use std::{future::Future, time::Duration}; +use tokio::time::Instant; + +use crate::metrics::{OperationMetrics, ReadinessMetricsSnapshot}; + +use super::{ChunkIoClient, DiskWriter, IoError, Result}; + +impl ChunkIoClient { + /// Observe current ChunkDB routing and read a bounded header from each online disk. + /// Every phase shares the caller's total budget. This never allocates or repairs chunks. + /// + /// # Errors + /// Returns discovery, ownership, transport or physical read failures. + pub async fn observe_readiness(&self, budget: Duration) -> Result { + let deadline = Instant::now() + budget; + let mut total = self.readiness.total.start(); + let topology = self + .topology + .as_ref() + .ok_or_else(|| IoError::Topology("readiness requires authoritative discovery".into()))?; + let (disks, chunks) = tokio::join!( + observe("hardware", deadline, &self.readiness.hardware, async { + topology + .hardware + .list_all_disks() + .await + .map_err(|error| IoError::Topology(error.to_string())) + }), + observe("ChunkDB", deadline, &self.readiness.chunkdb, async { + Box::pin(topology.chunkdb.observe_readiness()) + .await + .map_err(|error| IoError::ReadFailed(format!("ChunkDB read: {error}"))) + }), + ); + chunks?; + let disks = disks?; + let online: Vec<_> = disks + .iter() + .filter(|disk| disk.value.status == HwStatus::Up as i32) + .collect(); + if online.is_empty() { + return Err(IoError::Topology("no online disks are available".into())); + } + // Keep background observation work bounded independently of cluster size. + for batch in online.chunks(8) { + let mut reads = tokio::task::JoinSet::new(); + for disk in batch { + let writer = topology.disk_writer.clone(); + let metrics = self.readiness.clone(); + let id = disk.disk_id; + reads.spawn(async move { + let segment = Segment { + disk_id: Some(id), + unit_count: 1, + ..Default::default() + }; + observe("DiskIO", deadline, &metrics.diskio, async { + let bytes = writer.read(&segment, 4096, 0, 4096).await?; + if bytes.len() != 4096 { + return Err(IoError::ReadFailed("short disk header".into())); + } + Ok(()) + }) + .await + .map_err(|error| IoError::ReadFailed(format!("DiskIO disk {id:?}: {error}"))) + }); + } + while let Some(read) = reads.join_next().await { + read.map_err(|error| IoError::ReadFailed(error.to_string()))??; + } + } + total.mark_success(); + Ok(format!( + "ChunkDB read and 4096-byte DiskIO reads succeeded on {} online disks", + online.len() + )) + } + + /// Cumulative backend probe timings; reading these never probes dependencies. + #[must_use] + pub fn readiness_metrics(&self) -> ReadinessMetricsSnapshot { + self.readiness.snapshot() + } +} + +async fn observe( + phase: &str, + deadline: Instant, + metrics: &OperationMetrics, + future: impl Future>, +) -> Result { + let mut measurement = metrics.start(); + let result = tokio::time::timeout_at(deadline, future) + .await + .unwrap_or_else(|_| { + Err(IoError::ReadFailed(format!( + "{phase} read exceeded the readiness deadline" + ))) + }); + if result.is_ok() { + measurement.mark_success(); + } + result +} + +#[cfg(feature = "test-util")] +/// Exercise the production phase deadline and accounting with controlled work. +pub async fn observe_readiness_phase_for_tests( + metrics: &crate::metrics::ReadinessMetrics, + deadline: Instant, + work: impl Future>, +) -> Result<()> { + observe("DiskIO", deadline, &metrics.diskio, work).await +} diff --git a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs index e2f2eff5f..d5dacf4a3 100644 --- a/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs +++ b/lib/crowdb-chunk-client/src/disk_io/disk_writer.rs @@ -54,6 +54,79 @@ pub trait DiskWriter: Send + Sync { )) } + /// Complete an offset write only after its disk synchronization. + /// Production adapters combine write and flush in one supported wire request. + async fn write_synchronized_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Bytes, + ) -> Result<()> { + self.write_at_byte_offset(seg, unit_bytes, byte_offset, data) + .await?; + self.fsync(seg).await + } + + /// Initialize fresh storage with one synchronized submission and no retries. + /// An ambiguous result must never authorize subsequent writes or reuse. + async fn initialize_synchronized_at_byte_offset( + &self, + _seg: &Segment, + _unit_bytes: u64, + _byte_offset: u64, + _data: Bytes, + ) -> Result<()> { + Err(IoError::WriteFailed( + "disk writer does not support single-attempt initialization".into(), + )) + } + + /// Synchronize ordered immutable views while retaining their owners. + async fn write_synchronized_views_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Vec, + ) -> Result<()> { + self.write_views_at_byte_offset(seg, unit_bytes, byte_offset, data) + .await?; + self.fsync(seg).await + } + + /// Submit synchronized views under the caller's original owner deadline. + async fn write_synchronized_views_until( + &self, + seg: &Segment, + unit_bytes: u64, + offset: u64, + data: Vec, + deadline: std::time::Instant, + ) -> Result<()> { + if std::time::Instant::now() >= deadline { + return Err(IoError::WriteFailed("owner IO deadline expired".into())); + } + self.write_synchronized_views_at_byte_offset(seg, unit_bytes, offset, data) + .await + } + + /// Initialize once under the original owner deadline; never retry zeroing. + async fn initialize_synchronized_until( + &self, + seg: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + deadline: std::time::Instant, + ) -> Result<()> { + if std::time::Instant::now() >= deadline { + return Err(IoError::WriteFailed("owner IO deadline expired".into())); + } + self.initialize_synchronized_at_byte_offset(seg, unit_bytes, offset, data) + .await + } + /// Write a conversion-data range without queueing behind ordinary writes. async fn write_priority_at_byte_offset( &self, @@ -71,6 +144,19 @@ pub trait DiskWriter: Send + Sync { Ok(()) } + /// Drain the segment's ordered writes and synchronize before this deadline. + /// The caller must exclude future producers before observing or reusing it. + async fn drain_segment( + &self, + _seg: &Segment, + _unit_bytes: u64, + _deadline: std::time::Instant, + ) -> Result<()> { + Err(IoError::WriteFailed( + "disk writer does not support ordered segment drains".into(), + )) + } + /// Flush conversion data through the priority lane when available. async fn fsync_priority(&self, seg: &Segment) -> Result<()> { self.fsync(seg).await diff --git a/lib/crowdb-chunk-client/src/disk_io/routing.rs b/lib/crowdb-chunk-client/src/disk_io/routing.rs index bfe381289..c16615dc2 100644 --- a/lib/crowdb-chunk-client/src/disk_io/routing.rs +++ b/lib/crowdb-chunk-client/src/disk_io/routing.rs @@ -55,6 +55,12 @@ impl RoutedDiskWriter { }) } + #[cfg(feature = "test-util")] + #[must_use] + pub fn from_client_for_tests(client: Arc) -> Self { + Self { client } + } + /// Publish a refreshed complete route generation. pub async fn refresh(&self, _service: &ServiceRegistryClient, _hardware: &HardwareClient) -> Result<()> { self.client @@ -146,6 +152,100 @@ impl DiskWriter for RoutedDiskWriter { .map_err(map_write_error) } + async fn write_synchronized_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Bytes, + ) -> Result<()> { + let target = Self::target(seg, unit_bytes)?; + self.client + .write( + target, + byte_offset, + data, + Durability::Fsync, + self.client.normal_options(), + ) + .await + .map_err(map_write_error) + } + + async fn write_synchronized_views_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Vec, + ) -> Result<()> { + let target = Self::target(seg, unit_bytes)?; + self.client + .write_views( + target, + byte_offset, + data, + Durability::Fsync, + self.client.normal_options(), + ) + .await + .map_err(map_write_error) + } + + async fn initialize_synchronized_at_byte_offset( + &self, + seg: &Segment, + unit_bytes: u64, + byte_offset: u64, + data: Bytes, + ) -> Result<()> { + let target = Self::target(seg, unit_bytes)?; + self.client + .write( + target, + byte_offset, + data, + Durability::Fsync, + self.client.normal_options().without_write_retries(), + ) + .await + .map_err(map_write_error) + } + + async fn write_synchronized_views_until( + &self, + seg: &Segment, + unit_bytes: u64, + offset: u64, + data: Vec, + deadline: std::time::Instant, + ) -> Result<()> { + let target = Self::target(seg, unit_bytes)?; + let mut options = self.client.normal_options(); + options.deadline = options.deadline.min(deadline); + self.client + .write_views(target, offset, data, Durability::Fsync, options) + .await + .map_err(map_write_error) + } + + async fn initialize_synchronized_until( + &self, + seg: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + deadline: std::time::Instant, + ) -> Result<()> { + let target = Self::target(seg, unit_bytes)?; + let mut options = self.client.normal_options().without_write_retries(); + options.deadline = options.deadline.min(deadline); + self.client + .write(target, offset, data, Durability::Fsync, options) + .await + .map_err(map_write_error) + } + async fn write_priority_at_byte_offset( &self, seg: &Segment, @@ -194,6 +294,21 @@ impl DiskWriter for RoutedDiskWriter { .map_err(map_write_error) } + async fn drain_segment( + &self, + seg: &Segment, + unit_bytes: u64, + deadline: std::time::Instant, + ) -> Result<()> { + let target = Self::target(seg, unit_bytes)?; + let mut options = self.client.normal_options(); + options.deadline = deadline; + self.client + .drain_segment(target, options) + .await + .map_err(map_write_error) + } + async fn fsync_priority(&self, seg: &Segment) -> Result<()> { let disk_id = disk_id(seg).map_err(map_write_error)?; self.client diff --git a/lib/crowdb-chunk-client/src/lib.rs b/lib/crowdb-chunk-client/src/lib.rs index 65765ba5d..e19502a64 100644 --- a/lib/crowdb-chunk-client/src/lib.rs +++ b/lib/crowdb-chunk-client/src/lib.rs @@ -39,10 +39,15 @@ pub use benchmark::{ LargeWriteBenchmarkResult, ReadBenchmarkConfig, ReadBenchmarkResult, ReadBenchmarkWorkload, SmallWriteBenchmarkConfig, SmallWriteBenchmarkResult, }; +pub use chunk::frame_observation::{ + observe_frame_prefix, replay_frame_prefix, select_frame_prefix, FrameExtentSink, FrameObservation, + FrameObservationError, FrameObservationResult, FrameReplicaSelection, FrameWindowReader, + MirroredFrameReaders, MAX_FRAME_OBSERVATION_REPLICAS, +}; pub use chunk::{ ChunkPrefetch, ChunkReadPolicy, ChunkReadStream, ChunkReader, ChunkWriter, EcStripWriter, - FailedReadRange, MirrorChunkWriter, MirrorStripFlow, MirrorStripWriter, PartialReadResult, ReadRangeData, - StripReader, StripResult, StripWriter, STREAM_CHUNK_BYTES, + FailedReadRange, MirrorStripFlow, MirrorStripWriter, PartialReadResult, ReadRangeData, StripReader, + StripResult, StripWriter, STREAM_CHUNK_BYTES, STREAM_STRIP_BYTES, }; pub use client::{ ChunkIoClient, ChunkIoClientConfig, LargeWritePolicy, LargeWriteResult, PreparedLargeWrite, diff --git a/lib/crowdb-chunk-client/src/metrics.rs b/lib/crowdb-chunk-client/src/metrics.rs index 886b6a7ac..11e9d6579 100644 --- a/lib/crowdb-chunk-client/src/metrics.rs +++ b/lib/crowdb-chunk-client/src/metrics.rs @@ -3,6 +3,9 @@ //! Aggregate chunk-client write-path metrics. +mod readiness; +pub use readiness::{ReadinessMetrics, ReadinessMetricsSnapshot}; + use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Instant; @@ -68,6 +71,7 @@ impl Drop for OperationGuard { /// Hierarchical metrics for the large-write client path. pub struct ChunkClientMetrics { + pub readiness: Arc, pub object_write: OperationMetrics, pub chunk_allocate: OperationMetrics, pub chunk_append: OperationMetrics, @@ -92,6 +96,7 @@ impl ChunkClientMetrics { #[must_use] pub fn register(registry: &mut MetricsRegistry) -> Self { Self { + readiness: Arc::new(ReadinessMetrics::register(registry)), object_write: OperationMetrics::register(registry, "chunkio.object.write"), chunk_allocate: OperationMetrics::register(registry, "chunkio.chunk.allocate"), chunk_append: OperationMetrics::register(registry, "chunkio.chunk.append"), @@ -127,6 +132,7 @@ pub struct ReadFlowMetrics { pub(crate) stream_bytes_reserved: Arc, pub(crate) stream_bytes_released: Arc, pub(crate) layout_queries: Arc, + pub(crate) layout_cache_hits: Arc, pub(crate) layout_query_wait_ns: Arc, pub(crate) strip_read_wait_ns: Arc, pub(crate) chunk_read_wait_ns: Arc, @@ -148,6 +154,7 @@ pub struct ReadFlowMetricsSnapshot { pub stream_bytes_reserved: u64, pub stream_bytes_released: u64, pub layout_queries: u64, + pub layout_cache_hits: u64, pub layout_query_wait_ns: u64, pub strip_read_wait_ns: u64, pub chunk_read_wait_ns: u64, @@ -185,6 +192,7 @@ impl ReadFlowMetricsSnapshot { .stream_bytes_released .saturating_sub(earlier.stream_bytes_released), layout_queries: self.layout_queries.saturating_sub(earlier.layout_queries), + layout_cache_hits: self.layout_cache_hits.saturating_sub(earlier.layout_cache_hits), layout_query_wait_ns: self .layout_query_wait_ns .saturating_sub(earlier.layout_query_wait_ns), @@ -224,6 +232,7 @@ impl ReadFlowMetrics { stream_bytes_reserved: counter("chunkio.read.stream_bytes_reserved.c"), stream_bytes_released: counter("chunkio.read.stream_bytes_released.c"), layout_queries: counter("chunkio.read.layout_queries.c"), + layout_cache_hits: counter("chunkio.read.layout_cache_hits.c"), layout_query_wait_ns: counter("chunkio.read.layout_query_wait_ns.c"), strip_read_wait_ns: counter("chunkio.read.strip_read_wait_ns.c"), chunk_read_wait_ns: counter("chunkio.read.chunk_read_wait_ns.c"), @@ -246,6 +255,7 @@ impl ReadFlowMetrics { stream_bytes_reserved: self.stream_bytes_reserved.snapshot().total, stream_bytes_released: self.stream_bytes_released.snapshot().total, layout_queries: self.layout_queries.snapshot().total, + layout_cache_hits: self.layout_cache_hits.snapshot().total, layout_query_wait_ns: self.layout_query_wait_ns.snapshot().total, strip_read_wait_ns: self.strip_read_wait_ns.snapshot().total, chunk_read_wait_ns: self.chunk_read_wait_ns.snapshot().total, diff --git a/lib/crowdb-chunk-client/src/metrics/readiness.rs b/lib/crowdb-chunk-client/src/metrics/readiness.rs new file mode 100644 index 000000000..c608c237e --- /dev/null +++ b/lib/crowdb-chunk-client/src/metrics/readiness.rs @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Background readiness latency, including errors and cancelled observations. + +use crowdb_common::metrics::MetricsRegistry; + +use super::OperationMetrics; + +pub struct ReadinessMetrics { + pub(crate) total: OperationMetrics, + pub(crate) hardware: OperationMetrics, + pub(crate) chunkdb: OperationMetrics, + pub(crate) diskio: OperationMetrics, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)] +pub struct ReadinessPhaseSnapshot { + pub observations: u64, + pub errors: u64, + pub inflight: u64, + pub p50_ns: u64, + pub p99_ns: u64, + pub max_ns: u64, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, serde::Serialize)] +pub struct ReadinessMetricsSnapshot { + pub total: ReadinessPhaseSnapshot, + pub hardware: ReadinessPhaseSnapshot, + pub chunkdb: ReadinessPhaseSnapshot, + pub diskio: ReadinessPhaseSnapshot, +} + +impl Default for ReadinessMetrics { + fn default() -> Self { + Self::register(&mut MetricsRegistry::new()) + } +} + +impl ReadinessMetrics { + pub(super) fn register(registry: &mut MetricsRegistry) -> Self { + Self { + total: OperationMetrics::register(registry, "chunkio.readiness.total"), + hardware: OperationMetrics::register(registry, "chunkio.readiness.hardware"), + chunkdb: OperationMetrics::register(registry, "chunkio.readiness.chunkdb"), + diskio: OperationMetrics::register(registry, "chunkio.readiness.diskio"), + } + } + + #[must_use] + pub fn snapshot(&self) -> ReadinessMetricsSnapshot { + ReadinessMetricsSnapshot { + total: snapshot(&self.total), + hardware: snapshot(&self.hardware), + chunkdb: snapshot(&self.chunkdb), + diskio: snapshot(&self.diskio), + } + } +} + +fn snapshot(metrics: &OperationMetrics) -> ReadinessPhaseSnapshot { + let latency = metrics.latency.snapshot_total(); + ReadinessPhaseSnapshot { + observations: latency.total_count, + errors: metrics.errors.snapshot().total, + inflight: metrics.inflight.snapshot(), + p50_ns: latency.p50, + p99_ns: latency.p99, + max_ns: latency.max, + } +} diff --git a/lib/crowdb-chunk-client/src/traits.rs b/lib/crowdb-chunk-client/src/traits.rs index bf212617f..1c81629ae 100644 --- a/lib/crowdb-chunk-client/src/traits.rs +++ b/lib/crowdb-chunk-client/src/traits.rs @@ -19,7 +19,6 @@ use crowdb_protocol::chunkdb::rpc::{ PrepareMirrorToEcConversionRequest, PrepareMirrorToEcConversionResponse, QueryChunkRequest, QueryChunkResponse, ReplaceChunkStripRangeRequest, ReplaceChunkStripRangeResponse, ReserveStripGroupRequest, ReserveStripGroupResponse, SealChunkRequest, SealChunkResponse, - UpdateChunkStripRequest, UpdateChunkStripResponse, }; use std::sync::Arc; @@ -54,7 +53,6 @@ pub trait ChunkAllocator: Send + Sync { async fn delete_chunk_range(&self, _req: DeleteChunkRangeRequest) -> Result { Err(crate::IoError::Unsupported("chunk range deletion".into())) } - async fn update_chunk_strip(&self, req: UpdateChunkStripRequest) -> Result; async fn query_chunk(&self, req: QueryChunkRequest) -> Result; async fn ad_hoc_ec_recovery(&self, _req: AdHocEcRecoveryRequest) -> Result { Err(crate::IoError::MetadataConflict( @@ -134,9 +132,6 @@ impl ChunkAllocator for Arc { async fn delete_chunk_range(&self, req: DeleteChunkRangeRequest) -> Result { (**self).delete_chunk_range(req).await } - async fn update_chunk_strip(&self, req: UpdateChunkStripRequest) -> Result { - (**self).update_chunk_strip(req).await - } async fn query_chunk(&self, req: QueryChunkRequest) -> Result { (**self).query_chunk(req).await } @@ -209,9 +204,6 @@ impl ChunkAllocator for crowdb_chunkdb_client::ChunkdbClient { async fn delete_chunk_range(&self, req: DeleteChunkRangeRequest) -> Result { Ok(crowdb_chunkdb_client::ChunkdbClient::delete_chunk_range(self, req).await?) } - async fn update_chunk_strip(&self, req: UpdateChunkStripRequest) -> Result { - Ok(crowdb_chunkdb_client::ChunkdbClient::update_chunk_strip(self, req).await?) - } async fn query_chunk(&self, req: QueryChunkRequest) -> Result { Ok(crowdb_chunkdb_client::ChunkdbClient::query_chunk(self, req).await?) } diff --git a/lib/crowdb-chunk-client/src/writer/large_async_object.rs b/lib/crowdb-chunk-client/src/writer/large_async_object.rs index 818884615..26f23da2c 100644 --- a/lib/crowdb-chunk-client/src/writer/large_async_object.rs +++ b/lib/crowdb-chunk-client/src/writer/large_async_object.rs @@ -17,7 +17,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use bytes::{Bytes, BytesMut}; use crowdb_protocol::frame::{ - encode_frame, FrameMagic, FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, MAX_FRAME_PAYLOAD_BYTES, + encode_frame, FrameMagic, FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES, }; use tokio::sync::mpsc; use tokio::task::JoinHandle; @@ -317,9 +317,9 @@ impl LargeAsyncObjectWriter { fn strip_prefetch_plan(&self, chunk: &Chunk) -> Option { let remaining_bytes = self.object_size?.saturating_sub(self.logical_offset); - let frames = remaining_bytes.div_ceil(MAX_FRAME_PAYLOAD_BYTES as u64); + let frames = remaining_bytes.div_ceil(MAX_FRAME_V2_PAYLOAD_BYTES as u64); let physical_bytes = remaining_bytes - .saturating_add(frames.saturating_mul((FRAME_HEADER_PREFIX_BYTES + FRAME_FOOTER_BYTES) as u64)); + .saturating_add(frames.saturating_mul((FRAME_V2_HEADER_BYTES + FRAME_FOOTER_BYTES) as u64)); let strip_bytes = u64::from(chunk.strips.first()?.capacity) .checked_mul(1024)? .max(1); @@ -581,7 +581,7 @@ impl LargeAsyncObjectWriter { let payload_len = buffer .frame_payload_len(frame_index) .ok_or_else(|| IoError::WriteFailed("framed owner slot is missing".into()))?; - let frame_len = FRAME_HEADER_PREFIX_BYTES + let frame_len = FRAME_V2_HEADER_BYTES .checked_add(payload_len) .and_then(|length| length.checked_add(FRAME_FOOTER_BYTES)) .ok_or_else(|| IoError::WriteFailed("framed owner length overflows".into()))?; @@ -590,7 +590,7 @@ impl LargeAsyncObjectWriter { break; } let range = buffer - .finalize_frame(frame_index, FrameMagic::RepoLargeV1, chunk_id, write_time_ms) + .finalize_frame(frame_index, FrameMagic::RepoLargeV2, chunk_id, write_time_ms) .map_err(|error| IoError::WriteFailed(error.to_string()))?; if let Some(group) = &mut group_range { if group.end != range.start { @@ -646,7 +646,7 @@ impl LargeAsyncObjectWriter { self.buffer_metrics.payload_copy_operations.inc(); self.buffer_metrics.payload_copy_bytes.inc_by(buffer.len() as u64); self.frame_tail.extend_from_slice(&buffer); - while self.frame_tail.len() >= MAX_FRAME_PAYLOAD_BYTES { + while self.frame_tail.len() >= MAX_FRAME_V2_PAYLOAD_BYTES { self.push_full_payload_frames().await?; } Ok(()) @@ -654,7 +654,7 @@ impl LargeAsyncObjectWriter { async fn push_full_payload_frames(&mut self) -> Result<()> { const FRAMES_PER_BATCH: usize = 16; - const FRAME_BYTES: usize = FRAME_HEADER_PREFIX_BYTES + MAX_FRAME_PAYLOAD_BYTES + FRAME_FOOTER_BYTES; + const FRAME_BYTES: usize = FRAME_V2_HEADER_BYTES + MAX_FRAME_V2_PAYLOAD_BYTES + FRAME_FOOTER_BYTES; loop { self.ensure_open().await?; @@ -671,7 +671,7 @@ impl LargeAsyncObjectWriter { self.rotate_chunk().await?; continue; } - let frame_count = (self.frame_tail.len() / MAX_FRAME_PAYLOAD_BYTES) + let frame_count = (self.frame_tail.len() / MAX_FRAME_V2_PAYLOAD_BYTES) .min(FRAMES_PER_BATCH) .min(available_frames); let chunk_id = writer @@ -683,11 +683,11 @@ impl LargeAsyncObjectWriter { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) }); let mut framed = Vec::with_capacity(frame_count * FRAME_BYTES); - for payload in - self.frame_tail[..frame_count * MAX_FRAME_PAYLOAD_BYTES].chunks_exact(MAX_FRAME_PAYLOAD_BYTES) + for payload in self.frame_tail[..frame_count * MAX_FRAME_V2_PAYLOAD_BYTES] + .chunks_exact(MAX_FRAME_V2_PAYLOAD_BYTES) { framed.extend_from_slice( - &encode_frame(FrameMagic::RepoLargeV1, chunk_id, payload, write_time_ms) + &encode_frame(FrameMagic::RepoLargeV2, chunk_id, payload, write_time_ms) .map_err(|error| IoError::WriteFailed(error.to_string()))?, ); } @@ -696,7 +696,7 @@ impl LargeAsyncObjectWriter { .inc_by(u64::try_from(frame_count).unwrap_or(u64::MAX)); self.buffer_metrics .payload_copy_bytes - .inc_by(u64::try_from(frame_count * MAX_FRAME_PAYLOAD_BYTES).unwrap_or(u64::MAX)); + .inc_by(u64::try_from(frame_count * MAX_FRAME_V2_PAYLOAD_BYTES).unwrap_or(u64::MAX)); let status = self .chunk_writer .as_mut() @@ -707,10 +707,10 @@ impl LargeAsyncObjectWriter { self.rotate_chunk().await?; continue; } - let _ = self.frame_tail.split_to(frame_count * MAX_FRAME_PAYLOAD_BYTES); + let _ = self.frame_tail.split_to(frame_count * MAX_FRAME_V2_PAYLOAD_BYTES); self.logical_bytes_in_chunk = self .logical_bytes_in_chunk - .saturating_add((frame_count * MAX_FRAME_PAYLOAD_BYTES) as u64); + .saturating_add((frame_count * MAX_FRAME_V2_PAYLOAD_BYTES) as u64); return Ok(()); } } @@ -728,7 +728,7 @@ impl LargeAsyncObjectWriter { .map_or(0, |duration| { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) }); - let frame = encode_frame(FrameMagic::RepoLargeV1, chunk_id, &payload, write_time_ms) + let frame = encode_frame(FrameMagic::RepoLargeV2, chunk_id, &payload, write_time_ms) .map_err(|error| IoError::WriteFailed(error.to_string()))?; self.buffer_metrics.payload_copy_operations.inc(); self.buffer_metrics diff --git a/lib/crowdb-chunk-client/src/writer/large_object.rs b/lib/crowdb-chunk-client/src/writer/large_object.rs index 5ec90a3f3..5c51c0a84 100644 --- a/lib/crowdb-chunk-client/src/writer/large_object.rs +++ b/lib/crowdb-chunk-client/src/writer/large_object.rs @@ -14,7 +14,7 @@ use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use bytes::{Bytes, BytesMut}; -use crowdb_protocol::frame::{encode_frame, FrameMagic, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_protocol::frame::{encode_frame, FrameMagic, MAX_FRAME_V2_PAYLOAD_BYTES}; use tokio::sync::mpsc; use tokio::task::JoinHandle; @@ -234,8 +234,8 @@ impl ChunkIoWriter for LargeObjectWriter { self.start_pipeline(None); } self.frame_tail.extend_from_slice(&buffer); - while self.frame_tail.len() >= MAX_FRAME_PAYLOAD_BYTES { - let payload = self.frame_tail.split_to(MAX_FRAME_PAYLOAD_BYTES).freeze(); + while self.frame_tail.len() >= MAX_FRAME_V2_PAYLOAD_BYTES { + let payload = self.frame_tail.split_to(MAX_FRAME_V2_PAYLOAD_BYTES).freeze(); self.push_payload_frame(payload).await?; } Ok(FeedStatus::Continue) @@ -278,7 +278,7 @@ impl LargeObjectWriter { .map_or(0, |duration| { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) }); - let frame = encode_frame(FrameMagic::RepoLargeV1, chunk_id, &payload, write_time_ms) + let frame = encode_frame(FrameMagic::RepoLargeV2, chunk_id, &payload, write_time_ms) .map_err(|error| IoError::WriteFailed(error.to_string()))?; let remaining = self .chunk_writer diff --git a/lib/crowdb-chunk-client/src/writer/small_pipeline.rs b/lib/crowdb-chunk-client/src/writer/small_pipeline.rs index c25bb47c7..b0fb92b64 100644 --- a/lib/crowdb-chunk-client/src/writer/small_pipeline.rs +++ b/lib/crowdb-chunk-client/src/writer/small_pipeline.rs @@ -1404,26 +1404,21 @@ impl OwnedChunk { chunk_id: Some(chunk_id), }) .await?; - } else { - let seal_length = u32::try_from(self.cursor.div_ceil(1024)).unwrap_or(u32::MAX); - if let Err(error) = self - .allocator - .seal_chunk(SealChunkRequest { - chunk_id: Some(chunk_id), - seal_length, - seal_bytes: self.cursor, - }) - .await - { - // The chunkdb background sealer (`seal_expired_writer_chunks`) - // may have already sealed this chunk when our writer lease - // expired during a conversion fault. Treat an already-sealed - // chunk as success; otherwise propagate the original error. - if !self.accept_external_seal().await? { - return Err(IoError::MetadataConflict(format!( - "seal reserved chunk failed: {error}" - ))); - } + } else if let Err(error) = self + .allocator + .seal_chunk(SealChunkRequest { + chunk_id: Some(chunk_id), + seal_bytes: self.cursor, + }) + .await + { + // The background finalizer may have sealed this chunk when its lease + // expired during a conversion fault. Reconcile the exact seal before + // returning the original failure. + if !self.accept_external_seal().await? { + return Err(IoError::MetadataConflict(format!( + "seal reserved chunk failed: {error}" + ))); } } Ok(()) @@ -1485,7 +1480,8 @@ fn batch_shape(batch: &[PendingObject]) -> Result<(usize, usize, usize)> { } fn frame_bytes(payload_bytes: usize) -> Result { - crowdb_protocol::frame::framed_physical_length(payload_bytes as u64) + crowdb_protocol::frame::FrameLayout::OrdinaryV2 + .physical_length(payload_bytes as u64) .map_err(|error| IoError::WriteFailed(error.to_string())) .and_then(|bytes| usize::try_from(bytes).map_err(|error| IoError::WriteFailed(error.to_string()))) } diff --git a/lib/crowdb-chunk-client/src/writer/small_pipeline/batch.rs b/lib/crowdb-chunk-client/src/writer/small_pipeline/batch.rs index 841cb12f0..e0404966a 100644 --- a/lib/crowdb-chunk-client/src/writer/small_pipeline/batch.rs +++ b/lib/crowdb-chunk-client/src/writer/small_pipeline/batch.rs @@ -86,7 +86,6 @@ impl OwnedChunk { .mirror_flow .write_views( &mut self.chunk, - self.cursor, strip.strip_sequence, block_offset, views, @@ -158,7 +157,7 @@ fn pack_batch( .frame_payload_len(index) .ok_or_else(|| IoError::WriteFailed("missing frame payload".into()))?; let range = owner - .finalize_frame(index, FrameMagic::RepoSmallV1, chunk_id, write_time_ms) + .finalize_frame(index, FrameMagic::RepoSmallV2, chunk_id, write_time_ms) .map_err(|error| IoError::WriteFailed(error.to_string()))?; if range.start != end { return Err(IoError::WriteFailed( @@ -182,7 +181,7 @@ fn pack_batch( let mut fragments: VecDeque = object.fragments.iter().cloned().collect(); let mut remaining = object.len; while remaining > 0 { - let length = remaining.min(crowdb_protocol::frame::MAX_FRAME_PAYLOAD_BYTES); + let length = remaining.min(crowdb_protocol::frame::MAX_FRAME_V2_PAYLOAD_BYTES); let payload = take_views(&mut fragments, length); if payload.iter().map(Bytes::len).sum::() != length { return Err(IoError::SourceRead( @@ -191,7 +190,7 @@ fn pack_batch( } views.extend( crowdb_protocol::frame::encode_frame_views( - FrameMagic::RepoSmallV1, + FrameMagic::RepoSmallV2, chunk_id, payload, write_time_ms, diff --git a/lib/crowdb-chunk-client/tests/chunk_reader_test.rs b/lib/crowdb-chunk-client/tests/chunk_reader_test.rs index df59616ed..d8c5f2e0d 100644 --- a/lib/crowdb-chunk-client/tests/chunk_reader_test.rs +++ b/lib/crowdb-chunk-client/tests/chunk_reader_test.rs @@ -18,16 +18,22 @@ use crowdb_protocol::chunkdb::rpc::{ AllocateChunkResponse, AppendChunkRequest, AppendChunkResponse, Chunk, ChunkState, ChunkStrip, DeleteChunkRequest, DeleteChunkResponse, EcState, EcStrip, Location, MirrorStrip, QueryChunkRequest, QueryChunkResponse, ReplaceChunkStripRangeRequest, ReplaceChunkStripRangeResponse, SealChunkRequest, - SealChunkResponse, Strip, StripType, UpdateChunkStripRequest, UpdateChunkStripResponse, + SealChunkResponse, Strip, StripType, }; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::Segment; -use crowdb_protocol::frame::{encode_frame, FrameMagic, MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_protocol::frame::{encode_frame, FrameMagic, MAX_FRAME_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES}; use tokio::sync::{Notify, Semaphore}; const KIB: usize = 1024; const SHARD: usize = 64 * KIB; +#[path = "chunk_reader_test/frame_revision.rs"] +mod frame_revision; + +#[path = "chunk_reader_test/layout_cache.rs"] +mod layout_cache; + struct MemoryDiskIo { shards: Vec<(DiskId, Bytes)>, failed: Vec, @@ -92,13 +98,6 @@ impl ChunkAllocator for SequenceAllocator { unreachable!("reader test does not delete") } - async fn update_chunk_strip( - &self, - _request: UpdateChunkStripRequest, - ) -> Result { - unreachable!("reader test does not update") - } - async fn query_chunk(&self, _request: QueryChunkRequest) -> Result { let query = self.queries.fetch_add(1, Ordering::AcqRel); Ok(QueryChunkResponse { @@ -446,7 +445,7 @@ async fn framed_read_stream_keeps_whole_frame_under_small_window_setting() { let id = ChunkId { high: 13, low: 37 }; let segment = segment(1); let payload = b"abcdefghij"; - let frame = Bytes::from(encode_frame(FrameMagic::RepoLargeV1, id, payload, 1).unwrap()); + let frame = Bytes::from(encode_frame(FrameMagic::RepoLargeV2, id, payload, 1).unwrap()); let chunk = Chunk { id: Some(id), state: ChunkState::Sealed as i32, @@ -512,7 +511,7 @@ async fn read_stream_selects_only_locations_overlapping_each_window() { let mut physical = Vec::new(); let mut locations = Vec::new(); for (index, payload) in [b"abcd", b"efgh", b"ijkl"].into_iter().enumerate() { - let frame = encode_frame(FrameMagic::RepoLargeV1, id, payload, 1).unwrap(); + let frame = encode_frame(FrameMagic::RepoLargeV2, id, payload, 1).unwrap(); locations.push(Location { chunk_id: Some(id), offset: physical.len() as u64, @@ -582,7 +581,7 @@ async fn stream_waits_for_first_result_and_retains_three_slots_until_consumer_re let mut physical = Vec::new(); let mut locations = Vec::new(); for (index, payload) in [b"aaaa", b"bbbb", b"cccc", b"dddd"].into_iter().enumerate() { - let frame = encode_frame(FrameMagic::RepoLargeV1, id, payload, 1).unwrap(); + let frame = encode_frame(FrameMagic::RepoLargeV2, id, payload, 1).unwrap(); locations.push(Location { chunk_id: Some(id), offset: physical.len() as u64, @@ -676,8 +675,8 @@ async fn global_read_budget_remains_charged_while_http_keeps_frame_views() { let id = ChunkId { high: 29, low: 57 }; let mut segment = segment(1); segment.unit_count = 2; - let payload = vec![0x5a; MAX_FRAME_PAYLOAD_BYTES]; - let frame = encode_frame(FrameMagic::RepoLargeV1, id, &payload, 1).unwrap(); + let payload = vec![0x5a; MAX_FRAME_V2_PAYLOAD_BYTES]; + let frame = encode_frame(FrameMagic::RepoLargeV2, id, &payload, 1).unwrap(); assert_eq!(frame.len(), MAX_FRAME_BYTES); let physical = Bytes::from(frame.repeat(32)); let chunk = Chunk { @@ -726,7 +725,7 @@ async fn global_read_budget_remains_charged_while_http_keeps_frame_views() { let location = Location { chunk_id: Some(id), length: 32 * MAX_FRAME_BYTES as u64, - logical_length: 32 * MAX_FRAME_PAYLOAD_BYTES as u64, + logical_length: 32 * MAX_FRAME_V2_PAYLOAD_BYTES as u64, ..Location::default() }; let mut stream = reader.read_stream(&[location]).unwrap(); @@ -751,7 +750,7 @@ async fn frame_crossing_physical_parts_returns_two_original_payload_views() { let first_segment = segment(1); let second_segment = segment(2); let payload = b"abcdefghijklmnopqrst"; - let frame = encode_frame(FrameMagic::RepoSmallV1, id, payload, 1).unwrap(); + let frame = encode_frame(FrameMagic::RepoSmallV2, id, payload, 1).unwrap(); let frame_offset = 1000_usize; let first_length = 1024 - frame_offset; let mut first_data = vec![0; 1024]; @@ -821,7 +820,12 @@ async fn frame_crossing_physical_parts_returns_two_original_payload_views() { .unwrap(); assert_eq!(views.concat(), payload); assert_eq!(views.len(), 2); - assert_eq!(views[0].as_ptr(), first_data.slice(frame_offset + 14..).as_ptr()); + assert_eq!( + views[0].as_ptr(), + first_data + .slice(frame_offset + crowdb_protocol::frame::FRAME_V2_HEADER_BYTES..) + .as_ptr() + ); assert_eq!(views[1].as_ptr(), second_data.as_ptr()); } @@ -831,7 +835,7 @@ async fn framed_mirror_crc_failure_uses_a_verified_fallback() { let first = segment(1); let second = segment(2); let payload = b"verified mirror payload"; - let valid = Bytes::from(encode_frame(FrameMagic::RepoSmallV1, id, payload, 1).unwrap()); + let valid = Bytes::from(encode_frame(FrameMagic::RepoSmallV2, id, payload, 1).unwrap()); let mut corrupt = valid.to_vec(); corrupt[14] ^= 0x80; let chunk = Chunk { diff --git a/lib/crowdb-chunk-client/tests/chunk_reader_test/frame_revision.rs b/lib/crowdb-chunk-client/tests/chunk_reader_test/frame_revision.rs new file mode 100644 index 000000000..ee48f85e7 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/chunk_reader_test/frame_revision.rs @@ -0,0 +1,79 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "../common/frame_revision.rs"] +mod fixture; + +use bytes::Bytes; +use crowdb_protocol::frame::{FrameMagic, MAX_FRAME_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES}; +use fixture::TestFramedReader; +use std::sync::atomic::Ordering; + +#[tokio::test] +async fn ordinary_ranges_cross_frames_and_objects_without_payload_copies() { + let old = Bytes::from(vec![0x35; 71]); + let payload = Bytes::from( + (0..2 * MAX_FRAME_V2_PAYLOAD_BYTES + 17) + .map(|index| u8::try_from(index % 251).unwrap()) + .collect::>(), + ); + let fixture = TestFramedReader::new( + &[ + (FrameMagic::RepoSmallV2, old.clone()), + (FrameMagic::RepoLargeV2, payload.clone()), + ], + 1, + ); + let start = old.len() as u64 + MAX_FRAME_V2_PAYLOAD_BYTES as u64 - 1; + let end = start + 9; + let actual = fixture + .reader + .read_range(&fixture.locations, start, end) + .await + .unwrap(); + assert_eq!( + actual.concat(), + payload[MAX_FRAME_V2_PAYLOAD_BYTES - 1..MAX_FRAME_V2_PAYLOAD_BYTES + 8] + ); + let mut stream = fixture.reader.read_stream(&fixture.locations).unwrap(); + let mut actual = Vec::new(); + while let Some(part) = stream.next_chunk().await { + actual.extend_from_slice(&part.unwrap()); + } + assert_eq!(actual, [old.as_ref(), payload.as_ref()].concat()); + let metrics = fixture.reader.flow_metrics_snapshot(); + assert!(metrics.stream_windows >= 4); + assert!(fixture.reads.load(Ordering::Relaxed) >= 4); + assert!(fixture.queries() <= 3); +} + +#[tokio::test] +async fn v2_small_frames_and_short_final_frames_work_under_one_byte_windows() { + for length in [1, MAX_FRAME_V2_PAYLOAD_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES + 1] { + let payload = Bytes::from(vec![0x47; length]); + let fixture = TestFramedReader::new(&[(FrameMagic::RepoSmallV2, payload.clone())], 1); + let mut stream = fixture.reader.read_stream(&fixture.locations).unwrap(); + let mut actual = Vec::new(); + while let Some(part) = stream.next_chunk().await { + actual.extend_from_slice(&part.unwrap()); + } + assert_eq!(actual, payload); + assert_eq!(fixture.queries(), 1); + } +} + +#[tokio::test] +async fn location_geometry_must_match_the_verified_revision() { + let mut fixture = TestFramedReader::new( + &[(FrameMagic::RepoLargeV2, Bytes::from_static(b"payload"))], + MAX_FRAME_BYTES, + ); + fixture.locations[0].length -= 1; + assert!(fixture.reader.read_object(&fixture.locations).await.is_err()); + let mut fixture = TestFramedReader::new( + &[(FrameMagic::RepoLargeV2, Bytes::from_static(b"payload"))], + MAX_FRAME_BYTES, + ); + fixture.locations[0].length += 1; + assert!(fixture.reader.read_object(&fixture.locations).await.is_err()); +} diff --git a/lib/crowdb-chunk-client/tests/chunk_reader_test/layout_cache.rs b/lib/crowdb-chunk-client/tests/chunk_reader_test/layout_cache.rs new file mode 100644 index 000000000..8bd499387 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/chunk_reader_test/layout_cache.rs @@ -0,0 +1,123 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +const ID: ChunkId = ChunkId { high: 83, low: 89 }; +const MAGIC: FrameMagic = FrameMagic::RepoSmallV2; + +fn fixture( + first_end: Option, + validity_ms: u64, + corrupt_second: bool, +) -> (ChunkReader, Arc, Vec) { + let frames: Vec<_> = [b"first".as_slice(), b"second".as_slice()] + .into_iter() + .map(|bytes| Bytes::from(encode_frame(MAGIC, ID, bytes, 1).unwrap())) + .collect(); + let bytes = Bytes::from(frames.concat()); + let replicas = vec![segment(1), segment(2)]; + let current = Chunk { + id: Some(ID), + state: ChunkState::Active as i32, + modify_ts: 1, + capacity: 1, + acknowledged_cursor: bytes.len() as u64, + strips: vec![ChunkStrip { + capacity: 1, + unit_kb: 1, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: replicas.clone(), + })), + ..ChunkStrip::default() + }], + ..Chunk::default() + }; + let mut first = current.clone(); + if let Some(end) = first_end { + first.acknowledged_cursor = end; + } + let allocator = Arc::new(SequenceAllocator { + queries: AtomicUsize::new(0), + first, + current: Mutex::new(current), + first_layout_validity_ms: validity_ms, + full_reply: None, + full_calls: AtomicUsize::new(0), + }); + let mut damaged = bytes.to_vec(); + if corrupt_second { + damaged[frames[0].len() + crowdb_protocol::frame::FRAME_V2_HEADER_BYTES] ^= 1; + } + let disks = Arc::new(MemoryDiskIo { + shards: vec![ + (replicas[0].disk_id.unwrap(), Bytes::from(damaged)), + (replicas[1].disk_id.unwrap(), bytes), + ], + failed: vec![], + reads: Arc::new(AtomicUsize::new(0)), + }); + let reader = ChunkReader::new( + allocator.clone(), + disks, + ChunkReadPolicy { + layout_safety_margin: Duration::ZERO, + ..ChunkReadPolicy::default() + }, + ) + .unwrap(); + (reader, allocator, frames) +} + +async fn read(reader: &ChunkReader, frames: &[Bytes], index: usize) -> Bytes { + let offset = if index == 0 { 0 } else { frames[0].len() as u64 }; + reader + .read_verified_frame(ID, offset, frames[index].len() as u64, MAGIC) + .await + .unwrap() +} + +#[tokio::test] +async fn immutable_frames_share_one_bounded_layout_across_reader_clones() { + let (reader, allocator, frames) = fixture(None, 1_000, false); + for index in 0..32 { + assert_eq!(read(&reader.clone(), &frames, index % 2).await, frames[index % 2]); + } + assert_eq!(allocator.queries.load(Ordering::Acquire), 1); + assert_eq!(reader.flow_metrics_snapshot().layout_cache_hits, 31); +} + +#[tokio::test] +async fn an_append_beyond_the_cached_durable_prefix_refreshes_layout() { + let first_length = encode_frame(MAGIC, ID, b"first", 1).unwrap().len() as u64; + let (reader, allocator, frames) = fixture(Some(first_length), 1_000, false); + assert_eq!(read(&reader, &frames, 0).await, frames[0]); + assert_eq!(read(&reader, &frames, 1).await, frames[1]); + assert_eq!(read(&reader, &frames, 0).await, frames[0]); + assert_eq!(allocator.queries.load(Ordering::Acquire), 2); + assert_eq!(reader.flow_metrics_snapshot().layout_cache_hits, 1); +} + +#[tokio::test] +async fn expired_frame_layout_is_refreshed_before_a_successful_reply() { + let (reader, allocator, frames) = fixture(None, 0, false); + assert_eq!(read(&reader, &frames, 0).await, frames[0]); + assert_eq!(read(&reader, &frames, 1).await, frames[1]); + assert_eq!(allocator.queries.load(Ordering::Acquire), 2); +} + +#[tokio::test] +async fn corrupt_cached_replica_invalidates_layout_and_retries_verified_bytes() { + let (reader, allocator, frames) = fixture(None, 1_000, true); + assert_eq!(read(&reader, &frames, 0).await, frames[0]); + assert_eq!(read(&reader, &frames, 1).await, frames[1]); + // Initial layout, confirmed corrupt-replica observation, then a fresh grant. + assert_eq!(allocator.queries.load(Ordering::Acquire), 3); + assert_eq!( + allocator.current.lock().unwrap().strips[0].unavailable_segments, + vec![segment(1)] + ); + assert_eq!(read(&reader, &frames, 0).await, frames[0]); + assert_eq!(allocator.queries.load(Ordering::Acquire), 3); + assert_eq!(reader.flow_metrics_snapshot().layout_cache_hits, 2); +} diff --git a/lib/crowdb-chunk-client/tests/chunk_writer_test.rs b/lib/crowdb-chunk-client/tests/chunk_writer_test.rs index 94a5ddc92..61b30a88f 100644 --- a/lib/crowdb-chunk-client/tests/chunk_writer_test.rs +++ b/lib/crowdb-chunk-client/tests/chunk_writer_test.rs @@ -29,8 +29,7 @@ use crowdb_protocol::chunkdb::rpc::Strip as StripOneof; use crowdb_protocol::chunkdb::rpc::{ AllocateChunkRequest, AllocateChunkResponse, AppendChunkRequest, AppendChunkResponse, Chunk, ChunkState, ChunkStrip, ChunkType, DeleteChunkRequest, DeleteChunkResponse, EcStrip, MirrorStrip, QueryChunkRequest, - QueryChunkResponse, SealChunkRequest, SealChunkResponse, StripType, UpdateChunkStripRequest, - UpdateChunkStripResponse, + QueryChunkResponse, SealChunkRequest, SealChunkResponse, StripType, }; use crowdb_protocol::common::{ChunkId, DiskId as ProtoDiskId}; use crowdb_protocol::diskdb::rpc::Segment; @@ -207,7 +206,7 @@ struct MockChunkAllocator { #[derive(Debug, Default)] struct MockChunkState { - chunks: HashMap<(u64, u64), (Vec, u32, bool)>, + chunks: HashMap<(u64, u64), (Vec, u64, bool)>, next_segment_offset: u64, allocate_calls: usize, append_calls: usize, @@ -353,7 +352,7 @@ impl ChunkAllocator for MockChunkAllocator { .chunks .get_mut(&(chunk_id.high, chunk_id.low)) .expect("seal unknown chunk"); - entry.1 = req.seal_length; + entry.1 = req.seal_bytes; Ok(SealChunkResponse { chunk: None }) } @@ -369,10 +368,6 @@ impl ChunkAllocator for MockChunkAllocator { Ok(DeleteChunkResponse { chunk: None }) } - async fn update_chunk_strip(&self, _req: UpdateChunkStripRequest) -> Result { - Ok(UpdateChunkStripResponse { chunk: None }) - } - async fn query_chunk(&self, req: QueryChunkRequest) -> Result { let chunk_id = req.chunk_id.unwrap(); let state = self.state.lock().unwrap(); @@ -389,8 +384,8 @@ impl ChunkAllocator for MockChunkAllocator { } else { ChunkState::Sealed as i32 }, - sealed_length: *length, - acknowledged_cursor: u64::from(*length) * 1024, + sealed_length: u32::try_from(length.div_ceil(1024)).unwrap(), + acknowledged_cursor: *length, ..Chunk::default() }; Ok(QueryChunkResponse { diff --git a/lib/crowdb-chunk-client/tests/common/frame_observation.rs b/lib/crowdb-chunk-client/tests/common/frame_observation.rs new file mode 100644 index 000000000..1d51162e7 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/frame_observation.rs @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{FrameExtentSink, FrameWindowReader, IoError}; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::frame::{encode_append_group_views, FrameScanConfig, FrameScanMode, ScannedFrameExtent}; +use std::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use std::time::Duration; + +pub struct TestObservation; + +impl TestObservation { + pub const CHUNK: ChunkId = ChunkId { + high: 4 << 56, + low: 23, + }; + pub fn group(offset: usize, payload: &[u8]) -> Vec { + encode_append_group_views(Self::CHUNK, offset as u64, &Bytes::copy_from_slice(payload), 1) + .unwrap() + .concat() + } + pub fn config(capacity: usize) -> FrameScanConfig { + FrameScanConfig { + chunk_id: Self::CHUNK, + mode: FrameScanMode::Grouped, + physical_start: 0, + physical_limit: capacity as u64, + required_end: 0, + logical_start: 0, + max_append_bytes: 64 * 1024 * 1024, + } + } + pub fn backing(prefix: &[u8], capacity: usize) -> Bytes { + let mut bytes = prefix.to_vec(); + bytes.resize(capacity, 0); + Bytes::from(bytes) + } +} + +pub struct TestWindowReader { + data: Bytes, + pub delay: Duration, + pub fail_at: Option, + pub short_at: Option, + pub fail_on_call: Option, + pub stall_on_call: Option, + calls: AtomicUsize, + largest: AtomicU32, + last_end: AtomicU64, +} + +impl TestWindowReader { + pub fn new(data: Bytes) -> Self { + Self { + data, + delay: Duration::ZERO, + fail_at: None, + short_at: None, + fail_on_call: None, + stall_on_call: None, + calls: AtomicUsize::new(0), + largest: AtomicU32::new(0), + last_end: AtomicU64::new(0), + } + } + pub fn calls(&self) -> usize { + self.calls.load(Ordering::Relaxed) + } + pub fn largest(&self) -> u32 { + self.largest.load(Ordering::Relaxed) + } + pub fn last_end(&self) -> u64 { + self.last_end.load(Ordering::Relaxed) + } +} + +#[async_trait] +impl FrameWindowReader for TestWindowReader { + async fn read_window(&self, offset: u64, length: u32) -> crowdb_chunk_client::Result { + let call = self.calls.fetch_add(1, Ordering::Relaxed) + 1; + self.largest.fetch_max(length, Ordering::Relaxed); + self.last_end.store(offset + u64::from(length), Ordering::Relaxed); + if !self.delay.is_zero() { + tokio::time::sleep(self.delay).await; + } + if self.stall_on_call == Some(call) { + tokio::time::sleep(Duration::from_secs(60)).await; + } + if self.fail_on_call == Some(call) || self.fail_at.is_some_and(|at| offset >= at) { + return Err(IoError::ReadFailed("injected replica read failure".into())); + } + let start = + usize::try_from(offset).map_err(|_| IoError::ReadFailed("read offset overflows".into()))?; + let end = start + length as usize; + if end > self.data.len() { + return Err(IoError::ReadFailed("read exceeds allocated data".into())); + } + let end = if self.short_at.is_some_and(|at| offset >= at) { + end - 1 + } else { + end + }; + Ok(self.data.slice(start..end)) + } +} + +#[derive(Default)] +pub struct TestExtentSink { + pub records: Vec, + pub largest_batch: usize, + pub delay: Duration, + pub fail: bool, +} + +#[async_trait] +impl FrameExtentSink for TestExtentSink { + async fn append_extents(&mut self, extents: &[ScannedFrameExtent]) -> crowdb_chunk_client::Result<()> { + if !self.delay.is_zero() { + tokio::time::sleep(self.delay).await; + } + if self.fail { + return Err(IoError::Internal("injected mapping failure".into())); + } + self.largest_batch = self.largest_batch.max(extents.len()); + self.records.extend_from_slice(extents); + Ok(()) + } +} diff --git a/lib/crowdb-chunk-client/tests/common/frame_revision.rs b/lib/crowdb-chunk-client/tests/common/frame_revision.rs new file mode 100644 index 000000000..1198ab9cf --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/frame_revision.rs @@ -0,0 +1,93 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::super::{segment, MemoryDiskIo, SequenceAllocator}; +use bytes::Bytes; +use crowdb_chunk_client::{ChunkReadPolicy, ChunkReader}; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkStrip, Location, MirrorStrip, Strip}; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::frame::{encode_frames, FrameMagic}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +pub struct TestFramedReader { + pub reader: ChunkReader, + pub locations: Vec, + pub allocator: Arc, + pub reads: Arc, +} + +impl TestFramedReader { + pub fn new(objects: &[(FrameMagic, Bytes)], window: usize) -> Self { + let id = ChunkId { high: 13, low: 37 }; + let mut bytes = Vec::new(); + let mut locations = Vec::new(); + let mut logical_offset = 0; + for (magic, payload) in objects { + let frame = encode_frames(*magic, id, payload, 42).unwrap().concat(); + locations.push(Location { + chunk_id: Some(id), + offset: bytes.len() as u64, + length: frame.len() as u64, + logical_offset, + logical_length: payload.len() as u64, + }); + logical_offset += payload.len() as u64; + bytes.extend(frame); + } + let kib = u32::try_from(bytes.len().div_ceil(1024)).unwrap(); + bytes.resize(kib as usize * 1024, 0); + let mut segment = segment(1); + segment.unit_count = kib; + let chunk = Chunk { + id: Some(id), + state: ChunkState::Sealed as i32, + capacity: kib, + sealed_length: kib, + strips: vec![ChunkStrip { + unit_kb: 1, + capacity: kib, + sealed_length: kib, + sealed_ts_ms: 1, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: vec![segment], + })), + ..ChunkStrip::default() + }], + ..Chunk::default() + }; + let allocator = Arc::new(SequenceAllocator { + queries: AtomicUsize::new(0), + first: chunk.clone(), + current: Mutex::new(chunk), + first_layout_validity_ms: 1_000, + full_reply: None, + full_calls: AtomicUsize::new(0), + }); + let reads = Arc::new(AtomicUsize::new(0)); + let disk = Arc::new(MemoryDiskIo { + shards: vec![(segment.disk_id.unwrap(), Bytes::from(bytes))], + failed: Vec::new(), + reads: reads.clone(), + }); + let reader = ChunkReader::new( + allocator.clone(), + disk, + ChunkReadPolicy { + stream_window_bytes: window, + ..ChunkReadPolicy::default() + }, + ) + .unwrap(); + Self { + reader, + locations, + allocator, + reads, + } + } + + pub fn queries(&self) -> usize { + self.allocator.queries.load(Ordering::Relaxed) + } +} diff --git a/lib/crowdb-chunk-client/tests/common/mirror_sync_allocator.rs b/lib/crowdb-chunk-client/tests/common/mirror_sync_allocator.rs new file mode 100644 index 000000000..7b92397ae --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/mirror_sync_allocator.rs @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_chunk_client::{ChunkAllocator, IoError, Result}; +use crowdb_protocol::chunkdb::rpc::{ + AllocateChunkRequest, AllocateChunkResponse, AllocateReplacementSegmentRequest, + AllocateReplacementSegmentResponse, AppendChunkRequest, AppendChunkResponse, Chunk, ChunkStrip, + DeleteChunkRequest, DeleteChunkResponse, MirrorStrip, QueryChunkRequest, QueryChunkResponse, + ReplaceChunkStripRangeRequest, ReplaceChunkStripRangeResponse, SealChunkRequest, SealChunkResponse, + Strip, +}; +use crowdb_protocol::common::{ChunkId, DiskId}; +use crowdb_protocol::diskdb::rpc::Segment; +use std::sync::atomic::{AtomicUsize, Ordering}; + +pub fn segment(high: u64) -> Segment { + Segment { + disk_id: Some(DiskId { high, low: 0 }), + unit_count: 1, + ..Default::default() + } +} + +pub fn chunk() -> Chunk { + Chunk { + id: Some(ChunkId { high: 7, low: 9 }), + modify_ts: 1, + strips: vec![ChunkStrip { + unit_kb: 4, + capacity: 4, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: (1..=3).map(segment).collect(), + })), + ..Default::default() + }], + ..Default::default() + } +} + +#[derive(Default)] +pub struct TestAllocator { + pub publications: AtomicUsize, +} + +#[async_trait] +impl ChunkAllocator for TestAllocator { + async fn allocate_chunk(&self, _: AllocateChunkRequest) -> Result { + Err(IoError::Internal("unexpected allocation".into())) + } + async fn append_chunk(&self, _: AppendChunkRequest) -> Result { + Err(IoError::Internal("unexpected growth".into())) + } + async fn seal_chunk(&self, _: SealChunkRequest) -> Result { + Err(IoError::Internal("unexpected seal".into())) + } + async fn delete_chunk(&self, _: DeleteChunkRequest) -> Result { + Err(IoError::Internal("unexpected deletion".into())) + } + async fn query_chunk(&self, _: QueryChunkRequest) -> Result { + Err(IoError::Internal("unexpected metadata query".into())) + } + async fn allocate_replacement_segment( + &self, + request: AllocateReplacementSegmentRequest, + ) -> Result { + assert_eq!(request.old_segment, Some(segment(3))); + assert_eq!(request.surviving_segments.len(), 2); + assert!(request.exclude_disk_ids.contains(&segment(3).disk_id.unwrap())); + Ok(AllocateReplacementSegmentResponse { + segment: Some(segment(4)), + }) + } + async fn replace_chunk_strip_range( + &self, + request: ReplaceChunkStripRangeRequest, + ) -> Result { + assert_eq!(request.old_strips, chunk().strips); + let Some(Strip::MirrorStrip(mirror)) = &request.replacement_strips[0].strip else { + panic!("mirror") + }; + assert_eq!(mirror.segments, vec![segment(1), segment(2), segment(4)]); + self.publications.fetch_add(1, Ordering::Relaxed); + Ok(ReplaceChunkStripRangeResponse { + chunk: Some(Chunk { + strips: request.replacement_strips, + modify_ts: 2, + ..chunk() + }), + }) + } +} diff --git a/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs b/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs new file mode 100644 index 000000000..2af4fa8b9 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/common/mirrored_frames.rs @@ -0,0 +1,118 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{DiskWriter, IoError}; +use crowdb_protocol::chunk_stream::StreamName; +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkState, ChunkStrip, ChunkType, MirrorStrip, Strip}; +use crowdb_protocol::common::DiskId; +use crowdb_protocol::diskdb::rpc::Segment; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + Arc, +}; + +use crate::fixture::TestObservation as Test; + +pub struct TestMirroredFrames { + pub chunk: Chunk, + pub disks: Arc, + pub body_length: u64, + pub capacity: u64, +} + +impl TestMirroredFrames { + pub const NAME: StreamName = StreamName { high: 31, low: 37 }; + pub fn new() -> Self { + let body = Test::group(0, &vec![0x5a; 100_000]); + let strips = body.len().div_ceil(8192); + let capacity = strips * 8192; + let chunk = Chunk { + id: Some(Test::CHUNK), + chunk_type: ChunkType::Stream as i32, + owner_key: Self::NAME.chunk_owner_key(), + writer_epoch: 1, + state: ChunkState::Active as i32, + acknowledged_cursor: 0, + strips: (0..strips) + .map(|index| ChunkStrip { + strip_sequence: u32::try_from(index).unwrap(), + chunk_offset: u32::try_from(index).unwrap() * 8, + capacity: 8, + unit_kb: 4, + strip: Some(Strip::MirrorStrip(MirrorStrip { + segments: (1..=2) + .map(|disk| Segment { + disk_id: Some(DiskId { high: disk, low: 1 }), + zone_index: u32::try_from(index).unwrap(), + unit_offset: 17 + index as u64 * 3, + unit_count: 2, + owner_chunk: Some(Test::CHUNK), + allocation_ts: 1, + }) + .collect(), + })), + ..ChunkStrip::default() + }) + .collect(), + ..Chunk::default() + }; + let body_length = body.len() as u64; + let disks = Arc::new(TestFrameDisks { + bytes: Test::backing(&body, capacity), + calls: AtomicUsize::new(0), + short: AtomicBool::new(false), + delay_ms: AtomicU64::new(0), + }); + Self { + chunk, + disks, + body_length, + capacity: capacity as u64, + } + } +} + +pub struct TestFrameDisks { + bytes: Bytes, + pub calls: AtomicUsize, + pub short: AtomicBool, + pub delay_ms: AtomicU64, +} + +#[async_trait] +impl DiskWriter for TestFrameDisks { + async fn write(&self, _: &Segment, _: u64, _: Bytes) -> crowdb_chunk_client::Result<()> { + Err(IoError::WriteFailed("read-only observation fixture".into())) + } + async fn write_at_byte_offset( + &self, + _: &Segment, + _: u64, + _: u64, + _: Bytes, + ) -> crowdb_chunk_client::Result<()> { + Err(IoError::WriteFailed("read-only observation fixture".into())) + } + async fn read( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + length: u32, + ) -> crowdb_chunk_client::Result { + self.calls.fetch_add(1, Ordering::Relaxed); + let delay = self.delay_ms.load(Ordering::Relaxed); + if delay > 0 { + tokio::time::sleep(std::time::Duration::from_millis(delay)).await; + } + assert_eq!(unit_bytes, 4096); + assert_eq!(segment.owner_chunk, Some(Test::CHUNK)); + assert_eq!(segment.unit_offset, 17 + u64::from(segment.zone_index) * 3); + assert!(offset + u64::from(length) <= 8192); + let start = usize::try_from(segment.zone_index).unwrap() * 8192 + usize::try_from(offset).unwrap(); + let end = start + length as usize - usize::from(self.short.load(Ordering::Relaxed)); + Ok(self.bytes.slice(start..end)) + } +} diff --git a/lib/crowdb-chunk-client/tests/frame_observation_test.rs b/lib/crowdb-chunk-client/tests/frame_observation_test.rs new file mode 100644 index 000000000..690ece842 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/frame_observation_test.rs @@ -0,0 +1,169 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/frame_observation.rs"] +mod fixture; +#[path = "common/mirrored_frames.rs"] +mod mirrored_fixture; + +use crowdb_chunk_client::{ + observe_frame_prefix, replay_frame_prefix, select_frame_prefix, FrameObservationError, FrameWindowReader, +}; +use fixture::{TestExtentSink, TestObservation as Test, TestWindowReader}; +use std::{sync::Arc, time::Duration}; +use tokio::time::Instant; + +#[path = "frame_observation_test/faults.rs"] +mod faults; +#[path = "frame_observation_test/mirrored.rs"] +mod mirrored; + +#[tokio::test] +async fn shortest_complete_replica_prefix_is_selected_and_replayed_without_history_buffers() { + let first = Test::group(0, &vec![0x41; 1024]); + let second = Test::group(first.len(), &vec![0x42; 100_000]); + let longer = [first.as_slice(), second.as_slice()].concat(); + let capacity = longer.len() + 512; + let short = Arc::new(TestWindowReader::new(Test::backing(&first, capacity))); + let long = Arc::new(TestWindowReader::new(Test::backing(&longer, capacity))); + let readers: Vec> = vec![short.clone(), long.clone()]; + let deadline = Instant::now() + Duration::from_secs(5); + let selected = select_frame_prefix(&readers, Test::config(capacity), 4096, deadline) + .await + .unwrap(); + assert_eq!(selected.selected_index(), 0); + assert_eq!(selected.healthy_replicas(), 2); + assert!(selected.failures().is_empty()); + assert_eq!(selected.observation().summary().physical_end, first.len() as u64); + assert_eq!(selected.observation().summary().logical_end, 1024); + let mut sink = TestExtentSink::default(); + let replay = replay_frame_prefix(short.as_ref(), selected.observation(), deadline, &mut sink) + .await + .unwrap(); + assert_eq!(sink.records.len(), 1); + assert_eq!(sink.records[0].logical_length, 1024); + assert_eq!(short.last_end(), first.len() as u64); + assert!(short.largest() <= 4096 && long.largest() <= 4096); + assert_eq!(replay.reads(), 1); + assert_eq!(replay.read_bytes(), first.len() as u64); +} + +#[tokio::test] +async fn a_mid_scan_failure_and_a_corrupt_replica_never_force_a_shorter_prefix() { + let first = Test::group(0, &vec![0x41; 1024]); + let second = Test::group(first.len(), &vec![0x42; 100_000]); + let prefix = [first.as_slice(), second.as_slice()].concat(); + let capacity = prefix.len() + 512; + let good = Arc::new(TestWindowReader::new(Test::backing(&prefix, capacity))); + let mut failed = TestWindowReader::new(Test::backing(&prefix, capacity)); + failed.fail_at = Some(8192); + let mut corrupt = prefix.clone(); + corrupt[first.len() + 27] ^= 1; + let readers: Vec> = vec![ + good, + Arc::new(failed), + Arc::new(TestWindowReader::new(Test::backing(&corrupt, capacity))), + ]; + let selected = select_frame_prefix( + &readers, + Test::config(capacity), + 4096, + Instant::now() + Duration::from_secs(5), + ) + .await + .unwrap(); + assert_eq!(selected.selected_index(), 0); + assert_eq!(selected.healthy_replicas(), 1); + assert_eq!(selected.failures().len(), 2); + assert_eq!(selected.observation().summary().physical_end, prefix.len() as u64); + assert_eq!(selected.observation().summary().logical_end, 101_024); +} + +#[tokio::test] +async fn exact_capacity_replay_emits_bounded_location_batches_without_an_extra_read() { + let mut bytes = Vec::new(); + for _ in 0..3000 { + bytes.extend(Test::group(bytes.len(), b"a")); + } + let capacity = bytes.len(); + let source = TestWindowReader::new(Test::backing(&bytes, capacity)); + let deadline = Instant::now() + Duration::from_secs(5); + let observed = observe_frame_prefix(&source, Test::config(capacity), 1024 * 1024, deadline) + .await + .unwrap(); + let mut sink = TestExtentSink::default(); + replay_frame_prefix(&source, &observed, deadline, &mut sink) + .await + .unwrap(); + assert_eq!(sink.records.len(), 3000); + assert_eq!(sink.largest_batch, 1024); + assert_eq!(source.calls(), 2); + assert_eq!(source.last_end(), capacity as u64); +} + +#[tokio::test(start_paused = true)] +async fn one_shared_io_deadline_bounds_concurrent_replicas_without_per_replica_resets() { + let bytes = Test::group(0, b"a"); + let capacity = bytes.len() + 14; + let mut quick = TestWindowReader::new(Test::backing(&bytes, capacity)); + quick.delay = Duration::from_millis(1); + let mut slow = TestWindowReader::new(Test::backing(&bytes, capacity)); + slow.delay = Duration::from_secs(60); + let readers: Vec> = vec![Arc::new(quick), Arc::new(slow)]; + let start = Instant::now(); + let selected = select_frame_prefix( + &readers, + Test::config(capacity), + 1024, + start + Duration::from_millis(20), + ) + .await + .unwrap(); + assert!(start.elapsed() < Duration::from_millis(20)); + assert_eq!(selected.healthy_replicas(), 1); + assert_eq!(selected.failures().len(), 1); + assert_eq!(selected.observation().summary().physical_end, bytes.len() as u64); + let mut sink = TestExtentSink::default(); + replay_frame_prefix( + readers[selected.selected_index()].as_ref(), + selected.observation(), + start + Duration::from_millis(20), + &mut sink, + ) + .await + .unwrap(); + assert_eq!(sink.records.len(), 1); + assert!(start.elapsed() < Duration::from_millis(20)); +} + +#[tokio::test(start_paused = true)] +async fn a_hung_mirror_leaves_time_to_compare_unequal_healthy_ends_and_replay() { + let first = Test::group(0, b"first"); + let longer = [first.clone(), Test::group(first.len(), b"second")].concat(); + let capacity = longer.len() + 14; + let mut readers: Vec> = Vec::new(); + for (data, delay) in [ + (first.as_slice(), Duration::from_millis(1)), + (longer.as_slice(), Duration::from_millis(1)), + (longer.as_slice(), Duration::from_secs(60)), + ] { + let mut reader = TestWindowReader::new(Test::backing(data, capacity)); + reader.delay = delay; + readers.push(Arc::new(reader)); + } + let start = Instant::now(); + let deadline = start + Duration::from_millis(20); + let selected = select_frame_prefix(&readers, Test::config(capacity), 1024, deadline) + .await + .unwrap(); + assert_eq!(selected.selected_index(), 0); + assert_eq!(selected.healthy_replicas(), 2); + assert_eq!(selected.failures().len(), 1); + let mut sink = TestExtentSink::default(); + replay_frame_prefix(readers[0].as_ref(), selected.observation(), deadline, &mut sink) + .await + .unwrap(); + assert_eq!(sink.records.len(), 1); + assert_eq!(sink.records[0].logical_length, 5); + assert!(start.elapsed() < Duration::from_millis(20)); +} diff --git a/lib/crowdb-chunk-client/tests/frame_observation_test/faults.rs b/lib/crowdb-chunk-client/tests/frame_observation_test/faults.rs new file mode 100644 index 000000000..cb26c5fcf --- /dev/null +++ b/lib/crowdb-chunk-client/tests/frame_observation_test/faults.rs @@ -0,0 +1,161 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn conflicting_valid_complete_contents_fail_instead_of_waiting_for_an_arbiter() { + let left = Test::group(0, b"left"); + let right = Test::group(0, b"next"); + let capacity = left.len() + 14; + let readers: Vec> = vec![ + Arc::new(TestWindowReader::new(Test::backing(&left, capacity))), + Arc::new(TestWindowReader::new(Test::backing(&right, capacity))), + ]; + assert!(matches!( + select_frame_prefix( + &readers, + Test::config(capacity), + 1024, + Instant::now() + Duration::from_secs(5) + ) + .await, + Err(FrameObservationError::ConflictingPrefixes { .. }) + )); +} + +#[tokio::test] +async fn unequal_valid_ends_still_require_identical_overlapping_contents() { + let short = Test::group(0, b"left"); + let first = Test::group(0, b"next"); + let longer = [first.as_slice(), Test::group(first.len(), b"tail").as_slice()].concat(); + let capacity = longer.len() + 14; + let readers: Vec> = vec![ + Arc::new(TestWindowReader::new(Test::backing(&short, capacity))), + Arc::new(TestWindowReader::new(Test::backing(&longer, capacity))), + ]; + assert!(matches!( + select_frame_prefix( + &readers, + Test::config(capacity), + 1024, + Instant::now() + Duration::from_secs(5) + ) + .await, + Err(FrameObservationError::ConflictingPrefixes { .. }) + )); +} + +#[tokio::test] +async fn no_healthy_observation_or_a_short_read_is_explicitly_unavailable() { + let bytes = Test::group(0, b"a"); + let capacity = bytes.len() + 14; + let mut failed = TestWindowReader::new(Test::backing(&bytes, capacity)); + failed.fail_at = Some(0); + let mut short = TestWindowReader::new(Test::backing(&bytes, capacity)); + short.short_at = Some(0); + let readers: Vec> = vec![Arc::new(failed), Arc::new(short)]; + let result = select_frame_prefix( + &readers, + Test::config(capacity), + 1024, + Instant::now() + Duration::from_secs(5), + ) + .await; + let Err(FrameObservationError::NoHealthyReplica { failures }) = result else { + panic!("failed reads must not become empty candidates") + }; + assert_eq!(failures.len(), 2); +} + +#[tokio::test] +async fn replay_rejects_changed_contents_and_sink_failure() { + let bytes = Test::group(0, b"a"); + let source = TestWindowReader::new(Test::backing(&bytes, bytes.len())); + let deadline = Instant::now() + Duration::from_secs(5); + let observed = observe_frame_prefix(&source, Test::config(bytes.len()), 1024, deadline) + .await + .unwrap(); + let changed = Test::group(0, b"b"); + let changed = TestWindowReader::new(Test::backing(&changed, changed.len())); + let mut sink = TestExtentSink::default(); + assert!(matches!( + replay_frame_prefix(&changed, &observed, deadline, &mut sink).await, + Err(FrameObservationError::ChangedPrefix) + )); + let mut sink = TestExtentSink { + fail: true, + ..TestExtentSink::default() + }; + assert!(matches!( + replay_frame_prefix(&source, &observed, deadline, &mut sink).await, + Err(FrameObservationError::Sink(_)) + )); +} + +#[tokio::test(start_paused = true)] +async fn mapping_io_and_past_deadlines_cannot_create_a_new_budget() { + let bytes = Test::group(0, b"a"); + let source = TestWindowReader::new(Test::backing(&bytes, bytes.len())); + let deadline = Instant::now() + Duration::from_millis(20); + let observed = observe_frame_prefix(&source, Test::config(bytes.len()), 1024, deadline) + .await + .unwrap(); + let mut sink = TestExtentSink { + delay: Duration::from_secs(1), + ..TestExtentSink::default() + }; + assert!(matches!( + replay_frame_prefix(&source, &observed, deadline, &mut sink).await, + Err(FrameObservationError::DeadlineExpired) + )); + let calls = source.calls(); + assert!(matches!( + observe_frame_prefix(&source, Test::config(bytes.len()), 1024, deadline).await, + Err(FrameObservationError::DeadlineExpired) + )); + assert_eq!(source.calls(), calls); +} + +#[tokio::test(start_paused = true)] +async fn a_failed_or_hung_comparison_excludes_that_replica_and_preserves_replay_time() { + for stall in [false, true] { + let first = Test::group(0, b"first"); + let longer = [first.clone(), Test::group(first.len(), b"second")].concat(); + let capacity = longer.len() + 14; + let short = Arc::new(TestWindowReader::new(Test::backing(&first, capacity))); + let mut long = TestWindowReader::new(Test::backing(&longer, capacity)); + if stall { + long.stall_on_call = Some(2); + } else { + long.fail_on_call = Some(2); + } + let readers: Vec> = vec![short.clone(), Arc::new(long)]; + let start = Instant::now(); + let deadline = start + Duration::from_millis(20); + let selected = select_frame_prefix(&readers, Test::config(capacity), 1024, deadline) + .await + .unwrap(); + assert_eq!(selected.selected_index(), 0); + assert_eq!(selected.healthy_replicas(), 1); + assert_eq!(selected.failures().len(), 1); + assert!(selected.failures()[0].contains("replica 1 comparison")); + let mut sink = TestExtentSink::default(); + replay_frame_prefix(short.as_ref(), selected.observation(), deadline, &mut sink) + .await + .unwrap(); + assert_eq!(sink.records.len(), 1); + assert!(start.elapsed() < Duration::from_millis(20)); + } +} + +#[tokio::test(start_paused = true)] +async fn expired_selection_does_not_start_physical_reads() { + let reader = Arc::new(TestWindowReader::new(Test::backing(&[], 128))); + let readers: Vec> = vec![reader.clone()]; + assert!(matches!( + select_frame_prefix(&readers, Test::config(128), 1024, Instant::now()).await, + Err(FrameObservationError::DeadlineExpired) + )); + assert_eq!(reader.calls(), 0); +} diff --git a/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs b/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs new file mode 100644 index 000000000..ae512c44f --- /dev/null +++ b/lib/crowdb-chunk-client/tests/frame_observation_test/mirrored.rs @@ -0,0 +1,189 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_client::MirroredFrameReaders; +use crowdb_protocol::chunkdb::rpc::Strip; +use std::sync::atomic::Ordering; + +use super::*; +use crate::mirrored_fixture::TestMirroredFrames as Layout; + +#[tokio::test] +async fn pinned_layout_reads_grouped_data_beyond_old_cursor_across_frames_and_strips() { + let layout = Layout::new(); + assert_eq!(layout.chunk.acknowledged_cursor, 0); + let mut chunk = Arc::new(layout.chunk); + let deadline = Instant::now() + Duration::from_secs(5); + let readers = + MirroredFrameReaders::new(Arc::clone(&chunk), Layout::NAME, layout.disks.clone(), deadline).unwrap(); + Arc::make_mut(&mut chunk).strips.clear(); + assert_eq!(readers.capacity(), layout.capacity); + assert_eq!(readers.deadline(), deadline); + let config = readers.scan_config(0, 700, 0, 64 * 1024 * 1024).unwrap(); + let selected = select_frame_prefix(readers.readers(), config, 65536, deadline) + .await + .unwrap(); + assert_eq!(selected.healthy_replicas(), 2); + assert_eq!(selected.observation().summary().physical_end, layout.body_length); + assert_eq!(selected.observation().summary().logical_end, 100_700); + let mut sink = TestExtentSink::default(); + replay_frame_prefix( + readers.readers()[selected.selected_index()].as_ref(), + selected.observation(), + deadline, + &mut sink, + ) + .await + .unwrap(); + assert_eq!(sink.records.len(), 1); + assert_eq!(sink.records[0].physical_length, layout.body_length); + assert_eq!(sink.records[0].logical_offset, 700); + assert_eq!(sink.records[0].logical_length, 100_000); +} + +#[tokio::test] +async fn unavailable_replica_is_excluded_without_reading_or_mutating_its_layout() { + let mut layout = Layout::new(); + let Some(Strip::MirrorStrip(mirror)) = &layout.chunk.strips[0].strip else { + panic!("mirror missing") + }; + let unavailable = mirror.segments[0]; + layout.chunk.strips[0].unavailable_segments.push(unavailable); + let deadline = Instant::now() + Duration::from_secs(5); + let strips = layout.chunk.strips.len(); + let readers = MirroredFrameReaders::new( + Arc::new(layout.chunk), + Layout::NAME, + layout.disks.clone(), + deadline, + ) + .unwrap(); + let selected = select_frame_prefix( + readers.readers(), + readers.scan_config(0, 0, 0, 64 * 1024 * 1024).unwrap(), + 65536, + deadline, + ) + .await + .unwrap(); + assert_eq!(selected.selected_index(), 1); + assert_eq!(selected.healthy_replicas(), 1); + assert_eq!(layout.disks.calls.load(Ordering::Relaxed), strips); +} + +#[tokio::test(start_paused = true)] +async fn expired_layout_clones_and_inflight_reads_cannot_outlive_their_authority() { + let layout = Layout::new(); + layout.disks.delay_ms.store(40, Ordering::Relaxed); + let deadline = Instant::now() + Duration::from_millis(20); + let readers = MirroredFrameReaders::new( + Arc::new(layout.chunk), + Layout::NAME, + layout.disks.clone(), + deadline, + ) + .unwrap(); + let reader = Arc::clone(&readers.readers()[0]); + assert!(reader.read_window(0, 512).await.is_err()); + assert_eq!(Instant::now(), deadline); + let calls = layout.disks.calls.load(Ordering::Relaxed); + assert!(reader.read_window(0, 512).await.is_err()); + assert_eq!(layout.disks.calls.load(Ordering::Relaxed), calls); +} + +#[tokio::test] +async fn scope_bounds_and_short_segment_reads_fail_without_fabricated_bytes() { + let layout = Layout::new(); + let deadline = Instant::now() + Duration::from_secs(5); + let readers = MirroredFrameReaders::new( + Arc::new(layout.chunk), + Layout::NAME, + layout.disks.clone(), + deadline, + ) + .unwrap(); + let reader = &readers.readers()[0]; + assert!(reader.read_window(layout.capacity - 1, 2).await.is_err()); + assert!(reader.read_window(0, 1024 * 1024 + 1).await.is_err()); + assert_eq!(layout.disks.calls.load(Ordering::Relaxed), 0); + layout.disks.short.store(true, Ordering::Relaxed); + assert!(reader.read_window(0, 512).await.is_err()); + assert_eq!(layout.disks.calls.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn wrong_owner_gapped_layout_and_missing_allocation_identity_fail_before_io() { + let layout = Layout::new(); + let deadline = Instant::now() + Duration::from_secs(5); + let mut wrong = layout.chunk.clone(); + wrong.owner_key.clear(); + assert!( + MirroredFrameReaders::new(Arc::new(wrong), Layout::NAME, layout.disks.clone(), deadline).is_err() + ); + let mut gap = layout.chunk.clone(); + gap.strips[1].chunk_offset += 1; + assert!(MirroredFrameReaders::new(Arc::new(gap), Layout::NAME, layout.disks.clone(), deadline).is_err()); + let mut wrong = layout.chunk; + let Some(Strip::MirrorStrip(mirror)) = &mut wrong.strips[0].strip else { + panic!("mirror missing") + }; + mirror.segments[0].allocation_ts = 0; + assert!( + MirroredFrameReaders::new(Arc::new(wrong), Layout::NAME, layout.disks.clone(), deadline).is_err() + ); + assert_eq!(layout.disks.calls.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn the_current_five_copy_configuration_is_observed_without_an_artificial_three_copy_limit() { + use crowdb_protocol::common::DiskId; + let mut layout = Layout::new(); + for strip in &mut layout.chunk.strips { + let Some(Strip::MirrorStrip(mirror)) = &mut strip.strip else { + panic!("mirror missing") + }; + let original = mirror.segments[0]; + for high in 3..=5 { + mirror.segments.push(crowdb_protocol::diskdb::rpc::Segment { + disk_id: Some(DiskId { high, low: 1 }), + ..original + }); + } + } + let deadline = Instant::now() + Duration::from_secs(5); + let readers = + MirroredFrameReaders::new(Arc::new(layout.chunk), Layout::NAME, layout.disks, deadline).unwrap(); + let selected = select_frame_prefix( + readers.readers(), + readers.scan_config(0, 0, 0, 64 * 1024 * 1024).unwrap(), + 65536, + deadline, + ) + .await + .unwrap(); + assert_eq!(selected.healthy_replicas(), 5); + assert_eq!(selected.observation().summary().physical_end, layout.body_length); +} + +#[tokio::test] +async fn colocated_replicas_require_disjoint_allocations() { + let layout = Layout::new(); + for offset in [0, 1, 2] { + let mut chunk = layout.chunk.clone(); + for strip in &mut chunk.strips { + let Some(Strip::MirrorStrip(mirror)) = &mut strip.strip else { + panic!("mirror missing") + }; + mirror.segments[1].disk_id = mirror.segments[0].disk_id; + mirror.segments[1].unit_offset = mirror.segments[0].unit_offset + offset; + } + let result = MirroredFrameReaders::new( + Arc::new(chunk), + Layout::NAME, + layout.disks.clone(), + Instant::now() + Duration::from_secs(5), + ); + assert_eq!(result.is_ok(), offset == 2, "replica unit offset {offset}"); + } + assert_eq!(layout.disks.calls.load(Ordering::Relaxed), 0); +} diff --git a/lib/crowdb-chunk-client/tests/mirror_synchronized_rpc_test.rs b/lib/crowdb-chunk-client/tests/mirror_synchronized_rpc_test.rs new file mode 100644 index 000000000..ca801e9e0 --- /dev/null +++ b/lib/crowdb-chunk-client/tests/mirror_synchronized_rpc_test.rs @@ -0,0 +1,208 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/mirror_sync_allocator.rs"] +mod allocator; +#[path = "../../crowdb-diskio-client/tests/common/synchronized_peer.rs"] +#[allow(dead_code)] +mod peer; + +use bytes::Bytes; +use crowdb_chunk_client::{ChunkClientMetrics, ChunkIoClient, MirrorStripFlow, RoutedDiskWriter}; +use crowdb_common::metrics::MetricsRegistry; +use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig}; +use crowdb_protocol::chunkdb::rpc::ChunkType; +use crowdb_protocol::diskio_fb::FBDiskIoRetCode; +use peer::TestDiskioPeer; +use std::sync::{atomic::Ordering, Arc}; + +#[tokio::test] +async fn metrics_and_routing_forward_one_synchronized_rpc_per_required_replica() { + run_flow(false).await; +} + +#[tokio::test] +async fn replacement_is_synchronized_before_metadata_publication_and_completion() { + run_flow(true).await; +} + +#[tokio::test] +async fn metrics_and_routing_preserve_the_original_owner_deadline() { + use std::time::{Duration, Instant}; + for initialize in [false, true] { + let peer = TestDiskioPeer::start(true, true, FBDiskIoRetCode::Success).await; + let diskio = Arc::new( + DiskioClient::connect_for_tests( + vec![peer.route(DiskId::new(1, 0))], + DiskioClientConfig::default(), + ) + .unwrap(), + ); + let metrics = Arc::new(ChunkClientMetrics::register(&mut MetricsRegistry::new())); + let client = ChunkIoClient::from_parts( + Arc::new(allocator::TestAllocator::default()), + Arc::new(RoutedDiskWriter::from_client_for_tests(diskio)), + ChunkType::Stream, + ) + .with_metrics(&metrics); + let (_, writer) = client.storage_parts(); + let deadline = Instant::now() + Duration::from_millis(100); + let operation = tokio::spawn(async move { + let segment = allocator::segment(1); + let data = Bytes::from_static(&[7; 32]); + if initialize { + writer + .initialize_synchronized_until(&segment, 4096, 0, data, deadline) + .await + } else { + writer + .write_synchronized_views_until(&segment, 4096, 0, vec![data], deadline) + .await + } + }); + peer.wait_for_write().await; + let result = tokio::time::timeout(Duration::from_secs(1), operation) + .await + .expect("owner deadline must pass through both decorators") + .unwrap(); + assert!(result.is_err()); + assert_eq!(peer.counts.writes.load(Ordering::Acquire), 1); + } +} + +#[tokio::test] +async fn metrics_and_routing_preserve_single_attempt_initialization() { + let peer = TestDiskioPeer::start(true, false, FBDiskIoRetCode::ConnectionError).await; + let diskio = Arc::new( + DiskioClient::connect_for_tests( + vec![peer.route(DiskId::new(1, 0))], + DiskioClientConfig { + retry_attempts: 3, + ..Default::default() + }, + ) + .unwrap(), + ); + let metrics = Arc::new(ChunkClientMetrics::register(&mut MetricsRegistry::new())); + let client = ChunkIoClient::from_parts( + Arc::new(allocator::TestAllocator::default()), + Arc::new(RoutedDiskWriter::from_client_for_tests(diskio)), + ChunkType::Stream, + ) + .with_metrics(&metrics); + let (_, writer) = client.storage_parts(); + assert!( + writer + .initialize_synchronized_at_byte_offset( + &allocator::segment(1), + 4096, + 0, + Bytes::from_static(&[7; 32]), + ) + .await + .is_err() + ); + assert_eq!(peer.counts.writes.load(Ordering::Acquire), 1); + assert_eq!(peer.counts.sync_requests.load(Ordering::Acquire), 1); + assert_eq!(peer.counts.fsyncs.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn metrics_and_routing_preserve_ordered_drain_completion() { + let peer = TestDiskioPeer::start(true, true, FBDiskIoRetCode::Success).await; + let diskio = Arc::new( + DiskioClient::connect_for_tests(vec![peer.route(DiskId::new(1, 0))], DiskioClientConfig::default()) + .unwrap(), + ); + let metrics = Arc::new(ChunkClientMetrics::register(&mut MetricsRegistry::new())); + let client = ChunkIoClient::from_parts( + Arc::new(allocator::TestAllocator::default()), + Arc::new(RoutedDiskWriter::from_client_for_tests(diskio)), + ChunkType::Stream, + ) + .with_metrics(&metrics); + let (_, writer) = client.storage_parts(); + let operation = tokio::spawn(async move { + writer + .drain_segment( + &allocator::segment(1), + 4096, + std::time::Instant::now() + std::time::Duration::from_secs(3), + ) + .await + }); + peer.wait_for_write().await; + assert!(!operation.is_finished()); + peer.gate.notify_one(); + operation.await.unwrap().unwrap(); + assert_eq!(peer.counts.writes.load(Ordering::Acquire), 1); + assert_eq!(peer.counts.sync_requests.load(Ordering::Acquire), 1); + assert_eq!(peer.counts.bytes.load(Ordering::Acquire), 0); + assert_eq!(peer.counts.fsyncs.load(Ordering::Acquire), 0); +} + +async fn run_flow(repair: bool) { + let peers = [ + TestDiskioPeer::start(true, false, FBDiskIoRetCode::Success).await, + TestDiskioPeer::start(true, false, FBDiskIoRetCode::Success).await, + TestDiskioPeer::start( + true, + !repair, + if repair { + FBDiskIoRetCode::IoError + } else { + FBDiskIoRetCode::Success + }, + ) + .await, + TestDiskioPeer::start(true, repair, FBDiskIoRetCode::Success).await, + ]; + let routes = peers + .iter() + .enumerate() + .map(|(index, peer)| peer.route(DiskId::new(index as u64 + 1, 0))) + .collect(); + let diskio = Arc::new( + DiskioClient::connect_for_tests( + routes, + DiskioClientConfig { + retry_attempts: 1, + ..Default::default() + }, + ) + .unwrap(), + ); + let allocator = Arc::new(allocator::TestAllocator::default()); + let metrics = Arc::new(ChunkClientMetrics::register(&mut MetricsRegistry::new())); + let client = ChunkIoClient::from_parts( + allocator.clone(), + Arc::new(RoutedDiskWriter::from_client_for_tests(diskio)), + ChunkType::Stream, + ) + .with_metrics(&metrics); + let (allocation, writer) = client.storage_parts(); + let flow = MirrorStripFlow::new(allocation, writer, 1, 1, true).unwrap(); + let operation = tokio::spawn(async move { + let mut chunk = allocator::chunk(); + let data = Bytes::from_static(&[7; 32]); + flow.write(&mut chunk, 0, 0, data.clone(), data, &mut None) + .await + .unwrap(); + }); + let gated = if repair { 3 } else { 2 }; + peers[gated].wait_for_write().await; + assert!(!operation.is_finished()); + assert_eq!(allocator.publications.load(Ordering::Relaxed), 0); + peers[gated].gate.notify_one(); + operation.await.unwrap(); + for (index, peer) in peers.iter().enumerate() { + let expected = usize::from(index < 3 || repair); + assert_eq!(peer.counts.writes.load(Ordering::Relaxed), expected); + assert_eq!(peer.counts.sync_requests.load(Ordering::Relaxed), expected); + assert_eq!(peer.counts.fsyncs.load(Ordering::Relaxed), 0); + } + assert_eq!( + allocator.publications.load(Ordering::Relaxed), + usize::from(repair) + ); +} diff --git a/lib/crowdb-chunk-client/tests/readiness_test.rs b/lib/crowdb-chunk-client/tests/readiness_test.rs new file mode 100644 index 000000000..9c6d96bba --- /dev/null +++ b/lib/crowdb-chunk-client/tests/readiness_test.rs @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; +use std::time::Duration; + +use crowdb_chunk_client::{client::observe_readiness_phase_for_tests, metrics::ReadinessMetrics}; +use tokio::time::Instant; + +#[tokio::test(start_paused = true)] +async fn phase_deadlines_share_the_remaining_budget_and_identify_the_dependency() { + let metrics = ReadinessMetrics::default(); + let deadline = Instant::now() + Duration::from_millis(750); + tokio::time::advance(Duration::from_millis(600)).await; + let result = observe_readiness_phase_for_tests(&metrics, deadline, async { + tokio::time::sleep(Duration::from_millis(200)).await; + Ok(()) + }) + .await; + assert!(result.unwrap_err().to_string().contains("DiskIO read exceeded")); + assert_eq!(Instant::now(), deadline); + let snapshot = metrics.snapshot().diskio; + assert_eq!(snapshot.observations, 1); + assert_eq!(snapshot.errors, 1); + assert_eq!(snapshot.inflight, 0); +} + +#[tokio::test] +async fn cancelled_probes_record_latency_and_error_without_leaking_inflight_work() { + let metrics = Arc::new(ReadinessMetrics::default()); + let worker_metrics = Arc::clone(&metrics); + let (started, observing) = tokio::sync::oneshot::channel(); + let worker = tokio::spawn(async move { + observe_readiness_phase_for_tests(&worker_metrics, Instant::now() + Duration::from_secs(10), async { + started.send(()).unwrap(); + std::future::pending().await + }) + .await + }); + observing.await.unwrap(); + assert_eq!(metrics.snapshot().diskio.inflight, 1); + worker.abort(); + assert!(worker.await.unwrap_err().is_cancelled()); + let snapshot = metrics.snapshot().diskio; + assert_eq!(snapshot.observations, 1); + assert_eq!(snapshot.errors, 1); + assert_eq!(snapshot.inflight, 0); + observe_readiness_phase_for_tests(&metrics, Instant::now() + Duration::from_secs(1), async { + Ok(()) + }) + .await + .unwrap(); + assert_eq!(metrics.snapshot().diskio.observations, 2); + assert_eq!(metrics.snapshot().diskio.errors, 1); +} diff --git a/lib/crowdb-chunk-client/tests/reclamation_test.rs b/lib/crowdb-chunk-client/tests/reclamation_test.rs index 221b458f5..ae047e53f 100644 --- a/lib/crowdb-chunk-client/tests/reclamation_test.rs +++ b/lib/crowdb-chunk-client/tests/reclamation_test.rs @@ -5,8 +5,7 @@ use crowdb_chunk_client::{reclaim_location, ChunkAllocator, IoError, ReclaimOutc use crowdb_protocol::chunkdb::rpc::{ AllocateChunkRequest, AllocateChunkResponse, AppendChunkRequest, AppendChunkResponse, Chunk, ChunkState, DeleteChunkRangeRequest, DeleteChunkRangeResponse, DeleteChunkRequest, DeleteChunkResponse, Location, - QueryChunkRequest, QueryChunkResponse, SealChunkRequest, SealChunkResponse, UpdateChunkStripRequest, - UpdateChunkStripResponse, + QueryChunkRequest, QueryChunkResponse, SealChunkRequest, SealChunkResponse, }; use crowdb_protocol::common::ChunkId; @@ -62,10 +61,6 @@ impl ChunkAllocator for TestAllocator { unreachable!() } - async fn update_chunk_strip(&self, _: UpdateChunkStripRequest) -> Result { - unreachable!() - } - async fn query_chunk(&self, request: QueryChunkRequest) -> Result { assert_eq!(request.chunk_id, self.chunk.id); Ok(QueryChunkResponse { diff --git a/lib/crowdb-chunk-client/tests/small_object_test.rs b/lib/crowdb-chunk-client/tests/small_object_test.rs index a60ae2c0f..96fa371ed 100644 --- a/lib/crowdb-chunk-client/tests/small_object_test.rs +++ b/lib/crowdb-chunk-client/tests/small_object_test.rs @@ -9,8 +9,7 @@ use std::time::Duration; use async_trait::async_trait; use bytes::Bytes; use crowdb_chunk_client::{ - ChunkAllocator, ChunkIoClient, ChunkIoWriter, DiskWriter, IoError, MirrorChunkWriter, Result, - SmallWritePolicy, + ChunkAllocator, ChunkIoClient, ChunkIoWriter, DiskWriter, IoError, Result, SmallWritePolicy, }; use crowdb_protocol::chunkdb::rpc::{ AdvanceChunkWriteRequest, AdvanceChunkWriteResponse, AllocateChunkRequest, AllocateChunkResponse, @@ -20,14 +19,13 @@ use crowdb_protocol::chunkdb::rpc::{ MutateStripReservationRequest, MutateStripReservationResponse, QueryChunkRequest, QueryChunkResponse, ReplaceChunkStripRangeRequest, ReplaceChunkStripRangeResponse, ReserveStripGroupRequest, ReserveStripGroupResponse, SealChunkRequest, SealChunkResponse, Strip, StripReservationAction, - StripReservationGroup, StripReservationState, StripType, UpdateChunkStripRequest, - UpdateChunkStripResponse, + StripReservationGroup, StripReservationState, StripType, }; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::Segment; -use crowdb_protocol::frame::{parse_frame, MAX_FRAME_PAYLOAD_BYTES}; +use crowdb_protocol::frame::{parse_frame, MAX_FRAME_V2_PAYLOAD_BYTES}; -const MAX_SMALL: usize = MAX_FRAME_PAYLOAD_BYTES; +const MAX_SMALL: usize = MAX_FRAME_V2_PAYLOAD_BYTES; #[path = "common/small_durable.rs"] mod durable; @@ -36,7 +34,7 @@ mod durable; mod native; fn frame_bytes(payload_bytes: usize) -> u64 { - u64::try_from(payload_bytes).unwrap() + 34 + u64::try_from(payload_bytes).unwrap() + 35 } #[derive(Default)] @@ -310,7 +308,8 @@ impl ChunkAllocator for MockAllocator { state.seals += 1; let chunk = state.chunks.get_mut(&(id.high, id.low)).unwrap(); chunk.state = ChunkState::Sealed as i32; - chunk.sealed_length = req.seal_length; + chunk.sealed_length = u32::try_from(req.seal_bytes.div_ceil(1024)).unwrap(); + chunk.acknowledged_cursor = req.seal_bytes; Ok(SealChunkResponse { chunk: Some(chunk.clone()), }) @@ -327,10 +326,6 @@ impl ChunkAllocator for MockAllocator { }) } - async fn update_chunk_strip(&self, _req: UpdateChunkStripRequest) -> Result { - unreachable!() - } - async fn query_chunk(&self, req: QueryChunkRequest) -> Result { let id = req.chunk_id.unwrap(); let state = self.state.lock().unwrap(); @@ -1401,101 +1396,5 @@ async fn small_object_initialization_failure_retires_prepared_pipelines() { assert_eq!(allocator.snapshot().4, 1); } -#[tokio::test] -async fn direct_mirror_chunk_writer_replicates_advances_and_seals() { - let allocator = Arc::new(MockAllocator::default()); - let disk = Arc::new(RecordingDiskWriter::default()); - let allocator_trait: Arc = allocator.clone(); - let disk_trait: Arc = disk.clone(); - let mut writer = MirrorChunkWriter::allocate( - allocator_trait, - disk_trait, - crowdb_protocol::chunk_stream::StreamName { high: 1, low: 2 }, - 44, - 30_000, - crowdb_protocol::chunk_stream::StreamPurpose::Stream, - ) - .await - .unwrap(); - - assert_eq!(writer.cursor(), 0); - assert_eq!( - writer.append(Bytes::from_static(b"stream")).await.unwrap(), - (0, 6) - ); - assert_eq!(writer.cursor(), 6); - assert_eq!(disk.calls(), 2); - assert_eq!(allocator.snapshot().2, 1); - - writer.seal().await.unwrap(); - assert_eq!(allocator.snapshot().3, 1); -} - -#[tokio::test] -async fn direct_mirror_chunk_writer_supports_five_copies() { - let allocator: Arc = Arc::new(MockAllocator::default()); - let disk = Arc::new(RecordingDiskWriter::default()); - let disk_writer: Arc = disk.clone(); - let mut writer = MirrorChunkWriter::allocate_with_copy_count( - allocator, - disk_writer, - crowdb_protocol::chunk_stream::StreamName { high: 1, low: 5 }, - 44, - 30_000, - 5, - crowdb_protocol::chunk_stream::StreamPurpose::Stream, - ) - .await - .unwrap(); - writer.append(Bytes::from_static(b"stream")).await.unwrap(); - assert_eq!(disk.calls(), 5); -} - -#[tokio::test] -async fn direct_mirror_chunk_writer_accepts_protected_degraded_layout() { - let allocator: Arc = Arc::new(MockAllocator::default()); - let disk: Arc = Arc::new(RecordingDiskWriter::default()); - let stream = crowdb_protocol::chunk_stream::StreamName { high: 1, low: 3 }; - let healthy = MirrorChunkWriter::allocate_with_copy_count( - Arc::clone(&allocator), - Arc::clone(&disk), - stream, - 44, - 30_000, - 3, - crowdb_protocol::chunk_stream::StreamPurpose::Stream, - ) - .await - .unwrap(); - let mut chunk = healthy.chunk().clone(); - let Some(Strip::MirrorStrip(mirror)) = &mut chunk.strips[0].strip else { - panic!("allocated stream chunk must use mirrors"); - }; - mirror.segments.pop(); - assert!(MirrorChunkWriter::open_with_copy_count( - Arc::clone(&allocator), - Arc::clone(&disk), - chunk.clone(), - stream, - 44, - 30_000, - 3, - crowdb_protocol::chunk_stream::StreamPurpose::Stream - ) - .is_ok()); - let Some(Strip::MirrorStrip(mirror)) = &mut chunk.strips[0].strip else { - panic!("allocated stream chunk must use mirrors"); - }; - mirror.segments.pop(); - assert!(MirrorChunkWriter::open_with_copy_count( - allocator, - disk, - chunk, - stream, - 44, - 30_000, - 3, - crowdb_protocol::chunk_stream::StreamPurpose::Stream - ) - .is_err()); -} +#[path = "small_object_test/prefix_recovery.rs"] +mod prefix_recovery; diff --git a/lib/crowdb-chunk-client/tests/small_object_test/prefix_recovery.rs b/lib/crowdb-chunk-client/tests/small_object_test/prefix_recovery.rs new file mode 100644 index 000000000..1a274590a --- /dev/null +++ b/lib/crowdb-chunk-client/tests/small_object_test/prefix_recovery.rs @@ -0,0 +1,38 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{client, policy}; +use bytes::Bytes; +use crowdb_chunk_client::{ChunkIoWriter, IoError}; +use std::sync::atomic::Ordering; + +#[tokio::test] +async fn transient_write_failure_does_not_mark_the_acknowledged_prefix_corrupt() { + let mut settings = policy(); + settings.mirror_copies = 2; + let (client, allocator, disk) = client(settings); + let mut first = client.prepare_small_write(4096).await.unwrap(); + first.on_data(Bytes::from(vec![0x5a; 4096])).await.unwrap(); + let locations = first.on_finish().await.unwrap(); + let id = locations[0].chunk_id.unwrap(); + let before = allocator.state.lock().unwrap().chunks[&(id.high, id.low)].clone(); + assert!(before.acknowledged_cursor > 0); + disk.fail.store(true, Ordering::Relaxed); + let mut failed = client.prepare_small_write(4096).await.unwrap(); + failed.on_data(Bytes::from(vec![0xa5; 4096])).await.unwrap(); + assert!(failed.on_finish().await.is_err()); + let after = allocator.state.lock().unwrap().chunks[&(id.high, id.low)].clone(); + assert_eq!(after.acknowledged_cursor, before.acknowledged_cursor); + assert!( + after + .strips + .iter() + .all(|strip| strip.unavailable_segments.is_empty()), + "unknown writes cannot label every retained replica as corrupt" + ); + disk.fail.store(false, Ordering::Relaxed); + assert!(matches!( + client.shutdown_small_writes().await, + Err(IoError::WriteFailed(_)) + )); +} diff --git a/lib/crowdb-chunk-client/tests/write_stream.rs b/lib/crowdb-chunk-client/tests/write_stream.rs index 33d9c8b4b..67038da82 100644 --- a/lib/crowdb-chunk-client/tests/write_stream.rs +++ b/lib/crowdb-chunk-client/tests/write_stream.rs @@ -36,14 +36,13 @@ use crowdb_protocol::chunkdb::rpc::Strip as StripOneof; use crowdb_protocol::chunkdb::rpc::{ AllocateChunkRequest, AllocateChunkResponse, AppendChunkRequest, AppendChunkResponse, Chunk, ChunkStrip, ChunkType, DeleteChunkRequest, DeleteChunkResponse, EcStrip, MirrorStrip, QueryChunkRequest, - QueryChunkResponse, SealChunkRequest, SealChunkResponse, StripType, UpdateChunkStripRequest, - UpdateChunkStripResponse, + QueryChunkResponse, SealChunkRequest, SealChunkResponse, StripType, }; use crowdb_protocol::common::{ChunkId, DiskId as ProtoDiskId}; use crowdb_protocol::diskdb::rpc::Segment; use crowdb_protocol::frame::{ - encode_frame_regions, parse_frame, FrameError, FrameMagic, FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, - MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES, + encode_frame_regions, parse_frame, FrameError, FrameMagic, FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, + MAX_FRAME_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES, }; use common::LocalFileDiskWriter; @@ -58,8 +57,8 @@ fn framed_bytes(logical_bytes: usize) -> u64 { if logical_bytes == 0 { return 0; } - let frames = logical_bytes.div_ceil(MAX_FRAME_PAYLOAD_BYTES); - u64::try_from(logical_bytes + frames * (FRAME_HEADER_PREFIX_BYTES + FRAME_FOOTER_BYTES)).unwrap() + let frames = logical_bytes.div_ceil(MAX_FRAME_V2_PAYLOAD_BYTES); + u64::try_from(logical_bytes + frames * (FRAME_V2_HEADER_BYTES + FRAME_FOOTER_BYTES)).unwrap() } fn framed_io_bytes(logical_bytes: usize) -> u64 { @@ -74,7 +73,7 @@ fn decode_repo_large_frames(bytes: &[u8], chunk_id: crowdb_protocol::common::Chu let mut offset = 0; while offset < bytes.len() { let frame = parse_frame(&bytes[offset..], chunk_id).expect("durable RepoLarge frame"); - assert_eq!(frame.header.magic, FrameMagic::RepoLargeV1); + assert_eq!(frame.header.magic, FrameMagic::RepoLargeV2); payload.extend_from_slice(frame.payload); offset += frame.physical_length; } @@ -96,12 +95,12 @@ impl TestFramedOwner { fn full_frames(count: usize) -> Self { let mut bytes = vec![0; count * MAX_FRAME_BYTES]; for index in 0..count { - let start = index * MAX_FRAME_BYTES + FRAME_HEADER_PREFIX_BYTES; - bytes[start..start + MAX_FRAME_PAYLOAD_BYTES].fill(index as u8 + 1); + let start = index * MAX_FRAME_BYTES + FRAME_V2_HEADER_BYTES; + bytes[start..start + MAX_FRAME_V2_PAYLOAD_BYTES].fill(index as u8 + 1); } Self { bytes, - payload_lengths: vec![MAX_FRAME_PAYLOAD_BYTES; count], + payload_lengths: vec![MAX_FRAME_V2_PAYLOAD_BYTES; count], finalized_chunks: Arc::new(Mutex::new(Vec::new())), views: Arc::new(Mutex::new(Vec::new())), } @@ -130,9 +129,9 @@ impl FramedWriteBuffer for TestFramedOwner { ) -> std::result::Result, FrameError> { let frame_start = index * MAX_FRAME_BYTES; let payload_len = self.payload_lengths[index]; - let frame_end = frame_start + FRAME_HEADER_PREFIX_BYTES + payload_len + FRAME_FOOTER_BYTES; + let frame_end = frame_start + FRAME_V2_HEADER_BYTES + payload_len + FRAME_FOOTER_BYTES; let frame = &mut self.bytes[frame_start..frame_end]; - let (header, remainder) = frame.split_at_mut(FRAME_HEADER_PREFIX_BYTES); + let (header, remainder) = frame.split_at_mut(FRAME_V2_HEADER_BYTES); let (payload, footer) = remainder.split_at_mut(payload_len); encode_frame_regions(magic, chunk_id, payload, write_time_ms, header, footer)?; self.finalized_chunks.lock().unwrap().push(chunk_id); @@ -170,7 +169,7 @@ struct MockChunkAllocator { #[derive(Debug, Default)] struct MockChunkState { - chunks: HashMap<(u64, u64), (Vec, u32, bool)>, + chunks: HashMap<(u64, u64), (Vec, u64, bool)>, next_segment_offset: u64, allocate_calls: usize, allocated_strip_counts: Vec, @@ -372,7 +371,7 @@ impl ChunkAllocator for MockChunkAllocator { .chunks .get_mut(&(chunk_id.high, chunk_id.low)) .expect("seal unknown chunk"); - entry.1 = req.seal_length; + entry.1 = req.seal_bytes; Ok(SealChunkResponse { chunk: None }) } @@ -388,10 +387,6 @@ impl ChunkAllocator for MockChunkAllocator { Ok(DeleteChunkResponse { chunk: None }) } - async fn update_chunk_strip(&self, _req: UpdateChunkStripRequest) -> Result { - Ok(UpdateChunkStripResponse { chunk: None }) - } - async fn query_chunk(&self, _req: QueryChunkRequest) -> Result { Ok(QueryChunkResponse { chunk: None, @@ -424,10 +419,6 @@ impl ChunkAllocator for FailingChunkAllocator { async fn delete_chunk(&self, _req: DeleteChunkRequest) -> Result { unreachable!() } - - async fn update_chunk_strip(&self, _req: UpdateChunkStripRequest) -> Result { - unreachable!() - } } // ── Helpers ────────────────────────────────────────────────────── @@ -505,7 +496,7 @@ fn reconstruct_sealed_chunk( let strips = state .chunks .values() - .find_map(|(strips, sealed_length, deleted)| (*sealed_length > 0 && !deleted).then_some(strips)) + .find_map(|(strips, sealed_bytes, deleted)| (*sealed_bytes > 0 && !deleted).then_some(strips)) .expect("sealed chunk"); let mut reconstructed = Vec::new(); let mut remaining = usize::try_from(physical_bytes).expect("physical chunk length"); @@ -628,7 +619,7 @@ async fn write_stream_pads_only_parity_for_unaligned_tail() { assert!(state .chunks .values() - .any(|(_, sealed_length, _)| *sealed_length == 5)); + .any(|(_, sealed_bytes, _)| *sealed_bytes == framed_bytes(data.len()))); } #[tokio::test] @@ -843,8 +834,8 @@ async fn framed_owner_splits_into_views_only_at_chunk_boundaries() { assert_eq!(locations.len(), 2); assert_eq!(locations[0].length, MAX_FRAME_BYTES as u64); assert_eq!(locations[1].length, MAX_FRAME_BYTES as u64); - assert_eq!(locations[0].logical_length, MAX_FRAME_PAYLOAD_BYTES as u64); - assert_eq!(locations[1].logical_length, MAX_FRAME_PAYLOAD_BYTES as u64); + assert_eq!(locations[0].logical_length, MAX_FRAME_V2_PAYLOAD_BYTES as u64); + assert_eq!(locations[1].logical_length, MAX_FRAME_V2_PAYLOAD_BYTES as u64); assert_ne!(locations[0].chunk_id, locations[1].chunk_id); assert_eq!( *views.lock().unwrap(), @@ -856,7 +847,10 @@ async fn framed_owner_splits_into_views_only_at_chunk_boundaries() { ); assert_eq!(metrics.framed_owners, 1); assert_eq!(metrics.framed_views, 2); - assert_eq!(metrics.framed_payload_bytes, (2 * MAX_FRAME_PAYLOAD_BYTES) as u64); + assert_eq!( + metrics.framed_payload_bytes, + (2 * MAX_FRAME_V2_PAYLOAD_BYTES) as u64 + ); assert_eq!(metrics.payload_copy_operations, 0); assert_eq!(metrics.payload_copy_bytes, 0); } diff --git a/lib/crowdb-chunk-kv/src/lib.rs b/lib/crowdb-chunk-kv/src/lib.rs index 1b5756776..bbeb5f52b 100644 --- a/lib/crowdb-chunk-kv/src/lib.rs +++ b/lib/crowdb-chunk-kv/src/lib.rs @@ -17,7 +17,7 @@ pub use manager::PartitionManager; pub use metrics::{PartitionMetrics, PartitionMetricsSnapshot}; pub use partition::{ decode_frame, encode_frame, CheckpointReclaim, CrowdbPartitionTree, DecodedFrame, FrameDecode, - MaterializationProgress, MutationResponse, Partition, PartitionConfig, PartitionJournal, + JournalFrontier, MaterializationProgress, MutationResponse, Partition, PartitionConfig, PartitionJournal, PartitionSnapshot, PartitionTree, PreparedSplit, PreparedSplitWriter, ScanEntry, ScanPage, SplitHandoffStore, SplitIngress, SplitWriterTarget, StreamPartitionJournal, TreeObservation, MAX_FRAME_BYTES, diff --git a/lib/crowdb-chunk-kv/src/metrics.rs b/lib/crowdb-chunk-kv/src/metrics.rs index ab5c11d32..1029f36a1 100644 --- a/lib/crowdb-chunk-kv/src/metrics.rs +++ b/lib/crowdb-chunk-kv/src/metrics.rs @@ -24,7 +24,7 @@ pub struct PartitionMetrics { reclaimed_journal_bytes: AtomicU64, reclaimed_tree_bytes: AtomicU64, reclaimed_orphan_bytes: AtomicU64, - reclaimed_stream_metadata_pages: AtomicU64, + reclaimed_stream_metadata_records: AtomicU64, split_begins: AtomicU64, split_entries_examined: AtomicU64, split_entries_emitted: AtomicU64, @@ -68,7 +68,7 @@ pub struct PartitionMetricsSnapshot { pub reclaimed_journal_bytes: u64, pub reclaimed_tree_bytes: u64, pub reclaimed_orphan_bytes: u64, - pub reclaimed_stream_metadata_pages: u64, + pub reclaimed_stream_metadata_records: u64, pub split_begins: u64, pub split_entries_examined: u64, pub split_entries_emitted: u64, @@ -114,7 +114,7 @@ impl PartitionMetrics { reclaimed_journal_bytes: self.reclaimed_journal_bytes.load(Ordering::Relaxed), reclaimed_tree_bytes: self.reclaimed_tree_bytes.load(Ordering::Relaxed), reclaimed_orphan_bytes: self.reclaimed_orphan_bytes.load(Ordering::Relaxed), - reclaimed_stream_metadata_pages: self.reclaimed_stream_metadata_pages.load(Ordering::Relaxed), + reclaimed_stream_metadata_records: self.reclaimed_stream_metadata_records.load(Ordering::Relaxed), split_begins: self.split_begins.load(Ordering::Relaxed), split_entries_examined: self.split_entries_examined.load(Ordering::Relaxed), split_entries_emitted: self.split_entries_emitted.load(Ordering::Relaxed), @@ -203,10 +203,10 @@ impl PartitionMetrics { self.checkpoints.fetch_add(1, Ordering::Relaxed); } - pub(crate) fn reclaim(&self, journal: u64, metadata_pages: u64, tree: u64, orphans: u64) { + pub(crate) fn reclaim(&self, journal: u64, metadata_records: u64, tree: u64, orphans: u64) { self.reclaimed_journal_bytes.fetch_add(journal, Ordering::Relaxed); - self.reclaimed_stream_metadata_pages - .fetch_add(metadata_pages, Ordering::Relaxed); + self.reclaimed_stream_metadata_records + .fetch_add(metadata_records, Ordering::Relaxed); self.reclaimed_tree_bytes.fetch_add(tree, Ordering::Relaxed); self.reclaimed_orphan_bytes.fetch_add(orphans, Ordering::Relaxed); } diff --git a/lib/crowdb-chunk-kv/src/partition.rs b/lib/crowdb-chunk-kv/src/partition.rs index 3e603a6e9..94b27c4e6 100644 --- a/lib/crowdb-chunk-kv/src/partition.rs +++ b/lib/crowdb-chunk-kv/src/partition.rs @@ -5,7 +5,9 @@ //! Sub-modules: [`frame`] (wire codec), [`journal`] (durability), //! [`tree`] (ordered storage). +mod drain; mod frame; +mod frontier; mod ingress; mod journal; mod observation; @@ -16,6 +18,7 @@ mod tree; mod worker_sender; pub use frame::{decode_frame, encode_frame, DecodedFrame, FrameDecode, MAX_FRAME_BYTES}; +pub use frontier::JournalFrontier; pub use ingress::SplitIngress; pub use journal::{PartitionJournal, StreamPartitionJournal}; pub use observation::TreeObservation; @@ -51,7 +54,7 @@ pub struct PartitionConfig { pub max_key_bytes: usize, pub max_value_bytes: usize, pub retained_results: usize, - pub metadata_reclaim_pages_per_pass: usize, + pub metadata_reclaim_records_per_pass: usize, } impl Default for PartitionConfig { @@ -64,7 +67,7 @@ impl Default for PartitionConfig { max_key_bytes: 64 * 1024, max_value_bytes: 16 * 1024 * 1024, retained_results: 65_536, - metadata_reclaim_pages_per_pass: 128, + metadata_reclaim_records_per_pass: 128, } } } @@ -78,7 +81,7 @@ impl PartitionConfig { || self.max_key_bytes == 0 || self.max_value_bytes == 0 || self.retained_results == 0 - || self.metadata_reclaim_pages_per_pass == 0 + || self.metadata_reclaim_records_per_pass == 0 { return Err(ChunkKvError::InvalidRequest( "partition bounds must be nonzero".into(), @@ -122,7 +125,7 @@ pub struct PartitionSnapshot { #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct CheckpointReclaim { pub journal_bytes: u64, - pub metadata_pages: u64, + pub metadata_records: u64, pub tree_bytes: u64, pub orphan_bytes: u64, } @@ -143,6 +146,7 @@ struct MutationRequest { enum WorkerRequest { Mutation(MutationRequest), + CaptureJournalFrontier(frontier::CaptureRequest), ChildSplitCutover(Box), TransferCatchUp(Box), TransferAppend(Box), @@ -384,8 +388,8 @@ impl Partition { config.validate()?; if ownership_epoch == 0 || checkpoint.tree_id == 0 - || checkpoint.stream_manifest_generation == 0 - || checkpoint.stream_manifest_generation > journal.manifest_generation() + || checkpoint.stream_generation == 0 + || checkpoint.stream_generation > journal.record_generation() || checkpoint.stream_name != journal.stream_name() || checkpoint.tree_id != tree.tree_id() { @@ -483,7 +487,7 @@ impl Partition { .await } - /// Reopens the latest authoritative native tree root and stream manifest, + /// Reopens the latest authoritative native tree root and Stream chunk record, /// then replays WAL while remaining `Prepared`. /// /// The tree root supplies its own applied frontier and WAL replay offset; @@ -520,7 +524,7 @@ impl Partition { })? }; let stream_name = stream.stream_name(); - let stream_manifest_generation = stream.manifest_generation(); + let stream_generation = stream.record_generation(); let (tree, journal) = native_storage_parts(tree_id, &range, tree_config, page_store, stream)?; let (tree_manifest, applied_seq) = tree.checkpoint_state()?; let root_manifest_generation = tree.root_manifest_generation()?; @@ -534,7 +538,7 @@ impl Partition { root_manifest_generation, applied_seq, stream_name, - stream_manifest_generation, + stream_generation, replay_offset, }, config, @@ -595,8 +599,8 @@ impl Partition { if artifact.ownership_epoch == 0 || artifact.tree_id == 0 || artifact.root_manifest_generation == 0 - || checkpoint.stream_manifest_generation == 0 - || checkpoint.stream_manifest_generation > journal.manifest_generation() + || checkpoint.stream_generation == 0 + || checkpoint.stream_generation > journal.record_generation() || artifact.stream_name != checkpoint.stream_name || artifact.stream_name != journal.stream_name() || artifact.tree_id != checkpoint.tree_id @@ -739,7 +743,7 @@ impl Partition { root_manifest_generation: artifact.root_manifest_generation, applied_seq: artifact.base_applied_seq, stream_name: artifact.stream_name, - stream_manifest_generation: stream.manifest_generation(), + stream_generation: stream.record_generation(), replay_offset: 0, }; let range = artifact.range.clone(); @@ -1211,38 +1215,6 @@ impl Partition { &self.metrics } - /// Stops a transfer source after all previously admitted work completes. - /// - /// # Errors - /// - /// Returns a stale-epoch or incompatible-lifecycle error. - pub async fn suspend_for_transfer(&self, ownership_epoch: u64) -> Result<()> { - self.validate_epoch(ownership_epoch)?; - match self.lifecycle.compare_exchange( - lifecycle_code(PartitionLifecycle::Serving), - lifecycle_code(PartitionLifecycle::TransferFencing), - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => {} - Err(observed) if lifecycle_from_code(observed) == PartitionLifecycle::TransferFencing => {} - Err(observed) if lifecycle_from_code(observed) == PartitionLifecycle::TransferQuiesced => {} - Err(observed) => return Err(write_state_error(lifecycle_from_code(observed))), - } - self.wait_for_admitted_mutations().await; - match self.lifecycle.compare_exchange( - lifecycle_code(PartitionLifecycle::TransferFencing), - lifecycle_code(PartitionLifecycle::TransferQuiesced), - Ordering::AcqRel, - Ordering::Acquire, - ) { - Ok(_) => {} - Err(observed) if lifecycle_from_code(observed) == PartitionLifecycle::TransferQuiesced => {} - Err(observed) => return Err(write_state_error(lifecycle_from_code(observed))), - } - Ok(()) - } - /// Starts or idempotently resumes one exact split plan while writes continue. /// /// # Errors @@ -1713,7 +1685,8 @@ impl Partition { async fn create_checkpoint(&self) -> Result { let replay_offset = self.retry_replay_offset.load(Ordering::Acquire); - let stream_manifest_generation = self.journal.manifest_generation(); + self.journal.checkpoint().await?; + let stream_generation = self.journal.record_generation(); let (tree_manifest, applied_seq) = self.tree.checkpoint(replay_offset).await?; let root_manifest_generation = self.tree.root_manifest_generation()?; if applied_seq > self.journal_durable_seq.load(Ordering::Acquire) @@ -1728,7 +1701,7 @@ impl Partition { root_manifest_generation, applied_seq, stream_name: self.journal.stream_name(), - stream_manifest_generation, + stream_generation, replay_offset, }) } @@ -1745,8 +1718,8 @@ impl Partition { ) -> Result { self.validate_epoch(ownership_epoch)?; if checkpoint.stream_name != self.journal.stream_name() - || checkpoint.stream_manifest_generation == 0 - || checkpoint.stream_manifest_generation > self.journal.manifest_generation() + || checkpoint.stream_generation == 0 + || checkpoint.stream_generation > self.journal.record_generation() || checkpoint.applied_seq > self.applied_seq.load(Ordering::Acquire) || checkpoint.replay_offset > self.journal.tail() || checkpoint.replay_offset > self.retry_replay_offset.load(Ordering::Acquire) @@ -1774,20 +1747,20 @@ impl Partition { let journal_bytes = self .trim_published_checkpoint(ownership_epoch, checkpoint) .await?; - let metadata_pages = self + let metadata_records = self .journal .reclaim_metadata_before( - checkpoint.stream_manifest_generation, - self.config.metadata_reclaim_pages_per_pass, + checkpoint.stream_generation, + self.config.metadata_reclaim_records_per_pass, ) .await?; let tree_bytes = self.tree.reclaim_before(checkpoint.root_manifest_generation)?; let orphan_bytes = self.tree.reclaim_orphans()?; self.metrics - .reclaim(journal_bytes, metadata_pages, tree_bytes, orphan_bytes); + .reclaim(journal_bytes, metadata_records, tree_bytes, orphan_bytes); Ok(CheckpointReclaim { journal_bytes, - metadata_pages, + metadata_records, tree_bytes, orphan_bytes, }) @@ -1994,12 +1967,12 @@ fn validate_prepared_overlay( && artifact.parent_epoch != 0 && artifact.stream_name == checkpoint.stream_name && artifact.stream_name == journal.stream_name() - && checkpoint.stream_manifest_generation != 0 - && checkpoint.stream_manifest_generation <= journal.manifest_generation() + && checkpoint.stream_generation != 0 + && checkpoint.stream_generation <= journal.record_generation() && artifact.parent_stream_name == parent_journal.stream_name() && artifact.parent_stream_name != artifact.stream_name - && artifact.parent_stream_manifest_generation != 0 - && artifact.parent_stream_manifest_generation <= parent_journal.manifest_generation() + && artifact.parent_stream_generation != 0 + && artifact.parent_stream_generation <= parent_journal.record_generation() && artifact.parent_replay_offset <= artifact.parent_cutover_offset && artifact.parent_cutover_offset <= parent_journal.tail(); if !valid { @@ -2034,7 +2007,7 @@ async fn replay_parent_overlay( artifact.base_applied_seq }, stream_name: artifact.parent_stream_name, - stream_manifest_generation: artifact.parent_stream_manifest_generation, + stream_generation: artifact.parent_stream_generation, replay_offset: artifact.parent_replay_offset, }; let mut replay = ReplayState::new(&replay_checkpoint, retained_results); @@ -2363,6 +2336,10 @@ async fn run_worker(mut state: WorkerState, mut receiver: mpsc::Receiver { + frontier::capture(&state, request).await; + continue; + } WorkerRequest::ChildSplitCutover(request) => { split::stable_session::install_child_cutover(&mut state, *request).await; continue; @@ -2397,7 +2374,8 @@ async fn run_worker(mut state: WorkerState, mut receiver: mpsc::Receiver request, - control @ (WorkerRequest::ChildSplitCutover(_) + control @ (WorkerRequest::CaptureJournalFrontier(_) + | WorkerRequest::ChildSplitCutover(_) | WorkerRequest::TransferCatchUp(_) | WorkerRequest::TransferAppend(_)) => { pending = Some(control); diff --git a/lib/crowdb-chunk-kv/src/partition/drain.rs b/lib/crowdb-chunk-kv/src/partition/drain.rs new file mode 100644 index 000000000..08cb6829a --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/drain.rs @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::atomic::Ordering; + +use super::{lifecycle_code, lifecycle_from_code, write_state_error, Partition}; +use crate::{PartitionLifecycle, Result}; + +impl Partition { + /// Releases a healthy WAL writer after assignment and split tasks have stopped. + /// This does not authorize a transfer or commit an unfinished split. + /// + /// # Errors + /// + /// Preserves recovery and fault fences and propagates WAL release failures. + pub async fn suspend_for_shutdown(&self, ownership_epoch: u64) -> Result<()> { + self.validate_epoch(ownership_epoch)?; + loop { + let observed = self.lifecycle.load(Ordering::Acquire); + match lifecycle_from_code(observed) { + PartitionLifecycle::Serving + | PartitionLifecycle::Prepared + | PartitionLifecycle::SplitPreparing + | PartitionLifecycle::SplitFinalizing => { + if self + .lifecycle + .compare_exchange( + observed, + lifecycle_code(PartitionLifecycle::TransferFencing), + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + break; + } + } + PartitionLifecycle::TransferFencing => break, + PartitionLifecycle::TransferQuiesced => return Ok(()), + other => return Err(write_state_error(other)), + } + } + self.release_drained_writer().await + } + + pub(super) async fn release_drained_writer(&self) -> Result<()> { + self.wait_for_admitted_mutations().await; + if self.lifecycle() != PartitionLifecycle::TransferFencing { + return Err(write_state_error(self.lifecycle())); + } + self.journal.release_writer().await?; + match self.lifecycle.compare_exchange( + lifecycle_code(PartitionLifecycle::TransferFencing), + lifecycle_code(PartitionLifecycle::TransferQuiesced), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => Ok(()), + Err(observed) if lifecycle_from_code(observed) == PartitionLifecycle::TransferQuiesced => Ok(()), + Err(observed) => Err(write_state_error(lifecycle_from_code(observed))), + } + } +} diff --git a/lib/crowdb-chunk-kv/src/partition/frontier.rs b/lib/crowdb-chunk-kv/src/partition/frontier.rs new file mode 100644 index 000000000..921df5e4f --- /dev/null +++ b/lib/crowdb-chunk-kv/src/partition/frontier.rs @@ -0,0 +1,63 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Capture a published WAL prefix between complete mutation batches. + +use super::{lifecycle_from_code, write_state_error, Partition, WorkerRequest, WorkerState}; +use crate::{ChunkKvError, PartitionLifecycle, Result}; +use std::sync::atomic::Ordering; +use tokio::sync::oneshot; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct JournalFrontier { + pub mutation_seq: u64, + pub offset: u64, + pub stream_generation: u64, +} + +pub(super) struct CaptureRequest { + completion: oneshot::Sender>, +} + +impl Partition { + /// Publish and capture one exact WAL sequence/offset pair while continuing service. + /// # Errors + /// Rejects stale ownership, an unavailable worker or checkpoint failure. + pub async fn capture_journal_frontier(&self, epoch: u64) -> Result { + self.validate_epoch(epoch)?; + let (completion, receiver) = oneshot::channel(); + self.sender + .send(WorkerRequest::CaptureJournalFrontier(CaptureRequest { + completion, + })) + .await?; + let result = receiver.await.map_err(|_| ChunkKvError::WriteStalled)?; + // Retained parents reuse this sequencer under a newer logical epoch. + // Validate the calling partition, rather than the original worker epoch. + self.validate_epoch(epoch)?; + result + } +} + +pub(super) async fn capture(state: &WorkerState, request: CaptureRequest) { + let result = capture_inner(state).await; + let _ = request.completion.send(result); +} + +async fn capture_inner(state: &WorkerState) -> Result { + let lifecycle = lifecycle_from_code(state.lifecycle.load(Ordering::Acquire)); + if !matches!( + lifecycle, + PartitionLifecycle::Serving + | PartitionLifecycle::TransferQuiesced + | PartitionLifecycle::SplitFinalizing + ) { + return Err(write_state_error(lifecycle)); + } + state.journal.checkpoint().await?; + Ok(JournalFrontier { + mutation_seq: state.journal_durable_seq.load(Ordering::Acquire), + offset: state.journal.tail(), + stream_generation: state.journal.record_generation(), + }) +} diff --git a/lib/crowdb-chunk-kv/src/partition/journal.rs b/lib/crowdb-chunk-kv/src/partition/journal.rs index dadc0ac78..f5cb7cdb3 100644 --- a/lib/crowdb-chunk-kv/src/partition/journal.rs +++ b/lib/crowdb-chunk-kv/src/partition/journal.rs @@ -12,15 +12,21 @@ pub trait PartitionJournal: Send + Sync { async fn append_frames(&self, frames: &[Bytes]) -> Result>; async fn read_window(&self, offset: u64, max_bytes: usize) -> Result; async fn trim_prefix(&self, offset: u64) -> Result; - async fn reclaim_metadata_before(&self, generation: u64, max_pages: usize) -> Result; + async fn reclaim_metadata_before(&self, generation: u64, max_records: usize) -> Result; + /// Publishes the ordered WAL prefix before a durable checkpoint or cutover proof. + async fn checkpoint(&self) -> Result; + /// Drain and release writer authority before publishing an explicit transfer proof. + async fn release_writer(&self) -> Result<()> { + self.checkpoint().await.map(|_| ()) + } async fn close(&self) -> Result<()>; fn stream_name(&self) -> StreamName; - fn manifest_generation(&self) -> u64; + fn record_generation(&self) -> u64; fn tail(&self) -> u64; /// Returns published metadata only; implementations without inspection return `None`. /// /// # Errors - /// Rejects stale generations and invalid extent-index continuations. + /// Rejects stale generations and invalid chunk-record observation offsets. fn observe_metadata( &self, _generation: Option, @@ -112,9 +118,9 @@ impl PartitionJournal for StreamPartitionJournal { self.stream.trim_prefix(offset).await.map_err(map_stream_error) } - async fn reclaim_metadata_before(&self, generation: u64, max_pages: usize) -> Result { + async fn reclaim_metadata_before(&self, generation: u64, max_records: usize) -> Result { self.stream - .reclaim_metadata_before(generation, max_pages) + .reclaim_metadata_before(generation, max_records) .await .map_err(map_stream_error) } @@ -123,12 +129,20 @@ impl PartitionJournal for StreamPartitionJournal { self.stream.close().await.map_err(map_stream_error) } + async fn checkpoint(&self) -> Result { + self.stream.checkpoint().await.map_err(map_stream_error) + } + + async fn release_writer(&self) -> Result<()> { + self.stream.release_writer().await.map_err(map_stream_error) + } + fn stream_name(&self) -> StreamName { self.stream_name } - fn manifest_generation(&self) -> u64 { - self.stream.manifest_generation() + fn record_generation(&self) -> u64 { + self.stream.record_generation() } fn tail(&self) -> u64 { @@ -152,9 +166,7 @@ fn map_stream_error(error: StreamError) -> ChunkKvError { StreamError::InvalidRequest(message) => ChunkKvError::InvalidRequest(message), StreamError::Backpressure => ChunkKvError::Overloaded, StreamError::StaleWriter => ChunkKvError::StaleEpoch, - StreamError::DefinitelyNotCommitted(_) - | StreamError::ResolvingAmbiguousAppend - | StreamError::WriteStalled => ChunkKvError::WriteStalled, + StreamError::ResolvingAmbiguousAppend | StreamError::WriteStalled => ChunkKvError::WriteStalled, StreamError::ReadUnavailable(message) => ChunkKvError::TreeUnavailable(message), StreamError::Corruption(message) => ChunkKvError::JournalCorruption(message), StreamError::Internal(message) => ChunkKvError::Internal(message), diff --git a/lib/crowdb-chunk-kv/src/partition/observation.rs b/lib/crowdb-chunk-kv/src/partition/observation.rs index 0a12c2413..d92bb0c5b 100644 --- a/lib/crowdb-chunk-kv/src/partition/observation.rs +++ b/lib/crowdb-chunk-kv/src/partition/observation.rs @@ -43,7 +43,7 @@ impl Partition { }) } - /// Copies a bounded journal extent index from its published manifest. + /// Copies the journal's published tail Stream chunk record. /// /// # Errors /// Returns a stale-generation or invalid-continuation error. diff --git a/lib/crowdb-chunk-kv/src/partition/split.rs b/lib/crowdb-chunk-kv/src/partition/split.rs index e41842a45..2093650e1 100644 --- a/lib/crowdb-chunk-kv/src/partition/split.rs +++ b/lib/crowdb-chunk-kv/src/partition/split.rs @@ -314,6 +314,7 @@ impl Partition { child_state .child_tree .pin_generation(plan.transition_id, child_state.child_base.1)?; + self.journal.checkpoint().await?; let source = SplitSourceFrontier { parent_id: plan.parent_id, parent_epoch: plan.parent_epoch, @@ -371,7 +372,8 @@ impl Partition { )); } let replay_offset = self.retry_replay_offset.load(Ordering::Acquire); - let stream_manifest_generation = self.journal.manifest_generation(); + self.journal.checkpoint().await?; + let stream_generation = self.journal.record_generation(); let (tree_manifest, applied_seq, source) = self.tree.checkpoint_snapshot(replay_offset).await?; let root_manifest_generation = self.tree.root_manifest_generation()?; if applied_seq > self.journal_durable_seq.load(Ordering::Acquire) { @@ -384,7 +386,7 @@ impl Partition { root_manifest_generation, applied_seq, stream_name: self.journal.stream_name(), - stream_manifest_generation, + stream_generation, replay_offset, }, source, @@ -427,7 +429,7 @@ fn prepare_retained_parent( parent_id: source.parent_id, parent_epoch: source.parent_epoch, parent_stream_name: source.checkpoint.stream_name, - parent_stream_manifest_generation: source.checkpoint.stream_manifest_generation, + parent_stream_generation: source.checkpoint.stream_generation, parent_replay_offset: source.checkpoint.replay_offset, parent_cutover_offset: source.cutover_offset, applied_seq, @@ -441,7 +443,7 @@ fn validate_target(parent: &Partition, plan: &SplitPlan, target: &SplitWriterTar if target.tree_id == 0 || target.tree_id == parent.tree.tree_id() || target.journal.stream_name() == parent.journal.stream_name() - || target.journal.manifest_generation() == 0 + || target.journal.record_generation() == 0 { return Err(ChunkKvError::InvalidRequest( "split writer storage identities must be distinct".into(), @@ -607,7 +609,7 @@ fn prepare_writer( parent_id: source.parent_id, parent_epoch: source.parent_epoch, parent_stream_name: source.checkpoint.stream_name, - parent_stream_manifest_generation: source.checkpoint.stream_manifest_generation, + parent_stream_generation: source.checkpoint.stream_generation, parent_replay_offset: source.checkpoint.replay_offset, parent_cutover_offset: source.cutover_offset, applied_seq: source.cutover_seq, @@ -622,7 +624,7 @@ fn prepare_writer( root_manifest_generation, applied_seq: base_applied_seq, stream_name: target.journal.stream_name(), - stream_manifest_generation: target.journal.manifest_generation(), + stream_generation: target.journal.record_generation(), replay_offset: 0, }; Ok(PreparedSplitWriter { diff --git a/lib/crowdb-chunk-kv/src/partition/split/recovery.rs b/lib/crowdb-chunk-kv/src/partition/split/recovery.rs index b6309bc96..28d21e605 100644 --- a/lib/crowdb-chunk-kv/src/partition/split/recovery.rs +++ b/lib/crowdb-chunk-kv/src/partition/split/recovery.rs @@ -46,7 +46,7 @@ impl Partition { root_manifest_generation: self.tree.root_manifest_generation()?, applied_seq: applied, stream_name: self.journal.stream_name(), - stream_manifest_generation: self.journal.manifest_generation(), + stream_generation: self.journal.record_generation(), replay_offset: child_artifact.parent_replay_offset, }, cutover_offset: child_artifact.parent_cutover_offset, @@ -107,7 +107,7 @@ impl Partition { root_manifest_generation: child_artifact.root_manifest_generation, applied_seq: child_artifact.base_applied_seq, stream_name: child_artifact.stream_name, - stream_manifest_generation: child.journal.manifest_generation(), + stream_generation: child.journal.record_generation(), replay_offset: 0, }; Ok(super::PreparedSplit { @@ -144,7 +144,7 @@ impl Partition { root_manifest_generation: self.tree.root_manifest_generation()?, applied_seq, stream_name: self.journal.stream_name(), - stream_manifest_generation: self.journal.manifest_generation(), + stream_generation: self.journal.record_generation(), replay_offset: proof.parent_replay_offset, }; let target = super::SplitWriterTarget { @@ -236,7 +236,7 @@ impl Partition { root_manifest_generation: artifact.root_manifest_generation, applied_seq: artifact.base_applied_seq, stream_name: artifact.stream_name, - stream_manifest_generation: journal.manifest_generation(), + stream_generation: journal.record_generation(), replay_offset: 0, }; let partition = Self::recover_prepared_overlay( diff --git a/lib/crowdb-chunk-kv/src/partition/split/stable_session.rs b/lib/crowdb-chunk-kv/src/partition/split/stable_session.rs index cfb9f7ddf..083537066 100644 --- a/lib/crowdb-chunk-kv/src/partition/split/stable_session.rs +++ b/lib/crowdb-chunk-kv/src/partition/split/stable_session.rs @@ -48,7 +48,7 @@ impl Partition { root_manifest_generation: self.tree.root_manifest_generation()?, applied_seq: base_seq, stream_name: self.journal.stream_name(), - stream_manifest_generation: self.journal.manifest_generation(), + stream_generation: self.journal.record_generation(), replay_offset: proof.parent_replay_offset, }; let frontier = SplitSourceFrontier { @@ -87,7 +87,7 @@ impl Partition { root_manifest_generation: proof.root_manifest_generation, applied_seq: proof.base_applied_seq, stream_name: proof.stream_name, - stream_manifest_generation: child.journal.manifest_generation(), + stream_generation: child.journal.record_generation(), replay_offset: 0, }, tree: child.tree.clone(), @@ -170,11 +170,12 @@ async fn install_child_cutover_inner( request: &ChildSplitCutoverRequest, ) -> Result<(SplitArtifact, PreparedSplitWriter)> { let cutover_seq = state.applied_seq.load(Ordering::Acquire); + let cutover_offset = state.journal.checkpoint().await?; let source = SplitSourceFrontier { parent_id: request.plan.parent_id, parent_epoch: request.plan.parent_epoch, checkpoint: request.base_checkpoint.clone(), - cutover_offset: state.journal.tail(), + cutover_offset, cutover_seq, journal: Arc::clone(&request.parent.journal), }; diff --git a/lib/crowdb-chunk-kv/src/partition/transfer.rs b/lib/crowdb-chunk-kv/src/partition/transfer.rs index ac70247c1..24a695951 100644 --- a/lib/crowdb-chunk-kv/src/partition/transfer.rs +++ b/lib/crowdb-chunk-kv/src/partition/transfer.rs @@ -10,7 +10,10 @@ use std::sync::Arc; use bytes::{Buf, BytesMut}; use tokio::sync::oneshot; -use super::{MutationRequest, PartitionJournal, PreparedSplitWriterArtifact}; +use super::{ + lifecycle_code, lifecycle_from_code, write_state_error, MutationRequest, PartitionJournal, + PreparedSplitWriterArtifact, +}; use crate::{ canonical_operation_digest, ChunkKvError, JournalPosition, MutationOperation, MutationResponse, MutationResult, PartitionLifecycle, RequestId, Result, WalRecord, @@ -330,7 +333,7 @@ fn validate_extension( ) -> Result<()> { let same_base = validate_artifact_extension(current, final_artifact).is_ok(); let valid_journal = parent_journal.stream_name() == final_artifact.parent_stream_name - && parent_journal.manifest_generation() >= final_artifact.parent_stream_manifest_generation + && parent_journal.record_generation() >= final_artifact.parent_stream_generation && parent_journal.tail() >= final_artifact.parent_cutover_offset; if !same_base || !valid_journal { return Err(ChunkKvError::InvalidRequest( @@ -355,7 +358,7 @@ fn validate_artifact_extension( && current.parent_id == final_artifact.parent_id && current.parent_epoch == final_artifact.parent_epoch && current.parent_stream_name == final_artifact.parent_stream_name - && current.parent_stream_manifest_generation == final_artifact.parent_stream_manifest_generation + && current.parent_stream_generation == final_artifact.parent_stream_generation && current.parent_replay_offset == final_artifact.parent_replay_offset; let valid_frontier = current.applied_seq <= final_artifact.applied_seq && current.parent_cutover_offset <= final_artifact.parent_cutover_offset @@ -458,3 +461,28 @@ impl super::Partition { response.await.map_err(|_| ChunkKvError::WriteStalled)? } } + +impl super::Partition { + /// Stops a transfer source after all previously admitted work completes. + /// + /// # Errors + /// + /// Returns a stale-epoch or incompatible-lifecycle error. + pub async fn suspend_for_transfer(&self, ownership_epoch: u64) -> Result<()> { + self.validate_epoch(ownership_epoch)?; + match self.lifecycle.compare_exchange( + lifecycle_code(PartitionLifecycle::Serving), + lifecycle_code(PartitionLifecycle::TransferFencing), + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => {} + Err(observed) if lifecycle_from_code(observed) == PartitionLifecycle::TransferFencing => {} + Err(observed) if lifecycle_from_code(observed) == PartitionLifecycle::TransferQuiesced => { + return Ok(()) + } + Err(observed) => return Err(write_state_error(lifecycle_from_code(observed))), + } + self.release_drained_writer().await + } +} diff --git a/lib/crowdb-chunk-kv/src/types.rs b/lib/crowdb-chunk-kv/src/types.rs index 80cbfb04d..fda85975a 100644 --- a/lib/crowdb-chunk-kv/src/types.rs +++ b/lib/crowdb-chunk-kv/src/types.rs @@ -202,7 +202,7 @@ pub struct Checkpoint { pub root_manifest_generation: u64, pub applied_seq: u64, pub stream_name: StreamName, - pub stream_manifest_generation: u64, + pub stream_generation: u64, pub replay_offset: u64, } @@ -281,7 +281,7 @@ pub struct PreparedSplitWriterArtifact { pub parent_id: PartitionId, pub parent_epoch: u64, pub parent_stream_name: StreamName, - pub parent_stream_manifest_generation: u64, + pub parent_stream_generation: u64, pub parent_replay_offset: u64, pub parent_cutover_offset: u64, pub applied_seq: u64, diff --git a/lib/crowdb-chunk-kv/tests/bootstrap_recovery_test.rs b/lib/crowdb-chunk-kv/tests/bootstrap_recovery_test.rs index 65b106bb4..ac01d4e76 100644 --- a/lib/crowdb-chunk-kv/tests/bootstrap_recovery_test.rs +++ b/lib/crowdb-chunk-kv/tests/bootstrap_recovery_test.rs @@ -8,8 +8,8 @@ use crowdb_chunk_kv::{ }; use crowdb_chunk_stream::memory::MemoryStreamStore; use crowdb_chunk_stream::{ - ChunkStream, StreamBinding, StreamBindingState, StreamChunkStore, StreamConfig, StreamMetadataStore, - StreamName, StreamPurpose, StreamRegistry, + ChunkStream, StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, StreamPurpose, + StreamRecord, StreamRecordState, StreamRegistry, }; use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, PageStore}; @@ -124,13 +124,15 @@ fn operation(sequence: u64) -> MutationOperation { async fn create_stream(name: StreamName, store: &Arc) -> ChunkStream { ChunkStream::create( - StreamBinding { + StreamRecord { purpose: StreamPurpose::Wal, stream_name: name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 4, StreamConfig::default(), diff --git a/lib/crowdb-chunk-kv/tests/common/split_storage.rs b/lib/crowdb-chunk-kv/tests/common/split_storage.rs index 35125a9e1..063ae1000 100644 --- a/lib/crowdb-chunk-kv/tests/common/split_storage.rs +++ b/lib/crowdb-chunk-kv/tests/common/split_storage.rs @@ -6,7 +6,7 @@ use crowdb_chunk_kv::{ StreamPartitionJournal, }; use crowdb_chunk_stream::{ - memory::MemoryStreamStore, ChunkStream, StreamBinding, StreamBindingState, StreamConfig, StreamName, + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamRecord, StreamRecordState, }; use std::sync::Arc; @@ -23,13 +23,15 @@ impl TestSplitStorage { pub async fn journal(&self, stream_name: StreamName, epoch: u64) -> Arc { let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, epoch, StreamConfig::default(), diff --git a/lib/crowdb-chunk-kv/tests/journal_failure_test.rs b/lib/crowdb-chunk-kv/tests/journal_failure_test.rs index b89d674b2..6ac0d60e7 100644 --- a/lib/crowdb-chunk-kv/tests/journal_failure_test.rs +++ b/lib/crowdb-chunk-kv/tests/journal_failure_test.rs @@ -42,11 +42,15 @@ impl PartitionJournal for FailedJournal { Ok(()) } + async fn checkpoint(&self) -> crowdb_chunk_kv::Result { + Err(ChunkKvError::WriteStalled) + } + fn stream_name(&self) -> StreamName { StreamName { high: 1, low: 1 } } - fn manifest_generation(&self) -> u64 { + fn record_generation(&self) -> u64 { 1 } @@ -122,6 +126,10 @@ async fn assert_failed_journal_recovers(wrong_position_count: bool) { partition.suspend_for_transfer(1).await, Err(ChunkKvError::Recovering) ); + assert_eq!( + partition.suspend_for_shutdown(1).await, + Err(ChunkKvError::Recovering) + ); assert!(matches!( partition.checkpoint_quiesced(1).await, Err(ChunkKvError::InvalidRequest(_)) diff --git a/lib/crowdb-chunk-kv/tests/journal_frontier_test.rs b/lib/crowdb-chunk-kv/tests/journal_frontier_test.rs new file mode 100644 index 000000000..7691b8b38 --- /dev/null +++ b/lib/crowdb-chunk-kv/tests/journal_frontier_test.rs @@ -0,0 +1,112 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_chunk_kv::{ + memory::MemoryPartitionTree, ChunkKvError, MutationOperation, Partition, PartitionConfig, PartitionId, + PartitionRange, RequestId, StreamPartitionJournal, +}; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, +}; +use std::{sync::Arc, time::Duration}; + +#[tokio::test] +async fn captured_frontier_publishes_one_exact_prefix_before_later_mutations() { + let store = Arc::new(MemoryStreamStore::new(1024 * 1024)); + let name = StreamName::generate(); + let stream = ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Wal, + stream_name: name, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("frontier-test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + let partition = Partition::open( + PartitionId { high: 1, low: 1 }, + PartitionRange::default(), + 1, + PartitionConfig::default(), + Arc::new(MemoryPartitionTree::default()), + Arc::new(StreamPartitionJournal::new(stream, name).unwrap()), + ) + .unwrap(); + mutate(&partition, 1).await; + let first_end = partition.snapshot().journal_durable_offset; + assert_eq!( + partition.capture_journal_frontier(2).await, + Err(ChunkKvError::StaleEpoch) + ); + store.pause_publication_for_tests(); + let capturing = partition.capture_journal_frontier(1); + tokio::pin!(capturing); + assert!( + tokio::time::timeout(Duration::from_millis(10), capturing.as_mut()) + .await + .is_err() + ); + store.wait_for_publication_for_tests().await; + let later = mutate(&partition, 2); + tokio::pin!(later); + assert!(tokio::time::timeout(Duration::from_millis(10), later.as_mut()) + .await + .is_err()); + store.resume_publication_for_tests(); + let frontier = capturing.await.unwrap(); + later.await; + assert_eq!(frontier.mutation_seq, 1); + assert_eq!(frontier.offset, first_end); + assert_eq!(partition.snapshot().journal_durable_seq, 2); + assert!(partition.snapshot().journal_durable_offset > frontier.offset); + let reader = ChunkStream::open_read_only( + name, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await + .unwrap(); + assert_eq!(reader.tail(), frontier.offset); + assert_eq!(reader.record_generation(), frontier.stream_generation); + let bytes = reader + .read_at(0, usize::try_from(frontier.offset).unwrap()) + .await + .unwrap(); + let crowdb_chunk_kv::FrameDecode::Complete(frame) = crowdb_chunk_kv::decode_frame(&bytes).unwrap() else { + panic!("captured frontier must retain a complete WAL record"); + }; + assert_eq!(frame.record.mutation_seq, frontier.mutation_seq); + assert_eq!(frame.bytes_consumed, bytes.len()); +} + +async fn mutate(partition: &Partition, sequence: u64) { + partition + .mutate( + 1, + RequestId { + client_high: 1, + client_low: 1, + client_sequence: sequence, + }, + MutationOperation::Put { + key: sequence.to_be_bytes().to_vec(), + value: vec![7; 16], + }, + ) + .await + .unwrap(); +} diff --git a/lib/crowdb-chunk-kv/tests/partition_test.rs b/lib/crowdb-chunk-kv/tests/partition_test.rs index 4dd97f338..e5ba5cff5 100644 --- a/lib/crowdb-chunk-kv/tests/partition_test.rs +++ b/lib/crowdb-chunk-kv/tests/partition_test.rs @@ -12,8 +12,8 @@ use crowdb_chunk_kv::{ }; use crowdb_chunk_stream::memory::MemoryStreamStore; use crowdb_chunk_stream::{ - ChunkId, ChunkStream, CursorAdvance, StreamBinding, StreamBindingState, StreamChunkStore, StreamConfig, - StreamMetadataStore, StreamName, StreamRegistry, + ChunkId, ChunkStream, CursorAdvance, StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, + StreamRecord, StreamRecordState, StreamRegistry, }; use crowdb_tree_ffi::{ChunkPageStoreOptions, ChunkRootCatalog, PageStore}; @@ -60,7 +60,7 @@ fn split_artifact(plan: &SplitPlan, cutover_seq: u64) -> SplitArtifact { parent_id: plan.parent_id, parent_epoch: plan.parent_epoch, parent_stream_name: StreamName { high: 89, low: 1 }, - parent_stream_manifest_generation: 1, + parent_stream_generation: 1, parent_replay_offset: 0, parent_cutover_offset: cutover_seq, applied_seq: cutover_seq, @@ -103,7 +103,7 @@ fn empty_prepared_child( parent_id: PartitionId { high: 10, low: 1 }, parent_epoch: 19, parent_stream_name: StreamName { high: 12, low: 11 }, - parent_stream_manifest_generation: 1, + parent_stream_generation: 1, parent_replay_offset: 0, parent_cutover_offset: 0, applied_seq: 0, @@ -118,13 +118,15 @@ async fn partition( epoch: u64, config: PartitionConfig, ) -> Partition { - let binding = StreamBinding { + let binding = StreamRecord { purpose: crowdb_chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }; let registry: Arc = store.clone(); let metadata: Arc = store.clone(); @@ -165,13 +167,15 @@ async fn empty_journal( epoch: u64, ) -> Arc { let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, epoch, StreamConfig::default(), @@ -335,7 +339,7 @@ async fn assert_prepared_tree_identity_mismatch( root_manifest_generation: artifact.root_manifest_generation, applied_seq: artifact.applied_seq, stream_name: artifact.stream_name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), @@ -407,13 +411,15 @@ async fn native_partition_constructor_owns_tree_and_stream_storage() { let stream_name = StreamName { high: 1, low: 9 }; let store = Arc::new(MemoryStreamStore::new(4_096)); let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 4, StreamConfig::default(), @@ -492,13 +498,15 @@ async fn native_partition_reopens_the_latest_tree_root() { let stream_name = StreamName { high: 1, low: 10 }; let store = Arc::new(MemoryStreamStore::new(4_096)); let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 4, StreamConfig::default(), @@ -535,13 +543,15 @@ async fn chunk_root_checkpoint_supplies_the_wal_replay_offset() { let stream_name = StreamName { high: 1, low: 11 }; let stream_store = Arc::new(MemoryStreamStore::new(4_096)); let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 4, StreamConfig::default(), @@ -1055,7 +1065,7 @@ async fn recovery_replays_recorded_results_and_restores_deduplication() { root_manifest_generation: 1, applied_seq: 0, stream_name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), @@ -1134,7 +1144,7 @@ async fn recovery_rejects_a_tree_root_other_than_the_checkpoint() { root_manifest_generation: 1, applied_seq: 0, stream_name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), @@ -1210,7 +1220,7 @@ async fn recovery_rejects_a_frame_bound_to_another_physical_chunk() { root_manifest_generation: 1, applied_seq: 0, stream_name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), @@ -1607,13 +1617,15 @@ async fn prepared_child_serves_only_after_exact_catalog_proof() { let store = Arc::new(MemoryStreamStore::new(4_096)); let stream_name = StreamName { high: 12, low: 12 }; let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_chunk_stream::StreamPurpose::Wal, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 20, StreamConfig::default(), @@ -1643,7 +1655,7 @@ async fn prepared_child_serves_only_after_exact_catalog_proof() { root_manifest_generation: 1, applied_seq: 0, stream_name, - stream_manifest_generation: 1, + stream_generation: 1, replay_offset: 0, }, PartitionConfig::default(), @@ -2149,13 +2161,15 @@ async fn native_online_split_rebuilds_one_child_from_the_exact_parent_manifest() let store = Arc::new(MemoryStreamStore::new(4_096)); let parent_stream_name = StreamName { high: 120, low: 120 }; let parent_stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_chunk_stream::StreamPurpose::Wal, stream_name: parent_stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("chunk-kv-partition".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("chunk-kv-partition".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 18, StreamConfig::default(), @@ -2323,7 +2337,7 @@ async fn overlay_fixture() -> OverlayFixture { parent_id: PartitionId { high: 130, low: 1 }, parent_epoch: 18, parent_stream_name: parent_name, - parent_stream_manifest_generation: parent_journal.manifest_generation(), + parent_stream_generation: parent_journal.record_generation(), parent_replay_offset: 0, parent_cutover_offset: parent_journal.tail(), applied_seq: 5, @@ -2335,7 +2349,7 @@ async fn overlay_fixture() -> OverlayFixture { root_manifest_generation: manifest, applied_seq: 2, stream_name: child_name, - stream_manifest_generation: child_journal.manifest_generation(), + stream_generation: child_journal.record_generation(), replay_offset: 0, }; let proof = SplitCommitProof { @@ -2400,7 +2414,7 @@ async fn both_split_halves_recover_pre_split_keys_from_parent_overlay() { root_manifest_generation: child_artifact.root_manifest_generation, applied_seq: child_artifact.base_applied_seq, stream_name: child_artifact.stream_name, - stream_manifest_generation: child_journal.manifest_generation(), + stream_generation: child_journal.record_generation(), replay_offset: 0, }; let proof = SplitCommitProof { diff --git a/lib/crowdb-chunk-kv/tests/split_handoff_recovery_test.rs b/lib/crowdb-chunk-kv/tests/split_handoff_recovery_test.rs index 742a7fbab..a449b3fcb 100644 --- a/lib/crowdb-chunk-kv/tests/split_handoff_recovery_test.rs +++ b/lib/crowdb-chunk-kv/tests/split_handoff_recovery_test.rs @@ -59,7 +59,7 @@ async fn recover_after_handoff(child_has_write: bool, checkpoint_after_dispatch: parent_id, parent_epoch: 1, parent_stream_name: parent_journal.stream_name(), - parent_stream_manifest_generation: parent_journal.manifest_generation(), + parent_stream_generation: parent_journal.record_generation(), parent_replay_offset: base_offset, parent_cutover_offset: base_offset, applied_seq: 1, @@ -254,7 +254,7 @@ fn source_checkpoint_at_base( root_manifest_generation: 1, applied_seq: 1, stream_name: journal.stream_name(), - stream_manifest_generation: journal.manifest_generation(), + stream_generation: journal.record_generation(), replay_offset: base_offset, } } diff --git a/lib/crowdb-chunk-kv/tests/split_ingress_test.rs b/lib/crowdb-chunk-kv/tests/split_ingress_test.rs index 0dc3eb3c2..d700586d8 100644 --- a/lib/crowdb-chunk-kv/tests/split_ingress_test.rs +++ b/lib/crowdb-chunk-kv/tests/split_ingress_test.rs @@ -216,6 +216,9 @@ async fn retained_parent_can_complete_a_second_split_on_the_original_worker() { parent.release_generation_pin(plan.transition_id).unwrap(); parent = parent.split_ingress().unwrap().retained_parent(); assert_eq!(parent.snapshot().stream_name, original_stream); + let frontier = parent.capture_journal_frontier(epoch + 1).await.unwrap(); + assert_eq!(frontier.mutation_seq, parent.snapshot().journal_durable_seq); + assert_eq!(frontier.offset, parent.snapshot().journal_durable_offset); assert_eq!( parent.get(epoch + 1, b"b", None).await.unwrap().unwrap().value, b"left" diff --git a/lib/crowdb-chunk-stream/Cargo.toml b/lib/crowdb-chunk-stream/Cargo.toml index d40a21bb0..45a1fa88a 100644 --- a/lib/crowdb-chunk-stream/Cargo.toml +++ b/lib/crowdb-chunk-stream/Cargo.toml @@ -22,12 +22,13 @@ test-util = [] arc-swap = "1.9" async-trait = "0.1" bytes = "1" -bincode = "1" crc32fast = "1" +crowdb-common = { path = "../crowdb-common/rust" } crowdb-kv-client = { path = "../crowdb-kv-client" } crowdb-chunk-client = { path = "../crowdb-chunk-client" } crowdb-protocol = { path = "../crowdb-protocol" } crossbeam-skiplist = "0.1" +futures = "0.3" serde = "1" serde_json = "1" thiserror = { workspace = true } @@ -37,4 +38,5 @@ tracing = { workspace = true } [dev-dependencies] crowdb-chunk-stream = { path = ".", features = ["test-util"] } crowdb-test-harness = { path = "../crowdb-test-harness" } +crowdb-kv = { path = "../crowdb-kv", features = ["test-util"] } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "test-util"] } diff --git a/lib/crowdb-chunk-stream/benches/production_bounds.rs b/lib/crowdb-chunk-stream/benches/production_bounds.rs index dcc72f074..841696ac6 100644 --- a/lib/crowdb-chunk-stream/benches/production_bounds.rs +++ b/lib/crowdb-chunk-stream/benches/production_bounds.rs @@ -13,8 +13,11 @@ mod workload; use std::process::ExitCode; use std::sync::Arc; -use crowdb_chunk_client::{ChunkIoClient, ChunkIoClientConfig, ChunkReadPolicy, SmallWritePolicy}; +use crowdb_chunk_client::{ + ChunkClientMetrics, ChunkIoClient, ChunkIoClientConfig, ChunkReadPolicy, SmallWritePolicy, +}; use crowdb_chunk_stream::{ProductionStreamRuntime, StreamConfig}; +use crowdb_common::metrics; use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; use config::BenchConfig; @@ -42,6 +45,9 @@ fn main() -> ExitCode { } async fn run(config: BenchConfig) -> ExitCode { + let client_metrics = Arc::new(ChunkClientMetrics::register( + &mut metrics::global().lock().unwrap(), + )); let chunk_io = match ChunkIoClient::connect(ChunkIoClientConfig { management_seeds: vec![config.management_seed.clone()], diskio_connections_per_endpoint: config.diskio_connections, @@ -50,7 +56,7 @@ async fn run(config: BenchConfig) -> ExitCode { }) .await { - Ok(client) => client, + Ok(client) => client.with_metrics(&client_metrics), Err(error) => { eprintln!("chunk-stream benchmark discovery failed: {error}"); return ExitCode::FAILURE; @@ -72,9 +78,44 @@ async fn run(config: BenchConfig) -> ExitCode { return ExitCode::FAILURE; } }; + let status = std::fs::read_to_string("/proc/self/status").unwrap_or_default(); + let cpus = status + .lines() + .find_map(|line| line.strip_prefix("Cpus_allowed_list:").map(str::trim)) + .unwrap_or("unsupported"); + println!( + "bench-runtime: pid={} allowed_cpus={cpus} tokio_workers={}", + std::process::id(), + std::env::var("TOKIO_WORKER_THREADS").unwrap_or_else(|_| "default".into()), + ); match workload::run(&stream_runtime, &config).await { Ok(result) => { println!("{result}"); + for (name, phase) in crowdb_chunk_stream::stream_perf_metrics() { + println!( + "stream-phase: name={name} count={} avg_us={:.2} p50_us={} p99_us={} max_us={}", + phase.count, + phase.avg / 1_000.0, + phase.p50 / 1_000, + phase.p99 / 1_000, + phase.max / 1_000, + ); + } + for name in [ + "chunkio.diskio.write.e2e.lh", + "rpc.request.e2e", + "rpc.response.schedule.lh", + ] { + let phase = metrics::global_histogram(name).snapshot_total(); + println!( + "client-phase: name={name} count={} avg_us={:.2} p50_us={} p99_us={} max_us={}", + phase.count, + phase.avg / 1_000.0, + phase.p50 / 1_000, + phase.p99 / 1_000, + phase.max / 1_000, + ); + } if result.errors == 0 && result.operations > 0 { ExitCode::SUCCESS } else { diff --git a/lib/crowdb-chunk-stream/benches/production_bounds/config.rs b/lib/crowdb-chunk-stream/benches/production_bounds/config.rs index e7f2a5d24..c34147d31 100644 --- a/lib/crowdb-chunk-stream/benches/production_bounds/config.rs +++ b/lib/crowdb-chunk-stream/benches/production_bounds/config.rs @@ -53,7 +53,7 @@ impl Default for BenchConfig { fn default() -> Self { Self { workload: Workload::Append, - management_seed: "http://127.0.0.1:10000".into(), + management_seed: String::new(), duration_secs: 10, operations: u64::MAX, object_size: 4 * 1024, @@ -89,6 +89,9 @@ impl BenchConfig { _ => return Err(format!("unknown flag {flag}")), } } + if config.management_seed.is_empty() { + return Err("--management-seed is required for an existing target".into()); + } if config.duration_secs == 0 || config.operations == 0 || config.object_size == 0 diff --git a/lib/crowdb-chunk-stream/benches/production_bounds/result.rs b/lib/crowdb-chunk-stream/benches/production_bounds/result.rs index f3c4caa7b..4c7e796b4 100644 --- a/lib/crowdb-chunk-stream/benches/production_bounds/result.rs +++ b/lib/crowdb-chunk-stream/benches/production_bounds/result.rs @@ -98,8 +98,8 @@ impl Display for BenchResult { self.metrics.batch_requests, self.metrics.rollovers, self.metrics.metadata_publications, - self.metrics.extent_page_cache_hits, - self.metrics.extent_page_cache_misses, + self.metrics.record_cache_hits, + self.metrics.record_cache_misses, self.metrics.physical_read_requests, self.metrics.reclaimed_bytes, self.metrics.watchdog_observations, diff --git a/lib/crowdb-chunk-stream/benches/production_bounds/workload.rs b/lib/crowdb-chunk-stream/benches/production_bounds/workload.rs index b84638c56..c9530da1b 100644 --- a/lib/crowdb-chunk-stream/benches/production_bounds/workload.rs +++ b/lib/crowdb-chunk-stream/benches/production_bounds/workload.rs @@ -8,7 +8,7 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use bytes::Bytes; use crowdb_chunk_client::STREAM_CHUNK_BYTES; use crowdb_chunk_stream::{ - ChunkStream, ProductionStreamRuntime, ReadHint, Result, StreamBinding, StreamBindingState, StreamName, + ChunkStream, ProductionStreamRuntime, ReadHint, Result, StreamName, StreamRecord, StreamRecordState, StreamRegistry, }; @@ -57,13 +57,15 @@ async fn create_stream(runtime: &ProductionStreamRuntime) -> Result }; runtime .registry() - .create(StreamBinding { + .create(StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, stream_name, metadata_group_id: 1, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("production-benchmark".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("production-benchmark".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }) .await?; runtime.create_registered(stream_name, 0, 1).await diff --git a/lib/crowdb-chunk-stream/src/authority.rs b/lib/crowdb-chunk-stream/src/authority.rs new file mode 100644 index 000000000..23e3ad1bf --- /dev/null +++ b/lib/crowdb-chunk-stream/src/authority.rs @@ -0,0 +1,122 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Captured writer authority shared by admission, physical IO and acknowledgment. + +use crate::{Result, StreamError}; +use async_trait::async_trait; +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::Instant, +}; + +#[async_trait] +pub trait StreamOwnerAuthority: Send + Sync { + /// Conservative local deadline of the currently live owner scope. + /// # Errors + /// Rejects expired or replaced authority. + fn deadline(&self) -> Result; + /// Called only after stopping admission, draining IO and checkpointing. + /// # Errors + /// Returns an authority or durable release error. + async fn release(&self) -> Result<()>; +} + +pub struct StreamWriterSession { + authority: Arc, + fenced: AtomicBool, + released: AtomicBool, +} + +#[derive(Clone)] +pub struct StreamWriteContext { + session: Arc, + deadline: Instant, +} + +impl StreamWriterSession { + #[must_use] + pub fn new(authority: Arc) -> Arc { + Arc::new(Self { + authority, + fenced: AtomicBool::new(false), + released: AtomicBool::new(false), + }) + } + + /// # Errors + /// Rejects fenced or expired ownership. + pub fn capture(self: &Arc) -> Result { + let deadline = self.check()?; + Ok(StreamWriteContext { + session: Arc::clone(self), + deadline, + }) + } + + /// # Errors + /// Rejects fenced or expired ownership. + pub fn check(&self) -> Result { + if self.fenced.load(Ordering::Acquire) { + return Err(StreamError::StaleWriter); + } + let deadline = self.authority.deadline()?; + if Instant::now() >= deadline { + self.fence(); + return Err(StreamError::StaleWriter); + } + Ok(deadline) + } + + pub fn fence(&self) { + self.fenced.store(true, Ordering::Release); + } + + #[must_use] + pub fn is_released(&self) -> bool { + self.released.load(Ordering::Acquire) + } + + /// # Errors + /// Returns a durable owner-release error; admission stays fenced. + pub async fn release(&self) -> Result<()> { + self.fence(); + self.authority.release().await?; + self.released.store(true, Ordering::Release); + Ok(()) + } +} + +impl StreamWriteContext { + #[must_use] + pub const fn deadline(&self) -> Instant { + self.deadline + } + /// IO retains its original deadline even when the process lease renews. + /// # Errors + /// Rejects revoked authority or an elapsed submission deadline. + pub fn check_io(&self) -> Result<()> { + self.session.check()?; + if Instant::now() >= self.deadline { + return Err(StreamError::StaleWriter); + } + Ok(()) + } + /// An uninterrupted renewal may authorize acknowledgment after old IO drained. + /// # Errors + /// Rejects revoked or expired authority. + pub fn check_ack(&self) -> Result<()> { + self.session.check().map(|_| ()) + } +} + +impl std::fmt::Debug for StreamWriterSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StreamWriterSession") + .field("fenced", &self.fenced.load(Ordering::Acquire)) + .finish_non_exhaustive() + } +} diff --git a/lib/crowdb-chunk-stream/src/error.rs b/lib/crowdb-chunk-stream/src/error.rs index ca811e730..889342e41 100644 --- a/lib/crowdb-chunk-stream/src/error.rs +++ b/lib/crowdb-chunk-stream/src/error.rs @@ -11,8 +11,6 @@ pub enum StreamError { Backpressure, #[error("writer epoch is stale")] StaleWriter, - #[error("append definitely did not commit: {0}")] - DefinitelyNotCommitted(String), #[error("append outcome is being resolved")] ResolvingAmbiguousAppend, #[error("stream writes are stalled pending reopen")] diff --git a/lib/crowdb-chunk-stream/src/extent_cache.rs b/lib/crowdb-chunk-stream/src/extent_cache.rs deleted file mode 100644 index 6e21985e1..000000000 --- a/lib/crowdb-chunk-stream/src/extent_cache.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Lock-free extent-page snapshots shared by readers of one stream handle. - -use std::collections::HashMap; -use std::sync::Arc; - -use arc_swap::ArcSwap; -use crowdb_protocol::chunk_stream::{StreamExtentPage, StreamManifest}; - -#[derive(Clone)] -struct CacheSnapshot { - writer_epoch: u64, - generation: u64, - pages: HashMap>, -} - -pub(crate) struct ExtentPageCache { - snapshot: ArcSwap, -} - -impl ExtentPageCache { - pub(crate) fn new(manifest: &StreamManifest, pages: &[StreamExtentPage]) -> Self { - Self { - snapshot: ArcSwap::from_pointee(CacheSnapshot { - writer_epoch: manifest.writer_epoch, - generation: manifest.generation, - pages: pages - .iter() - .filter(|page| { - page.writer_epoch == manifest.writer_epoch && page.generation == manifest.generation - }) - .map(|page| (page.page_index, Arc::new(page.clone()))) - .collect(), - }), - } - } - - pub(crate) fn get( - &self, - writer_epoch: u64, - generation: u64, - page_index: u64, - ) -> Option> { - let snapshot = self.snapshot.load(); - (snapshot.writer_epoch == writer_epoch && snapshot.generation == generation) - .then(|| snapshot.pages.get(&page_index).cloned()) - .flatten() - } - - pub(crate) fn insert(&self, page: StreamExtentPage) -> Arc { - let page = Arc::new(page); - let inserted = Arc::clone(&page); - self.snapshot.rcu(move |current| { - let current_identity = (current.writer_epoch, current.generation); - let page_identity = (page.writer_epoch, page.generation); - if page_identity < current_identity { - return Arc::clone(current); - } - let mut pages = if page_identity == current_identity { - current.pages.clone() - } else { - HashMap::new() - }; - pages.insert(page.page_index, Arc::clone(&page)); - Arc::new(CacheSnapshot { - writer_epoch: page.writer_epoch, - generation: page.generation, - pages, - }) - }); - inserted - } -} diff --git a/lib/crowdb-chunk-stream/src/kv.rs b/lib/crowdb-chunk-stream/src/kv.rs index e0a17d31c..2f7e5409e 100644 --- a/lib/crowdb-chunk-stream/src/kv.rs +++ b/lib/crowdb-chunk-stream/src/kv.rs @@ -7,95 +7,106 @@ use std::sync::Arc; use async_trait::async_trait; use crowdb_kv_client::{CrowdbKvClient, GetOutcome, ReadMode}; -use crowdb_protocol::chunk_stream::{StreamBinding, StreamExtentPage, StreamManifest, StreamName}; -use crowdb_protocol::key::{ - BinaryKey, StreamBindingKey, StreamExtentPageKey, StreamManifestHeadKey, TextKey, -}; +use crowdb_protocol::chunk_stream::{StreamName, StreamRecord}; +use crowdb_protocol::key::{BinaryKey, StreamRecordKey}; + +mod owner; +mod record_backend; use crate::{Result, StreamError, StreamMetadataStore, StreamRegistry}; -/// Group-0 readable stream binding registry. +/// Group-0 readable stream record registry. pub struct KvStreamRegistry { kv: Arc, + processes: owner::OwnerProcesses, } impl KvStreamRegistry { #[must_use] pub fn new(kv: Arc) -> Self { - Self { kv } + Self { + kv, + processes: owner::OwnerProcesses::default(), + } } } #[async_trait] impl StreamRegistry for KvStreamRegistry { - async fn load(&self, stream_name: StreamName) -> Result> { - let key = StreamBindingKey { stream_name }.to_path(); + async fn claim_writer(&self, name: StreamName, epoch: u64) -> Result> { + let record = self + .load(name) + .await? + .ok_or_else(|| StreamError::InvalidRequest("stream record is absent".into()))?; + self.acquire_owner(name, epoch, record.purpose).await + } + async fn load(&self, stream_name: StreamName) -> Result> { + let key = StreamRecordKey { stream_name }.to_bytes(); match self .kv - .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .get(0, 0, &key, ReadMode::Linearizable, None) .await - .map_err(kv_error)? + .map_err(|error| kv_error(&error))? { GetOutcome::Found { value, .. } => { - let binding: StreamBinding = serde_json::from_slice(&value).map_err(|error| { - StreamError::Corruption(format!("stream binding decode failed: {error}")) + let record: StreamRecord = serde_json::from_slice(&value).map_err(|error| { + StreamError::Corruption(format!("stream record decode failed: {error}")) })?; - if binding.stream_name != stream_name || binding.metadata_group_id == 0 { + if record.stream_name != stream_name + || record.metadata_group_id == 0 + || record.record_version != 1 + || record.created_at_ms == 0 + { return Err(StreamError::Corruption( - "stream binding identity or metadata group is invalid".into(), + "stream record identity or metadata group is invalid".into(), )); } - Ok(Some(binding)) + Ok(Some(record)) } GetOutcome::NotFound => Ok(None), } } - async fn create(&self, binding: StreamBinding) -> Result<()> { - if binding.metadata_group_id == 0 || binding.binding_generation == 0 { + async fn create(&self, record: StreamRecord) -> Result<()> { + if record.metadata_group_id == 0 || record.record_version != 1 || record.created_at_ms == 0 { return Err(StreamError::InvalidRequest( - "stream binding requires a nonzero metadata group and generation".into(), + "stream record requires a valid current version, creation time and metadata group".into(), )); } - let key = StreamBindingKey { - stream_name: binding.stream_name, + let key = StreamRecordKey { + stream_name: record.stream_name, } - .to_path(); - let value = serde_json::to_vec(&binding) - .map_err(|error| StreamError::Internal(format!("stream binding encode failed: {error}")))?; - match self.kv.put_cas(0, 0, key.as_bytes(), &value, 0).await { + .to_bytes(); + let value = serde_json::to_vec(&record) + .map_err(|error| StreamError::Internal(format!("stream record encode failed: {error}")))?; + match self.kv.put_cas(0, 0, &key, &value, 0).await { Ok(_) => Ok(()), - Err(crowdb_kv_client::Error::CasFailed { .. }) => Err(StreamError::InvalidRequest( - "stream binding already exists".into(), - )), + Err(crowdb_kv_client::Error::CasFailed { .. }) => { + Err(StreamError::InvalidRequest("stream record already exists".into())) + } Err(crowdb_kv_client::Error::CasBusy) => Err(StreamError::Backpressure), - Err(crowdb_kv_client::Error::OutcomeUnknown) => match self.load(binding.stream_name).await? { - Some(observed) if observed == binding => Ok(()), + Err(crowdb_kv_client::Error::OutcomeUnknown) => match self.load(record.stream_name).await? { + Some(observed) if observed == record => Ok(()), _ => Err(StreamError::WriteStalled), }, - Err(error) => Err(kv_error(error)), + Err(error) => Err(kv_error(&error)), } } } -/// One nonzero KV-group metadata store. Extent pages are immutable fresh keys; -/// the manifest is a stable revision-CAS head. +/// KV adapter for one nonzero Stream metadata group. pub struct KvStreamMetadataStore { kv: Arc, store_id: u64, group_id: u64, } - impl KvStreamMetadataStore { - /// Creates an adapter for one nonzero metadata group. - /// /// # Errors - /// - /// Returns an error when group zero is selected for stream metadata. + /// Rejects group zero, reserved for Stream identity and ownership records. pub fn new(kv: Arc, store_id: u64, group_id: u64) -> Result { if group_id == 0 { return Err(StreamError::InvalidRequest( - "stream metadata must use a nonzero KV group".into(), + "Stream chunk records require a nonzero KV group".into(), )); } Ok(Self { @@ -104,278 +115,24 @@ impl KvStreamMetadataStore { group_id, }) } - - async fn get_head(&self, stream_name: StreamName) -> Result> { - let key = StreamManifestHeadKey { stream_name }.to_bytes(); - match self - .kv - .get(self.store_id, self.group_id, &key, ReadMode::Linearizable, None) - .await - .map_err(kv_error)? - { - GetOutcome::Found { value, revision } => { - let manifest: StreamManifest = decode(&value)?; - if manifest.stream_name != stream_name || manifest.metadata_group_id != self.group_id { - return Err(StreamError::Corruption( - "stream manifest identity or metadata group is invalid".into(), - )); - } - Ok(Some((manifest, revision))) - } - GetOutcome::NotFound => Ok(None), - } - } - - async fn put_immutable_page(&self, page: &StreamExtentPage) -> Result<()> { - if page.chunk_ids.is_empty() - || page.logical_offsets.len() != page.chunk_ids.len() + 1 - || page.physical_offsets.len() != page.chunk_ids.len() - || page - .logical_offsets - .windows(2) - .any(|window| window[0] >= window[1]) - { - return Err(StreamError::Corruption( - "refusing to publish malformed extent page".into(), - )); - } - let key = StreamExtentPageKey { - stream_name: page.stream_name, - writer_epoch: page.writer_epoch, - generation: page.generation, - page_index: page.page_index, - } - .to_bytes(); - let value = encode(page)?; - match self - .kv - .put_cas(self.store_id, self.group_id, &key, &value, 0) - .await - { - Ok(_) => Ok(()), - Err( - error @ (crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::OutcomeUnknown), - ) => { - match self - .kv - .get(self.store_id, self.group_id, &key, ReadMode::Linearizable, None) - .await - .map_err(kv_error)? - { - GetOutcome::Found { value, .. } => { - let observed: StreamExtentPage = decode(&value)?; - if observed == *page { - Ok(()) - } else { - tracing::warn!( - stream_high = page.stream_name.high, - stream_low = page.stream_name.low, - writer_epoch = page.writer_epoch, - generation = page.generation, - page_index = page.page_index, - candidate = ?page, - observed = ?observed, - "immutable extent-page key has conflicting contents" - ); - Err(StreamError::Corruption( - "immutable extent-page key contains another value".into(), - )) - } - } - GetOutcome::NotFound => { - tracing::warn!( - stream_high = page.stream_name.high, - stream_low = page.stream_name.low, - writer_epoch = page.writer_epoch, - generation = page.generation, - page_index = page.page_index, - %error, - "immutable extent-page CAS was unresolved and its key is absent" - ); - Err(StreamError::WriteStalled) - } - } - } - Err(crowdb_kv_client::Error::CasBusy) => Err(StreamError::Backpressure), - Err(error) => Err(kv_error(error)), - } - } } - -#[async_trait] impl StreamMetadataStore for KvStreamMetadataStore { - async fn load_current(&self, stream_name: StreamName) -> Result> { - Ok(self.get_head(stream_name).await?.map(|(manifest, _)| manifest)) - } - - async fn load_extent_page( - &self, - stream_name: StreamName, - writer_epoch: u64, - generation: u64, - page_index: u64, - ) -> Result> { - let key = StreamExtentPageKey { - stream_name, - writer_epoch, - generation, - page_index, - } - .to_bytes(); - match self - .kv - .get(self.store_id, self.group_id, &key, ReadMode::Linearizable, None) - .await - .map_err(kv_error)? - { - GetOutcome::Found { value, .. } => { - let page: StreamExtentPage = decode(&value)?; - if page.stream_name != stream_name - || page.writer_epoch != writer_epoch - || page.generation != generation - || page.page_index != page_index - { - return Err(StreamError::Corruption( - "stream extent-page identity is invalid".into(), - )); - } - Ok(Some(page)) - } - GetOutcome::NotFound => Ok(None), - } - } - - async fn publish( - &self, - expected: Option<(u64, u64)>, - manifest: StreamManifest, - extent_pages: Vec, - ) -> Result<()> { - if manifest.metadata_group_id != self.group_id { - return Err(StreamError::InvalidRequest( - "manifest metadata group differs from store binding".into(), - )); - } - if extent_pages.iter().any(|page| { - page.stream_name != manifest.stream_name - || page.writer_epoch != manifest.writer_epoch - || page.generation != manifest.generation - }) { - return Err(StreamError::Corruption( - "extent page identity differs from candidate manifest".into(), - )); - } - for page in &extent_pages { - self.put_immutable_page(page).await?; - } - let observed = self.get_head(manifest.stream_name).await?; - if observed - .as_ref() - .is_some_and(|(head, _)| head.purpose != manifest.purpose) - { - return Err(StreamError::Corruption("stream purpose is immutable".into())); - } - if observed - .as_ref() - .map(|(head, _)| (head.writer_epoch, head.generation)) - != expected - { - if observed.as_ref().is_some_and(|(head, _)| head == &manifest) { - return Ok(()); - } - if observed - .as_ref() - .is_some_and(|(head, _)| head.writer_epoch > manifest.writer_epoch) - { - return Err(StreamError::StaleWriter); - } - return Err(StreamError::Internal( - "stream manifest generation conflict".into(), - )); - } - let expected_revision = observed.map_or(0, |(_, revision)| revision); - let key = StreamManifestHeadKey { - stream_name: manifest.stream_name, - } - .to_bytes(); - let value = encode(&manifest)?; - match self - .kv - .put_cas(self.store_id, self.group_id, &key, &value, expected_revision) - .await - { - Ok(_) => Ok(()), - Err(crowdb_kv_client::Error::CasBusy) => Err(StreamError::Backpressure), - Err(crowdb_kv_client::Error::CasFailed { .. } | crowdb_kv_client::Error::OutcomeUnknown) => { - match self.get_head(manifest.stream_name).await? { - Some((current, _)) if current == manifest => Ok(()), - Some((current, _)) if current.writer_epoch > manifest.writer_epoch => { - Err(StreamError::StaleWriter) - } - _ => Err(StreamError::WriteStalled), - } - } - Err(error) => Err(kv_error(error)), - } - } - - async fn reclaim_extent_pages_before( - &self, - stream_name: StreamName, - retained_generation: u64, - max_pages: usize, - ) -> Result { - if retained_generation == 0 || max_pages == 0 { + fn record_store(&self, group: u64) -> Result { + if group != self.group_id { return Err(StreamError::InvalidRequest( - "metadata reclaim watermark and page bound must be nonzero".into(), + "Stream record group differs from store".into(), )); } - let prefix = StreamExtentPageKey::stream_prefix(stream_name); - let limit = u32::try_from(max_pages).unwrap_or(u32::MAX); - let candidates = self - .kv - .scan( - self.store_id, - self.group_id, - &prefix, - &[], - &[], - limit, - ReadMode::Linearizable, - None, - true, - None, - ) - .await - .map_err(kv_error)?; - let mut reclaimed = 0_u64; - for (key, _) in candidates.items { - let page_key = StreamExtentPageKey::from_bytes(&key).map_err(|error| { - StreamError::Corruption(format!("stream extent-page key decode failed: {error}")) - })?; - if page_key.stream_name == stream_name && page_key.generation < retained_generation { - self.kv - .delete(self.store_id, self.group_id, &key, None) - .await - .map_err(kv_error)?; - reclaimed += 1; - } - } - Ok(reclaimed) + crate::StreamChunkRecordStore::new( + Arc::new(record_backend::KvRecordBackend { + kv: self.kv.clone(), + store: self.store_id, + group, + }), + group, + ) } } - -fn encode(value: &T) -> Result> { - bincode::serialize(value) - .map_err(|error| StreamError::Internal(format!("stream metadata encode failed: {error}"))) -} - -fn decode(value: &[u8]) -> Result { - bincode::deserialize(value) - .map_err(|error| StreamError::Corruption(format!("stream metadata decode failed: {error}"))) -} - -#[allow(clippy::needless_pass_by_value)] -fn kv_error(error: crowdb_kv_client::Error) -> StreamError { - StreamError::Internal(format!("stream KV operation failed: {error}")) +fn kv_error(error: &crowdb_kv_client::Error) -> StreamError { + StreamError::ReadUnavailable(format!("Stream KV operation failed: {error}")) } diff --git a/lib/crowdb-chunk-stream/src/kv/owner.rs b/lib/crowdb-chunk-stream/src/kv/owner.rs new file mode 100644 index 000000000..3b86c8241 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/kv/owner.rs @@ -0,0 +1,320 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Group-0 owner assignment and independent process-lease renewal. + +use super::{kv_error, KvStreamRegistry}; +use crate::{Result, StreamError, StreamName, StreamOwnerAuthority, StreamWriterSession}; +use arc_swap::ArcSwapOption; +use async_trait::async_trait; +use crowdb_kv_client::{ + CrowdbKvClient, GetOutcome, ReadMode, StreamProcessAuthority, StreamProcessLeaseClient, + StreamProcessLeaseError, StreamProcessLeaseTiming, +}; +use crowdb_protocol::{ + chunk_stream::{StreamProcessLease, StreamPurpose, StreamWriterOwner}, + key::{StreamWriterOwnerKey, TextKey}, + timing::{MAX_WRITE_CLOCK_SKEW_MS, MAX_WRITE_REQUEST_AGE_MS}, +}; +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +const LEASE_MS: u64 = 15_000; + +pub(super) struct ProcessOwner { + authority: Arc, + renewal: Option>, +} + +impl Drop for ProcessOwner { + fn drop(&mut self) { + if let Some(task) = &self.renewal { + self.authority.fence(); + task.abort(); + } + } +} + +pub(super) struct OwnerProcesses { + slots: [ArcSwapOption; 2], + starting: AtomicBool, +} + +impl Default for OwnerProcesses { + fn default() -> Self { + Self { + slots: std::array::from_fn(|_| ArcSwapOption::empty()), + starting: AtomicBool::new(false), + } + } +} + +struct Starting<'a>(&'a AtomicBool); +impl Drop for Starting<'_> { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } +} + +impl KvStreamRegistry { + /// Compose WAL ownership with the server's already-renewed process scope. + /// # Errors + /// Rejects dead authority or a runtime already bound to a process. + pub fn bind_owner_process(&self, authority: Arc) -> Result<()> { + let context = authority + .capture(Instant::now()) + .map_err(|_| StreamError::StaleWriter)?; + let slot = &self.processes.slots[usize::from(context.identity().purpose == StreamPurpose::Wal)]; + let owner = Arc::new(ProcessOwner { + authority, + renewal: None, + }); + if slot + .compare_and_swap(&None::>, Some(owner)) + .is_some() + { + return Err(StreamError::InvalidRequest( + "stream runtime already has a process owner".into(), + )); + } + Ok(()) + } + + async fn owner_process(&self, purpose: StreamPurpose) -> Result> { + let slot = &self.processes.slots[usize::from(purpose == StreamPurpose::Wal)]; + if let Some(owner) = slot.load_full() { + return Ok(owner); + } + if self + .processes + .starting + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err(StreamError::Backpressure); + } + let _starting = Starting(&self.processes.starting); + if let Some(owner) = slot.load_full() { + return Ok(owner); + } + let nonce = + StreamProcessLease::generate_boot_nonce().map_err(|e| StreamError::Internal(e.to_string()))?; + let instance = u64::from_le_bytes(nonce[..8].try_into().expect("nonce half")).max(1); + let client = Arc::new(StreamProcessLeaseClient::new(self.kv.clone())); + let lease = client + .claim(purpose, instance, nonce, None, wall_ms(), LEASE_MS) + .await + .map_err(|e| StreamError::ReadUnavailable(e.to_string()))?; + let authority = Arc::new( + StreamProcessAuthority::new( + lease, + StreamProcessLeaseTiming::default(), + wall_ms(), + Instant::now(), + ) + .map_err(|e| StreamError::ReadUnavailable(e.to_string()))?, + ); + let renewal = tokio::spawn(renew_process(client, authority.clone())); + let owner = Arc::new(ProcessOwner { + authority, + renewal: Some(renewal), + }); + slot.store(Some(owner.clone())); + Ok(owner) + } + + pub(super) async fn acquire_owner( + &self, + name: StreamName, + epoch: u64, + purpose: StreamPurpose, + ) -> Result> { + let process = self.owner_process(purpose).await?; + let context = process + .authority + .capture(Instant::now()) + .map_err(|_| StreamError::StaleWriter)?; + let previous = read_owner(&self.kv, name).await?; + let mut revision = previous.as_ref().map_or(0, |(_, revision)| *revision); + if let Some((owner, _)) = &previous { + if owner.process.purpose != purpose { + return Err(StreamError::Corruption( + "owner purpose differs from stream record".into(), + )); + } + if epoch < owner.writer_epoch || (!owner.released && owner.process == context.identity()) { + return Err(StreamError::StaleWriter); + } + if !owner.released { + revision = exclude_previous(&self.kv, &process.authority, owner).await?; + } + } + process + .authority + .capture(Instant::now()) + .map_err(|_| StreamError::StaleWriter)?; + let owner = StreamWriterOwner { + stream_name: name, + writer_epoch: epoch, + process: context.identity(), + assignment_nonce: StreamProcessLease::generate_boot_nonce() + .map_err(|error| StreamError::Internal(error.to_string()))?, + released: false, + }; + write_owner(&self.kv, &owner, revision).await?; + Ok(StreamWriterSession::new(Arc::new(AssignedOwner { + kv: self.kv.clone(), + process, + owner, + }))) + } +} + +struct AssignedOwner { + kv: Arc, + process: Arc, + owner: StreamWriterOwner, +} + +#[async_trait] +impl StreamOwnerAuthority for AssignedOwner { + fn deadline(&self) -> Result { + self.process + .authority + .capture(Instant::now()) + .map(|context| context.deadline()) + .map_err(|_| StreamError::StaleWriter) + } + async fn release(&self) -> Result<()> { + let Some((mut current, revision)) = read_owner(&self.kv, self.owner.stream_name).await? else { + return Err(StreamError::StaleWriter); + }; + if current.process != self.owner.process + || current.writer_epoch != self.owner.writer_epoch + || current.assignment_nonce != self.owner.assignment_nonce + { + return Err(StreamError::StaleWriter); + } + if current.released { + return Ok(()); + } + current.released = true; + write_owner(&self.kv, ¤t, revision).await + } +} + +async fn exclude_previous( + kv: &Arc, + current: &StreamProcessAuthority, + owner: &StreamWriterOwner, +) -> Result { + let client = StreamProcessLeaseClient::new(kv.clone()); + loop { + current + .capture(Instant::now()) + .map_err(|_| StreamError::StaleWriter)?; + let (assignment, revision) = read_owner(kv, owner.stream_name) + .await? + .ok_or(StreamError::StaleWriter)?; + if assignment.process != owner.process + || assignment.writer_epoch != owner.writer_epoch + || assignment.assignment_nonce != owner.assignment_nonce + { + return Err(StreamError::StaleWriter); + } + if assignment.released { + return Ok(revision); + } + let previous = client + .load(owner.process.purpose, owner.process.instance_id) + .await + .map_err(|e| StreamError::ReadUnavailable(e.to_string()))? + .ok_or_else(|| StreamError::Corruption("stream owner process record is missing".into()))?; + let expiry = if previous.identity == owner.process { + previous.expires_at_ms.max(previous.grant_expires_at_ms) + } else if previous.identity.process_epoch > owner.process.process_epoch { + previous.prior_exclusion_deadline_ms + } else { + return Err(StreamError::StaleWriter); + }; + let safe = expiry + .saturating_add(MAX_WRITE_CLOCK_SKEW_MS) + .saturating_add(MAX_WRITE_REQUEST_AGE_MS); + if wall_ms() >= safe { + return Ok(revision); + } + tokio::time::sleep(Duration::from_millis((safe - wall_ms()).min(500))).await; + } +} + +async fn read_owner(kv: &CrowdbKvClient, name: StreamName) -> Result> { + let key = StreamWriterOwnerKey { stream_name: name }.to_path(); + match kv + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await + .map_err(|error| kv_error(&error))? + { + GetOutcome::NotFound => Ok(None), + GetOutcome::Found { value, revision } => { + let owner: StreamWriterOwner = + serde_json::from_slice(&value).map_err(|e| StreamError::Corruption(e.to_string()))?; + if owner.stream_name != name || owner.validate().is_err() { + return Err(StreamError::Corruption("invalid stream owner assignment".into())); + } + Ok(Some((owner, revision))) + } + } +} + +async fn write_owner(kv: &CrowdbKvClient, owner: &StreamWriterOwner, revision: u64) -> Result<()> { + let key = StreamWriterOwnerKey { + stream_name: owner.stream_name, + } + .to_path(); + let value = serde_json::to_vec(owner).map_err(|e| StreamError::Internal(e.to_string()))?; + match kv.put_cas(0, 0, key.as_bytes(), &value, revision).await { + Ok(_) => Ok(()), + Err(error) => { + if read_owner(kv, owner.stream_name) + .await? + .as_ref() + .is_some_and(|(current, _)| current == owner) + { + Ok(()) + } else { + Err(kv_error(&error)) + } + } + } +} + +async fn renew_process(client: Arc, authority: Arc) { + while let Ok(context) = authority.capture(Instant::now()) { + tokio::time::sleep(Duration::from_millis(LEASE_MS / 3)).await; + let renewal = client.renew(context.identity(), wall_ms(), LEASE_MS); + match tokio::time::timeout_at(context.deadline().into(), renewal).await { + Ok(Ok(lease)) => { + if authority.observe(lease, wall_ms(), Instant::now()).is_err() { + break; + } + } + Ok(Err(StreamProcessLeaseError::Replaced | StreamProcessLeaseError::Expired)) => break, + Ok(Err(error)) => { + tracing::warn!(%error, "stream process renewal unavailable; retaining deadline"); + } + _ => break, + } + } + authority.fence(); +} + +fn wall_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |time| u64::try_from(time.as_millis()).unwrap_or(u64::MAX)) +} diff --git a/lib/crowdb-chunk-stream/src/kv/record_backend.rs b/lib/crowdb-chunk-stream/src/kv/record_backend.rs new file mode 100644 index 000000000..437de03a4 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/kv/record_backend.rs @@ -0,0 +1,85 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::perf::{perf, PhaseTimer}; +use crate::records::RecordBackend; +use async_trait::async_trait; +use crowdb_kv_client::{BatchOp, CrowdbKvClient, GetOutcome, ReadMode, Result, ScanOutcome, WriteOutcome}; +use std::sync::Arc; + +pub(super) struct KvRecordBackend { + pub(super) kv: Arc, + pub(super) store: u64, + pub(super) group: u64, +} +#[async_trait] +impl RecordBackend for KvRecordBackend { + async fn get(&self, key: &[u8]) -> Result { + let _timer = PhaseTimer::start(&perf().record_get); + self.kv + .get(self.store, self.group, key, ReadMode::Linearizable, None) + .await + } + async fn scan(&self, prefix: &[u8], after: &[u8], limit: u32, reverse: bool) -> Result { + self.scan_records(prefix, after, limit, reverse, false).await + } + async fn scan_keys(&self, prefix: &[u8], after: &[u8], limit: u32) -> Result { + self.scan_records(prefix, after, limit, false, true).await + } + async fn batch(&self, ops: &[BatchOp], key: &[u8], revision: u64) -> Result { + let _timer = PhaseTimer::start(&perf().record_cas); + self.kv + .batch_write_cas(self.store, self.group, ops, key, revision) + .await + } +} + +impl KvRecordBackend { + async fn scan_records( + &self, + prefix: &[u8], + after: &[u8], + limit: u32, + reverse: bool, + keys_only: bool, + ) -> Result { + let _timer = PhaseTimer::start(&perf().record_scan); + let outcome = if reverse { + self.kv + .scan_reverse( + self.store, + self.group, + prefix, + after, + &[], + limit, + ReadMode::Linearizable, + None, + keys_only, + None, + ) + .await + } else { + self.kv + .scan( + self.store, + self.group, + prefix, + after, + &[], + limit, + ReadMode::Linearizable, + None, + keys_only, + None, + ) + .await + }?; + if outcome.timed_out || outcome.items.len() != outcome.commit_slots.len() { + return Err(crowdb_kv_client::Error::Server( + "incomplete Stream chunk record scan".into(), + )); + } + Ok(outcome) + } +} diff --git a/lib/crowdb-chunk-stream/src/lib.rs b/lib/crowdb-chunk-stream/src/lib.rs index 741408efd..56e53a851 100644 --- a/lib/crowdb-chunk-stream/src/lib.rs +++ b/lib/crowdb-chunk-stream/src/lib.rs @@ -3,36 +3,44 @@ //! Fenced, mirrored logical byte streams assembled from finite chunks. +mod authority; mod error; -mod extent_cache; mod kv; mod metadata; mod metrics; mod mirror_shadow; +mod perf; mod production; mod production_chunk; +mod records; +pub use records::{StreamChunkRecordSnapshot, StreamChunkRecordStore, StreamCollectionResult, StreamReadPin}; mod storage; mod stream; #[cfg(feature = "test-util")] pub mod memory; +pub use authority::{StreamOwnerAuthority, StreamWriteContext, StreamWriterSession}; pub use error::{Result, StreamError}; pub use kv::{KvStreamMetadataStore, KvStreamRegistry}; -pub use metadata::{resolve_extent, validate_manifest, ExtentLocation}; +pub use metadata::ExtentLocation; pub use metrics::{StreamMetrics, StreamMetricsSnapshot}; +pub use perf::{register_stream_perf_metrics, stream_perf_metrics}; pub use production::ProductionStreamRuntime; pub use production_chunk::ProductionStreamChunkStore; pub use storage::{ - CursorAdvance, DurableCursor, MirrorStripImage, StreamChunkStore, StreamMetadataStore, StreamRegistry, - TrimmedChunk, + CursorAdvance, DurableCursor, MirrorStripImage, RecoveredStreamTail, StreamChunkStore, + StreamMetadataStore, StreamMirrorWrite, StreamRegistry, TrimmedChunk, }; +#[cfg(feature = "test-util")] +pub use stream::capacity_target_for_tests; pub use stream::{ AppendRange, ChunkStream, ReadHint, ReadSegment, StreamConfig, StreamMetadataObservation, StreamReader, + StreamViewSource, }; pub use crowdb_protocol::chunk_stream::{ - ActiveChunkDescriptor, StreamBinding, StreamBindingState, StreamExtentPage, StreamExtentPageFence, - StreamManifest, StreamName, StreamPurpose, DEFAULT_STREAM_METADATA_GROUP_ID, + ActiveChunkDescriptor, StreamName, StreamPurpose, StreamRecord, StreamRecordState, + DEFAULT_STREAM_METADATA_GROUP_ID, }; pub use crowdb_protocol::common::ChunkId; diff --git a/lib/crowdb-chunk-stream/src/memory.rs b/lib/crowdb-chunk-stream/src/memory.rs index ecea3d213..73ac9fc8b 100644 --- a/lib/crowdb-chunk-stream/src/memory.rs +++ b/lib/crowdb-chunk-stream/src/memory.rs @@ -3,20 +3,25 @@ //! In-memory storage and deterministic fault controls for stream tests. -use std::collections::{HashMap, VecDeque}; +use std::collections::{BTreeMap, HashMap, VecDeque}; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +use std::sync::Arc; use async_trait::async_trait; use bytes::Bytes; -use crowdb_protocol::chunk_stream::{ - ActiveChunkDescriptor, StreamBinding, StreamExtentPage, StreamManifest, StreamName, -}; +use crowdb_protocol::chunk_stream::{ActiveChunkDescriptor, StreamName, StreamRecord}; use crowdb_protocol::common::ChunkId; -use crowdb_protocol::frame::{FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES}; +use crowdb_protocol::frame::FrameLayout; use tokio::sync::{Mutex, Notify}; +mod allocation; +mod authority; +mod metadata; +mod record_backend; +mod recovery; + use crate::{ - CursorAdvance, DurableCursor, Result, StreamChunkStore, StreamError, StreamMetadataStore, StreamRegistry, + CursorAdvance, DurableCursor, Result, StreamChunkStore, StreamError, StreamMirrorWrite, StreamRegistry, TrimmedChunk, }; @@ -38,6 +43,7 @@ impl Drop for ActiveRead<'_> { struct Chunk { bytes: Vec, cursor: u64, + synchronized_cursor: u64, capacity: u64, sealed: bool, released: bool, @@ -47,21 +53,26 @@ struct Chunk { #[derive(Debug, Default)] struct MemoryState { - bindings: HashMap, - manifests: HashMap, - pages: HashMap<(StreamName, u64, u64, u64), StreamExtentPage>, + stream_records: HashMap, + owners: HashMap)>, chunks: HashMap, cursor_faults: VecDeque, + metadata_records: BTreeMap<(u64, Vec), record_backend::MemoryRecord>, + metadata_revision: u64, + metadata_groups: HashMap, } /// Test-only implementation of all three injected storage contracts. #[derive(Debug)] pub struct MemoryStreamStore { - state: Mutex, + state: Arc>, chunk_capacity: u64, next_chunk: AtomicU64, - metadata_publishes: AtomicU64, - extent_page_loads: AtomicU64, + pause_allocation_number: AtomicU64, + fail_allocation_number: AtomicU64, + allocation_started: Notify, + allocation_resume: Notify, + record_faults: Arc, chunk_writes: AtomicU64, cursor_advances: AtomicU64, liveness_renews: AtomicU64, @@ -70,9 +81,11 @@ pub struct MemoryStreamStore { pause_reads: AtomicBool, active_reads: AtomicUsize, max_active_reads: AtomicUsize, - fail_next_publish: AtomicBool, fail_next_writes: AtomicUsize, + fail_next_release: AtomicBool, + fail_empty_release: AtomicBool, fail_next_renewals: AtomicUsize, + fail_chunk_renewal: AtomicU64, write_started: Notify, resume_write: Notify, read_started: Notify, @@ -81,23 +94,31 @@ pub struct MemoryStreamStore { impl MemoryStreamStore { /// Creates a test store with fixed logical-payload capacity chunks. The - /// backing capacity includes one public frame header and footer, so tiny + /// backing capacity includes canonical group framing, so tiny /// rollover tests retain their payload-oriented meaning. /// /// # Panics /// - /// Panics if `chunk_capacity` is zero. + /// Panics if `chunk_capacity` is zero or its physical geometry overflows. #[must_use] pub fn new(chunk_capacity: u64) -> Self { assert!(chunk_capacity > 0); Self { - state: Mutex::new(MemoryState::default()), - chunk_capacity: chunk_capacity.saturating_add( - u64::try_from(FRAME_HEADER_PREFIX_BYTES + FRAME_FOOTER_BYTES).unwrap_or(u64::MAX), - ), + state: Arc::new(Mutex::new(MemoryState::default())), + chunk_capacity: FrameLayout::StreamGroupV2 + .physical_length(chunk_capacity) + .and_then(|length| { + length + .checked_add(crowdb_protocol::frame::FRAME_HEADER_PREFIX_BYTES as u64) + .ok_or(crowdb_protocol::frame::FrameError::LengthOverflow) + }) + .expect("valid test chunk geometry"), next_chunk: AtomicU64::new(1), - metadata_publishes: AtomicU64::new(0), - extent_page_loads: AtomicU64::new(0), + pause_allocation_number: AtomicU64::new(0), + fail_allocation_number: AtomicU64::new(0), + allocation_started: Notify::new(), + allocation_resume: Notify::new(), + record_faults: Arc::new(record_backend::RecordFaults::default()), chunk_writes: AtomicU64::new(0), cursor_advances: AtomicU64::new(0), liveness_renews: AtomicU64::new(0), @@ -106,9 +127,11 @@ impl MemoryStreamStore { pause_reads: AtomicBool::new(false), active_reads: AtomicUsize::new(0), max_active_reads: AtomicUsize::new(0), - fail_next_publish: AtomicBool::new(false), fail_next_writes: AtomicUsize::new(0), + fail_next_release: AtomicBool::new(false), + fail_empty_release: AtomicBool::new(false), fail_next_renewals: AtomicUsize::new(0), + fail_chunk_renewal: AtomicU64::new(0), write_started: Notify::new(), resume_write: Notify::new(), read_started: Notify::new(), @@ -128,8 +151,66 @@ impl MemoryStreamStore { self.pause_writes.store(true, Ordering::Release); } + pub fn pause_allocation_for_tests(&self, number: u64) { + self.pause_allocation_number.store(number, Ordering::Release); + } + + pub fn fail_allocation_for_tests(&self, number: u64) { + self.fail_allocation_number.store(number, Ordering::Release); + } + + pub async fn wait_for_allocation_for_tests(&self) { + self.allocation_started.notified().await; + } + + pub fn resume_allocation_for_tests(&self) { + self.allocation_resume.notify_one(); + } + + pub async fn chunks_for_tests(&self) -> Vec<(ChunkId, u64, bool, bool)> { + let state = self.state.lock().await; + let mut chunks = state + .chunks + .iter() + .map(|(id, chunk)| (*id, chunk.synchronized_cursor, chunk.sealed, chunk.released)) + .collect::>(); + chunks.sort_by_key(|(id, ..)| id.low); + chunks + } + + pub fn pause_publication_for_tests(&self) { + self.record_faults.pause.store(true, Ordering::Release); + } + pub async fn wait_for_publication_for_tests(&self) { + self.record_faults.entered.notified().await; + } + pub fn resume_publication_for_tests(&self) { + self.record_faults.resume.notify_one(); + } + pub fn lose_publication_response_for_tests(&self) { + self.record_faults.lose_response.store(true, Ordering::Release); + } + pub fn fail_pin_renewals_for_tests(&self, count: u64) { + self.record_faults + .renewals_to_fail + .store(count, Ordering::Release); + } + pub fn pin_renewal_count_for_tests(&self) -> u64 { + self.record_faults.renewals.load(Ordering::Acquire) + } + pub fn fail_next_publish(&self) { - self.fail_next_publish.store(true, Ordering::Release); + self.record_faults.fail.store(true, Ordering::Release); + } + pub fn fail_record_delete_for_tests(&self) { + self.record_faults.fail_delete.store(true, Ordering::Release); + } + pub fn fail_release_for_tests(&self) { + self.fail_next_release.store(true, Ordering::Release); + } + + pub fn fail_empty_release_for_tests(&self) { + self.fail_empty_release.store(true, Ordering::Release); } pub fn fail_next_write(&self) { @@ -144,6 +225,10 @@ impl MemoryStreamStore { self.fail_next_renewals.store(count, Ordering::Release); } + pub fn fail_chunk_renewal_for_tests(&self, chunk: ChunkId) { + self.fail_chunk_renewal.store(chunk.low, Ordering::Release); + } + pub async fn wait_for_write(&self) { self.write_started.notified().await; } @@ -179,16 +264,72 @@ impl MemoryStreamStore { #[must_use] pub fn metadata_publish_count(&self) -> u64 { - self.metadata_publishes.load(Ordering::Acquire) + self.record_faults.publications.load(Ordering::Acquire) } #[must_use] - pub fn extent_page_load_count(&self) -> u64 { - self.extent_page_loads.load(Ordering::Acquire) + pub fn record_load_count(&self) -> u64 { + self.record_faults.loads.load(Ordering::Acquire) } - pub async fn extent_page_count(&self) -> usize { - self.state.lock().await.pages.len() + /// Returns retained physical bytes independently of the metadata checkpoint. + /// # Panics + /// Panics if the test chunk is absent or its cursor exceeds addressable memory. + pub async fn synchronized_bytes_for_tests(&self, chunk_id: ChunkId) -> Bytes { + let state = self.state.lock().await; + let chunk = state.chunks.get(&chunk_id).expect("allocated test chunk"); + Bytes::copy_from_slice(&chunk.bytes[..usize::try_from(chunk.synchronized_cursor).unwrap()]) + } + + /// Writes actual encoded groups through the same image contract as production. + /// # Errors + /// Returns invalid bounds, owner or write failures. + /// # Panics + /// Panics if the test chunk is absent or its cursor exceeds addressable memory. + pub async fn write_groups_for_tests( + &self, + name: StreamName, + epoch: u64, + chunk_id: ChunkId, + offset: u64, + data: Bytes, + ) -> Result<()> { + let start = usize::try_from(offset) + .map_err(|_| StreamError::InvalidRequest("test offset overflows".into()))?; + let mut prefix = self.synchronized_bytes_for_tests(chunk_id).await.to_vec(); + if prefix.len() != start { + return Err(StreamError::InvalidRequest("test write is not contiguous".into())); + } + prefix.extend_from_slice(&data); + let complete_end = u64::try_from(prefix.len()) + .map_err(|_| StreamError::InvalidRequest("test write overflows".into()))?; + let strip_bytes = usize::try_from(crowdb_chunk_client::STREAM_STRIP_BYTES).expect("bounded strip"); + let first = start / strip_bytes * strip_bytes; + let images: Vec<_> = (first..prefix.len()) + .step_by(strip_bytes) + .map(|strip| { + let end = (strip + strip_bytes).min(prefix.len()); + let offset = start.saturating_sub(strip); + let full_image = Bytes::copy_from_slice(&prefix[strip..end]); + crate::MirrorStripImage { + block_offset: offset as u64, + data: full_image.slice(offset..), + full_image, + } + }) + .collect(); + self.write_mirrors( + name, + epoch, + chunk_id, + StreamMirrorWrite { + authority: None, + physical_offset: offset, + complete_end, + images: &images, + }, + ) + .await } #[must_use] @@ -228,161 +369,60 @@ impl MemoryStreamStore { pub async fn flip_durable_byte(&self, chunk_id: ChunkId, offset: usize) { let mut state = self.state.lock().await; let chunk = state.chunks.get_mut(&chunk_id).expect("test chunk must exist"); - assert!(offset < usize::try_from(chunk.cursor).expect("test cursor must fit usize")); + assert!(offset < usize::try_from(chunk.synchronized_cursor).expect("test cursor must fit usize")); chunk.bytes[offset] ^= 1; } } #[async_trait] impl StreamRegistry for MemoryStreamStore { - async fn load(&self, stream_name: StreamName) -> Result> { - Ok(self.state.lock().await.bindings.get(&stream_name).cloned()) + async fn claim_writer(&self, name: StreamName, epoch: u64) -> Result> { + self.claim_memory_writer(name, epoch).await + } + async fn load(&self, stream_name: StreamName) -> Result> { + Ok(self.state.lock().await.stream_records.get(&stream_name).cloned()) } - async fn create(&self, binding: StreamBinding) -> Result<()> { + async fn create(&self, record: StreamRecord) -> Result<()> { let mut state = self.state.lock().await; - if state.bindings.insert(binding.stream_name, binding).is_some() { - return Err(StreamError::InvalidRequest( - "stream binding already exists".into(), - )); + if state.stream_records.contains_key(&record.stream_name) { + return Err(StreamError::InvalidRequest("stream record already exists".into())); } + state.stream_records.insert(record.stream_name, record); Ok(()) } } #[async_trait] -impl StreamMetadataStore for MemoryStreamStore { - async fn load_current(&self, stream_name: StreamName) -> Result> { - Ok(self.state.lock().await.manifests.get(&stream_name).cloned()) +impl StreamChunkStore for MemoryStreamStore { + fn bind_writer(&self, _name: StreamName, _epoch: u64, _session: Arc) {} + fn mirror_copies(&self) -> u32 { + 1 } - - async fn load_extent_page( - &self, - stream_name: StreamName, - writer_epoch: u64, - generation: u64, - page_index: u64, - ) -> Result> { - self.extent_page_loads.fetch_add(1, Ordering::AcqRel); - Ok(self - .state + async fn current_capacity(&self, chunk_id: ChunkId) -> Result { + self.state .lock() .await - .pages - .get(&(stream_name, writer_epoch, generation, page_index)) - .cloned()) - } - - async fn publish( - &self, - expected: Option<(u64, u64)>, - manifest: StreamManifest, - extent_pages: Vec, - ) -> Result<()> { - if self.fail_next_publish.swap(false, Ordering::AcqRel) { - return Err(StreamError::Internal( - "injected metadata publication failure".into(), - )); - } - let mut state = self.state.lock().await; - if state - .manifests - .get(&manifest.stream_name) - .is_some_and(|current| current.purpose != manifest.purpose) - { - return Err(StreamError::Corruption("stream purpose is immutable".into())); - } - let observed = state - .manifests - .get(&manifest.stream_name) - .map(|current| (current.writer_epoch, current.generation)); - if observed != expected { - return Err(StreamError::StaleWriter); - } - for page in extent_pages { - state.pages.insert( - ( - page.stream_name, - page.writer_epoch, - page.generation, - page.page_index, - ), - page, - ); - } - state.manifests.insert(manifest.stream_name, manifest); - self.metadata_publishes.fetch_add(1, Ordering::AcqRel); - Ok(()) - } - - async fn reclaim_extent_pages_before( - &self, - stream_name: StreamName, - retained_generation: u64, - max_pages: usize, - ) -> Result { - if retained_generation == 0 || max_pages == 0 { - return Err(StreamError::InvalidRequest( - "metadata reclaim watermark and page bound must be nonzero".into(), - )); - } - let mut state = self.state.lock().await; - let keys: Vec<_> = state - .pages - .keys() - .filter(|(name, _, generation, _)| *name == stream_name && *generation < retained_generation) - .take(max_pages) - .copied() - .collect(); - for key in &keys { - state.pages.remove(key); - } - Ok(keys.len() as u64) + .chunks + .get(&chunk_id) + .map(|chunk| chunk.capacity) + .ok_or_else(|| StreamError::ReadUnavailable("chunk is missing".into())) } -} - -#[async_trait] -impl StreamChunkStore for MemoryStreamStore { async fn allocate_mirrored( &self, _stream_name: StreamName, writer_epoch: u64, purpose: crowdb_protocol::chunk_stream::StreamPurpose, ) -> Result { - let low = self.next_chunk.fetch_add(1, Ordering::AcqRel); - let chunk_id = ChunkId { - high: (purpose.chunk_type() as u64) << 56, - low, - }; - self.state.lock().await.chunks.insert( - chunk_id, - Chunk { - bytes: Vec::new(), - cursor: 0, - capacity: self.chunk_capacity, - sealed: false, - released: false, - writer_epoch, - last_advance_checksum: None, - }, - ); - Ok(ActiveChunkDescriptor { - chunk_id, - physical_start: 0, - logical_start: 0, - acknowledged_cursor: 0, - capacity: self.chunk_capacity, - }) + self.allocate_active_chunk(writer_epoch, purpose).await } - async fn write_mirrors_with_images( + async fn write_mirrors( &self, _stream_name: StreamName, writer_epoch: u64, chunk_id: ChunkId, - physical_offset: u64, - data: Bytes, - _images: &[crate::MirrorStripImage], + write: StreamMirrorWrite<'_>, ) -> Result<()> { self.chunk_writes.fetch_add(1, Ordering::AcqRel); if self @@ -398,10 +438,16 @@ impl StreamChunkStore for MemoryStreamStore { self.resume_write.notified().await; } } + let physical_offset = write.physical_offset; let start = usize::try_from(physical_offset) .map_err(|_| StreamError::InvalidRequest("physical offset exceeds addressable range".into()))?; + let length = write + .images + .iter() + .try_fold(0_usize, |sum, image| sum.checked_add(image.data.len())) + .ok_or_else(|| StreamError::InvalidRequest("stream image length overflows".into()))?; let end = start - .checked_add(data.len()) + .checked_add(length) .ok_or_else(|| StreamError::InvalidRequest("physical write range overflows".into()))?; let mut state = self.state.lock().await; let chunk = state @@ -414,8 +460,33 @@ impl StreamChunkStore for MemoryStreamStore { if chunk.sealed || u64::try_from(end).unwrap_or(u64::MAX) > chunk.capacity { return Err(StreamError::InvalidRequest("write exceeds writable chunk".into())); } + let prefix: Vec<_> = write + .images + .iter() + .flat_map(|image| image.data.iter().copied()) + .take(15) + .collect(); + let grouped = crowdb_protocol::frame::parse_header(&prefix) + .is_ok_and(|header| header.magic == crowdb_protocol::frame::FrameMagic::StreamV2) + && prefix.get(14) == Some(&1); + if crate::storage::validate_write_bounds(write, chunk.capacity)? != end as u64 { + return Err(StreamError::InvalidRequest( + "image bytes differ from complete append end".into(), + )); + } + if !grouped || physical_offset != chunk.synchronized_cursor { + return Err(StreamError::InvalidRequest( + "stream writes require contiguous complete append groups".into(), + )); + } chunk.bytes.resize(chunk.bytes.len().max(end), 0); - chunk.bytes[start..end].copy_from_slice(&data); + let mut cursor = start; + for image in write.images { + let next = cursor + image.data.len(); + chunk.bytes[cursor..next].copy_from_slice(&image.data); + cursor = next; + } + chunk.synchronized_cursor = chunk.synchronized_cursor.max(write.complete_end); Ok(()) } @@ -455,6 +526,7 @@ impl StreamChunkStore for MemoryStreamStore { } if fault.commit { chunk.cursor = new_cursor; + chunk.synchronized_cursor = chunk.synchronized_cursor.max(new_cursor); chunk.last_advance_checksum = Some(checksum); } Ok(fault.outcome) @@ -477,8 +549,26 @@ impl StreamChunkStore for MemoryStreamStore { }) } + async fn recover_tail( + &self, + _name: StreamName, + epoch: u64, + active: &ActiveChunkDescriptor, + ) -> Result { + self.recover_chunk_tail(epoch, active).await + } + async fn renew_liveness(&self, chunk_id: ChunkId, writer_epoch: u64) -> Result<()> { self.liveness_attempts.fetch_add(1, Ordering::AcqRel); + if self + .fail_chunk_renewal + .compare_exchange(chunk_id.low, 0, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + return Err(StreamError::ReadUnavailable( + "injected candidate renewal failure".into(), + )); + } if self .fail_next_renewals .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_sub(1)) @@ -507,11 +597,12 @@ impl StreamChunkStore for MemoryStreamStore { if writer_epoch != chunk.writer_epoch { return Err(StreamError::StaleWriter); } - if chunk.cursor != cursor { + if chunk.synchronized_cursor != cursor { return Err(StreamError::Corruption( "seal cursor differs from durable cursor".into(), )); } + chunk.cursor = cursor; chunk.sealed = true; Ok(()) } @@ -535,7 +626,7 @@ impl StreamChunkStore for MemoryStreamStore { .chunks .get(&chunk_id) .ok_or_else(|| StreamError::ReadUnavailable("chunk is missing".into()))?; - if chunk.released || u64::try_from(end).unwrap_or(u64::MAX) > chunk.cursor { + if chunk.released || u64::try_from(end).unwrap_or(u64::MAX) > chunk.synchronized_cursor { return Err(StreamError::ReadUnavailable("chunk range is not durable".into())); } Ok(Bytes::copy_from_slice(&chunk.bytes[start..end])) @@ -543,12 +634,32 @@ impl StreamChunkStore for MemoryStreamStore { .await } - async fn release_trimmed(&self, chunk_id: ChunkId, _logical_end: u64) -> Result { + async fn release_trimmed( + &self, + chunk_id: ChunkId, + _logical_end: u64, + physical_end: u64, + ) -> Result { + if physical_end == 0 && self.fail_empty_release.swap(false, Ordering::AcqRel) { + return Err(StreamError::ReadUnavailable( + "injected empty chunk release failure".into(), + )); + } + if physical_end > 0 && self.fail_next_release.swap(false, Ordering::AcqRel) { + return Err(StreamError::ReadUnavailable( + "injected chunk release failure".into(), + )); + } let mut state = self.state.lock().await; let chunk = state .chunks .get_mut(&chunk_id) .ok_or_else(|| StreamError::ReadUnavailable("chunk is missing".into()))?; + if !chunk.released && (!chunk.sealed || chunk.cursor != physical_end) { + return Err(StreamError::Corruption( + "chunk release requires its exact sealed boundary".into(), + )); + } let reclaimed_bytes = if chunk.released { 0 } else { chunk.cursor }; chunk.released = true; Ok(TrimmedChunk { diff --git a/lib/crowdb-chunk-stream/src/memory/allocation.rs b/lib/crowdb-chunk-stream/src/memory/allocation.rs new file mode 100644 index 000000000..dc52cb387 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/memory/allocation.rs @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ActiveChunkDescriptor, Chunk, ChunkId, MemoryStreamStore, Ordering, Result}; + +impl MemoryStreamStore { + pub(super) async fn allocate_active_chunk( + &self, + writer_epoch: u64, + purpose: crowdb_protocol::chunk_stream::StreamPurpose, + ) -> Result { + let low = self.next_chunk.fetch_add(1, Ordering::AcqRel); + if self.fail_allocation_number.load(Ordering::Acquire) == low { + self.allocation_started.notify_one(); + return Err(crate::StreamError::ReadUnavailable( + "injected allocation outage".into(), + )); + } + if self.pause_allocation_number.load(Ordering::Acquire) == low { + self.allocation_started.notify_one(); + self.allocation_resume.notified().await; + } + let chunk_id = ChunkId { + high: (purpose.chunk_type() as u64) << 56, + low, + }; + self.state.lock().await.chunks.insert( + chunk_id, + Chunk { + bytes: Vec::new(), + cursor: 0, + synchronized_cursor: 0, + capacity: self.chunk_capacity, + sealed: false, + released: false, + writer_epoch, + last_advance_checksum: None, + }, + ); + Ok(ActiveChunkDescriptor { + chunk_id, + physical_start: 0, + logical_start: 0, + acknowledged_cursor: 0, + capacity: self.chunk_capacity, + }) + } +} diff --git a/lib/crowdb-chunk-stream/src/memory/authority.rs b/lib/crowdb-chunk-stream/src/memory/authority.rs new file mode 100644 index 000000000..aecc2fe1b --- /dev/null +++ b/lib/crowdb-chunk-stream/src/memory/authority.rs @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::MemoryStreamStore; +use crate::{Result, StreamError, StreamName, StreamOwnerAuthority, StreamWriterSession}; +use async_trait::async_trait; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +struct TestMemoryAuthority; + +#[async_trait] +impl StreamOwnerAuthority for TestMemoryAuthority { + fn deadline(&self) -> Result { + Ok(Instant::now() + Duration::from_secs(3600)) + } + async fn release(&self) -> Result<()> { + Ok(()) + } +} + +impl MemoryStreamStore { + pub(super) async fn claim_memory_writer( + &self, + name: StreamName, + epoch: u64, + ) -> Result> { + let mut state = self.state.lock().await; + if let Some((current_epoch, current)) = state.owners.get(&name) { + if *current_epoch > epoch { + return Err(StreamError::StaleWriter); + } + current.fence(); + } + let session = StreamWriterSession::new(Arc::new(TestMemoryAuthority)); + state.owners.insert(name, (epoch, session.clone())); + Ok(session) + } +} diff --git a/lib/crowdb-chunk-stream/src/memory/metadata.rs b/lib/crowdb-chunk-stream/src/memory/metadata.rs new file mode 100644 index 000000000..1da6d618e --- /dev/null +++ b/lib/crowdb-chunk-stream/src/memory/metadata.rs @@ -0,0 +1,70 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{record_backend::MemoryRecordBackend, MemoryStreamStore}; +use crate::{Result, StreamChunkRecordSnapshot, StreamChunkRecordStore, StreamMetadataStore, StreamName}; +use std::sync::Arc; +impl StreamMetadataStore for MemoryStreamStore { + fn record_store(&self, group: u64) -> Result { + StreamChunkRecordStore::new( + Arc::new(MemoryRecordBackend::new( + self.state.clone(), + group, + self.record_faults.clone(), + )), + group, + ) + } +} +impl MemoryStreamStore { + /// # Errors + /// Rejects corrupt stored record bytes. + pub async fn chunk_records_for_tests( + &self, + id: StreamName, + ) -> Result> { + use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}; + let state = self.state.lock().await; + state + .metadata_records + .iter() + .filter(|((_, key), _)| { + StreamChunkRecordKey::from_bytes(key).is_ok_and(|key| { + key.stream_id == id && matches!(key.order, StreamChunkRecordOrder::Attached(_)) + }) + }) + .map(|(_, record)| { + crowdb_protocol::chunk_stream::decode_stream_chunk_record(&record.value) + .map_err(|error| crate::StreamError::Corruption(error.to_string())) + }) + .collect() + } + + /// # Errors + /// Rejects missing/corrupt attached records and unavailable chunk capacity. + pub async fn active_checkpoint_for_tests(&self, id: StreamName) -> Result { + let snapshot = self + .load_tail_for_tests(id) + .await? + .ok_or(crate::StreamError::WriteStalled)?; + let record = snapshot.record(); + let capacity = crate::StreamChunkStore::current_capacity(self, record.chunk_id).await?; + let physical = snapshot.index().physical_end(); + Ok(crate::ActiveChunkDescriptor { + chunk_id: record.chunk_id, + physical_start: physical, + logical_start: record.logical_end.ok_or(crate::StreamError::WriteStalled)?, + acknowledged_cursor: physical, + capacity, + }) + } + /// # Errors + /// Returns the same typed errors as the production record state machine. + pub async fn load_tail_for_tests(&self, id: StreamName) -> Result> { + let group = self.state.lock().await.metadata_groups.get(&id).copied(); + match group { + Some(group) => self.record_store(group)?.load_tail(id).await, + None => Ok(None), + } + } +} diff --git a/lib/crowdb-chunk-stream/src/memory/record_backend.rs b/lib/crowdb-chunk-stream/src/memory/record_backend.rs new file mode 100644 index 000000000..121ed5df6 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/memory/record_backend.rs @@ -0,0 +1,226 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, +}; + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_kv_client::{BatchOp, Error, GetOutcome, Result, ScanOutcome, WriteOutcome}; +use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}; +use tokio::sync::{Mutex, Notify}; + +use super::MemoryState; +#[derive(Debug)] +pub(super) struct MemoryRecord { + pub(super) value: Bytes, + pub(super) revision: u64, +} +use crate::records::RecordBackend; + +#[derive(Debug, Default)] +pub(super) struct RecordFaults { + pub(super) fail: AtomicBool, + pub(super) lose_response: AtomicBool, + pub(super) pause: AtomicBool, + pub(super) entered: Notify, + pub(super) resume: Notify, + pub(super) publications: AtomicU64, + pub(super) loads: AtomicU64, + pub(super) renewals_to_fail: AtomicU64, + pub(super) renewals: AtomicU64, + pub(super) fail_delete: AtomicBool, +} + +pub(super) struct MemoryRecordBackend { + state: Arc>, + group_id: u64, + faults: Arc, +} +impl MemoryRecordBackend { + pub(super) fn new(state: Arc>, group_id: u64, faults: Arc) -> Self { + Self { + state, + group_id, + faults, + } + } +} + +#[async_trait] +impl RecordBackend for MemoryRecordBackend { + async fn get(&self, key: &[u8]) -> Result { + self.faults.loads.fetch_add(1, Ordering::Relaxed); + let state = self.state.lock().await; + Ok(state.metadata_records.get(&(self.group_id, key.to_vec())).map_or( + GetOutcome::NotFound, + |record| GetOutcome::Found { + value: record.value.clone(), + revision: record.revision, + }, + )) + } + + async fn scan(&self, prefix: &[u8], after: &[u8], limit: u32, reverse: bool) -> Result { + self.faults.loads.fetch_add(1, Ordering::Relaxed); + let state = self.state.lock().await; + let mut found: Vec<_> = state + .metadata_records + .iter() + .filter(|((group, key), _)| { + *group == self.group_id + && key.starts_with(prefix) + && (after.is_empty() + || if reverse { + key.as_slice() < after + } else { + key.as_slice() > after + }) + }) + .collect(); + if reverse { + found.reverse(); + } + let truncated = found.len() > limit as usize; + let items = found + .iter() + .take(limit as usize) + .map(|((_, key), record)| (Bytes::copy_from_slice(key), record.value.clone())) + .collect(); + let commit_slots = found + .iter() + .take(limit as usize) + .map(|(_, record)| record.revision) + .collect(); + Ok(ScanOutcome { + items, + commit_slots, + truncated, + timed_out: false, + read_slot: state.metadata_revision, + scan_cutoff: state.metadata_revision, + }) + } + async fn batch(&self, ops: &[BatchOp], key: &[u8], revision: u64) -> Result { + if !ops.iter().any(|op| match op { + BatchOp::Put { key: candidate, .. } | BatchOp::Delete { key: candidate } => { + candidate.as_ref() == key + } + }) { + return Err(Error::Server( + "conditional batch must mutate its guarded key".into(), + )); + } + let mut state = self.state.lock().await; + if ops.iter().any(|op| { + matches!(op, BatchOp::Delete {key} if StreamChunkRecordKey::from_bytes(key) + .is_ok_and(|key| matches!(key.order,StreamChunkRecordOrder::Attached(_)))) + }) && self.faults.fail_delete.swap(false, Ordering::AcqRel) + { + return Err(Error::Server("injected chunk record deletion failure".into())); + } + let renewing = ops.iter().any(|op| match op { + BatchOp::Put { key, .. } => { + key.starts_with(&[crowdb_protocol::key::CROWDB_KEY_MAGIC, 0, 0x1e]) + && state + .metadata_records + .contains_key(&(self.group_id, key.to_vec())) + } + BatchOp::Delete { .. } => false, + }); + if renewing + && self + .faults + .renewals_to_fail + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_sub(1)) + .is_ok() + { + return Err(Error::CasBusy); + } + let tail_name = ops.iter().find_map(|op| match op { + BatchOp::Put { key, .. } => StreamChunkRecordKey::from_bytes(key) + .ok() + .filter(|key| matches!(key.order, StreamChunkRecordOrder::Attached(_))) + .map(|key| key.stream_id), + BatchOp::Delete { .. } => None, + }); + let publication = is_publication(&state, self.group_id, ops, key); + if publication && self.faults.fail.swap(false, Ordering::AcqRel) { + return Err(Error::Server( + "injected Stream chunk record publication failure".into(), + )); + } + if publication && self.faults.pause.swap(false, Ordering::AcqRel) { + drop(state); + self.faults.entered.notify_one(); + self.faults.resume.notified().await; + state = self.state.lock().await; + } + let current = state + .metadata_records + .get(&(self.group_id, key.to_vec())) + .map_or(0, |record| record.revision); + if current != revision { + return Err(Error::CasFailed { + current_revision: current, + }); + } + let next = state + .metadata_revision + .checked_add(1) + .ok_or_else(|| Error::Server("test metadata revision overflows".into()))?; + state.metadata_revision = next; + if let Some(name) = tail_name { + state.metadata_groups.insert(name, self.group_id); + } + for op in ops { + match op { + BatchOp::Put { key, value } => { + state.metadata_records.insert( + (self.group_id, key.to_vec()), + MemoryRecord { + value: value.clone(), + revision: next, + }, + ); + } + BatchOp::Delete { key } => { + state.metadata_records.remove(&(self.group_id, key.to_vec())); + } + } + } + if renewing { + self.faults.renewals.fetch_add(1, Ordering::Relaxed); + } + if publication { + self.faults.publications.fetch_add(1, Ordering::AcqRel); + if self.faults.lose_response.swap(false, Ordering::AcqRel) { + return Err(Error::OutcomeUnknown); + } + } + Ok(WriteOutcome { + revision: next, + request_id: 0, + }) + } +} + +fn is_publication(state: &MemoryState, group: u64, ops: &[BatchOp], guarded: &[u8]) -> bool { + let generation = state + .metadata_records + .get(&(group, guarded.to_vec())) + .and_then(|record| crowdb_protocol::chunk_stream::decode_stream_chunk_record(&record.value).ok()) + .map_or(0, |record| record.generation); + ops.iter().any(|op| match op { + BatchOp::Put { key, value } + if StreamChunkRecordKey::from_bytes(key) + .is_ok_and(|key| matches!(key.order, StreamChunkRecordOrder::Attached(_))) => + { + crowdb_protocol::chunk_stream::decode_stream_chunk_record(value) + .is_ok_and(|record| record.generation > generation) + } + _ => false, + }) +} diff --git a/lib/crowdb-chunk-stream/src/memory/recovery.rs b/lib/crowdb-chunk-stream/src/memory/recovery.rs new file mode 100644 index 000000000..27b667cb1 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/memory/recovery.rs @@ -0,0 +1,100 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::MemoryStreamStore; +use crate::{ActiveChunkDescriptor, RecoveredStreamTail, Result, StreamError}; +use crowdb_protocol::frame::{ + FramePrefixScanner, FrameScanConfig, FrameScanMode, FrameScanStop, MAX_FRAME_SCAN_EXTENTS, + MAX_FRAME_SCAN_INPUT_BYTES, +}; + +impl MemoryStreamStore { + /// Stage an interrupted physical suffix without granting synchronization. + /// # Panics + /// Panics when the test chunk is absent or its cursor cannot fit in memory. + pub async fn stage_incomplete_tail_for_tests(&self, id: crate::ChunkId, bytes: bytes::Bytes) { + let mut state = self.state.lock().await; + let chunk = state.chunks.get_mut(&id).expect("test chunk exists"); + chunk + .bytes + .truncate(usize::try_from(chunk.synchronized_cursor).expect("test cursor fits memory")); + chunk.bytes.extend_from_slice(&bytes); + } + + /// # Panics + /// Panics when the test owner is absent. + pub async fn fence_writer_for_tests(&self, name: crate::StreamName) { + self.state + .lock() + .await + .owners + .get(&name) + .expect("test owner exists") + .1 + .fence(); + } + + pub(super) async fn recover_chunk_tail( + &self, + epoch: u64, + active: &ActiveChunkDescriptor, + ) -> Result { + let mut state = self.state.lock().await; + let chunk = state + .chunks + .get_mut(&active.chunk_id) + .ok_or_else(|| StreamError::ReadUnavailable("recovery chunk is missing".into()))?; + if chunk.writer_epoch > epoch { + return Err(StreamError::StaleWriter); + } + let limit = if chunk.sealed { + chunk.cursor + } else { + chunk.capacity + }; + let mut scanner = FramePrefixScanner::new(FrameScanConfig { + chunk_id: active.chunk_id, + mode: FrameScanMode::Grouped, + physical_start: active.acknowledged_cursor, + physical_limit: if chunk.sealed { + chunk.cursor + } else { + chunk.capacity + }, + required_end: active.acknowledged_cursor.max(chunk.cursor), + logical_start: active.logical_start, + max_append_bytes: chunk.capacity, + }) + .map_err(|error| frame_error(&error))?; + let mut offset = usize::try_from(active.acknowledged_cursor) + .map_err(|_| StreamError::Corruption("memory cursor exceeds address space".into()))?; + let limit = usize::try_from(limit) + .map_err(|_| StreamError::Corruption("memory capacity exceeds address space".into()))?; + let mut extents = Vec::new(); + while offset < limit && scanner.stop() == FrameScanStop::Running { + let end = offset.saturating_add(MAX_FRAME_SCAN_INPUT_BYTES).min(limit); + let mut window = vec![0; end - offset]; + let available = chunk.bytes.len().saturating_sub(offset).min(window.len()); + if available > 0 { + window[..available].copy_from_slice(&chunk.bytes[offset..offset + available]); + } + let batch = scanner + .push(offset as u64, &window, MAX_FRAME_SCAN_EXTENTS) + .map_err(|error| frame_error(&error))?; + offset += batch.consumed; + extents.extend(batch.extents); + } + let summary = scanner.finish().map_err(|error| frame_error(&error))?; + chunk.writer_epoch = epoch; + chunk.synchronized_cursor = summary.physical_end; + Ok(RecoveredStreamTail { + physical_end: summary.physical_end, + logical_end: summary.logical_end, + extents, + }) + } +} + +fn frame_error(error: &crowdb_protocol::frame::FrameScanError) -> StreamError { + StreamError::Corruption(error.to_string()) +} diff --git a/lib/crowdb-chunk-stream/src/metadata.rs b/lib/crowdb-chunk-stream/src/metadata.rs index 67e6972d4..f9b7710dd 100644 --- a/lib/crowdb-chunk-stream/src/metadata.rs +++ b/lib/crowdb-chunk-stream/src/metadata.rs @@ -1,9 +1,8 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_protocol::chunk_stream::{StreamExtentPage, StreamManifest}; use crowdb_protocol::common::ChunkId; -use crowdb_protocol::frame::FRAME_HEADER_PREFIX_BYTES; +use crowdb_protocol::frame::{FrameLayout, FRAME_FOOTER_BYTES, FRAME_GROUP_HEADER_BYTES, MAX_FRAME_BYTES}; use crate::{Result, StreamError}; @@ -14,161 +13,44 @@ pub struct ExtentLocation { pub frame_length: u32, pub payload_offset: u32, pub available: u64, + pub group_offset: u64, + pub group_length: u64, + pub ordinal: u32, } -/// Validates a complete manifest generation and returns its durable tail. -/// -/// # Errors -/// -/// Returns [`StreamError::Corruption`] for inconsistent identities, fences, -/// arrays, coverage, cursors, or arithmetic overflow. -pub fn validate_manifest(manifest: &StreamManifest, pages: &[StreamExtentPage]) -> Result { - if manifest - .active - .as_ref() - .is_some_and(|active| !manifest.purpose.matches(active.chunk_id)) - || pages - .iter() - .flat_map(|page| &page.chunk_ids) - .any(|id| !manifest.purpose.matches(*id)) - { +pub(crate) fn resolve_group( + chunk_id: ChunkId, + logical_start: u64, + logical_end: u64, + group_offset: u64, + logical_offset: u64, +) -> Result { + if !(logical_start..logical_end).contains(&logical_offset) { return Err(StreamError::Corruption( - "stream manifest references a different chunk purpose".into(), - )); - } - if manifest.trim_offset > manifest.sealed_tail { - return Err(StreamError::Corruption("trim offset exceeds sealed tail".into())); - } - if manifest.extent_pages.len() != pages.len() { - return Err(StreamError::Corruption( - "extent page directory is incomplete".into(), - )); - } - - let mut expected = manifest - .extent_pages - .first() - .map_or(manifest.sealed_tail, |fence| fence.first_logical); - if expected > manifest.trim_offset { - return Err(StreamError::Corruption( - "sealed extent coverage begins after the trim point".into(), - )); - } - for (fence, page) in manifest.extent_pages.iter().zip(pages) { - validate_extent_page(page)?; - let first = *page - .logical_offsets - .first() - .ok_or_else(|| StreamError::Corruption("extent page has no first offset".into()))?; - let end = *page - .logical_offsets - .last() - .ok_or_else(|| StreamError::Corruption("extent page has no end offset".into()))?; - if fence.page_index != page.page_index - || fence.first_logical != first - || fence.end_logical != end - || first != expected - || page.stream_name != manifest.stream_name - || page.writer_epoch != manifest.writer_epoch - || page.generation != manifest.generation - { - return Err(StreamError::Corruption( - "extent page fence or identity mismatch".into(), - )); - } - expected = end; - } - if expected != manifest.sealed_tail { - return Err(StreamError::Corruption( - "sealed extents do not cover sealed tail".into(), - )); - } - - let tail = if let Some(active) = &manifest.active { - if active.logical_start != manifest.sealed_tail - || active.acknowledged_cursor < active.physical_start - || active.acknowledged_cursor > active.capacity - { - return Err(StreamError::Corruption( - "active chunk descriptor is invalid".into(), - )); - } - active - .logical_start - .checked_add(active.acknowledged_cursor - active.physical_start) - .ok_or_else(|| StreamError::Corruption("active logical tail overflows".into()))? - } else { - manifest.sealed_tail - }; - if manifest.trim_offset > tail { - return Err(StreamError::Corruption("trim offset exceeds durable tail".into())); - } - Ok(tail) -} - -/// Resolves one logical offset within a validated extent page. -/// -/// # Errors -/// -/// Returns an error if the page is malformed, the offset is outside it, or -/// physical address arithmetic overflows. -pub fn resolve_extent(page: &StreamExtentPage, logical_offset: u64) -> Result { - validate_extent_page(page)?; - let index = page - .logical_offsets - .partition_point(|offset| *offset <= logical_offset); - if index == 0 || index >= page.logical_offsets.len() { - return Err(StreamError::InvalidRequest( - "logical offset is outside extent page".into(), - )); - } - let extent = index - 1; - let logical_start = page.logical_offsets[extent]; - let logical_end = page.logical_offsets[extent + 1]; - if logical_offset >= logical_end { - return Err(StreamError::InvalidRequest( - "logical offset is outside extent page".into(), + "logical offset is outside append group".into(), )); } let delta = logical_offset - logical_start; - let payload_offset = u32::try_from(FRAME_HEADER_PREFIX_BYTES) - .map_err(|_| StreamError::Corruption("frame header length exceeds u32".into()))? - .checked_add( - u32::try_from(delta) - .map_err(|_| StreamError::Corruption("physical extent payload offset overflows".into()))?, - ) - .ok_or_else(|| StreamError::Corruption("physical extent payload offset overflows".into()))?; + let capacity = FrameLayout::StreamGroupV2.payload_capacity(); + let ordinal = delta / capacity; + let frame_logical_start = ordinal * capacity; + let payload_length = (logical_end - logical_start - frame_logical_start).min(capacity); + let within = delta % capacity; + let frame_offset = group_offset + .checked_add(ordinal * MAX_FRAME_BYTES as u64) + .ok_or_else(|| StreamError::Corruption("stream frame offset overflows".into()))?; + let payload_offset = u32::try_from(FRAME_GROUP_HEADER_BYTES as u64 + within) + .map_err(|_| StreamError::Corruption("physical extent payload offset overflows".into()))?; Ok(ExtentLocation { - chunk_id: page.chunk_ids[extent], - frame_offset: page.physical_offsets[extent], - frame_length: page.frame_lengths[extent], + chunk_id, + frame_offset, + frame_length: u32::try_from(payload_length + (FRAME_GROUP_HEADER_BYTES + FRAME_FOOTER_BYTES) as u64) + .map_err(|_| StreamError::Corruption("stream frame length exceeds u32".into()))?, payload_offset, - available: logical_end - logical_offset, + available: payload_length - within, + group_offset, + group_length: logical_end - logical_start, + ordinal: u32::try_from(ordinal) + .map_err(|_| StreamError::Corruption("stream frame ordinal exceeds u32".into()))?, }) } - -fn validate_extent_page(page: &StreamExtentPage) -> Result<()> { - let count = page.chunk_ids.len(); - if count == 0 - || page.physical_offsets.len() != count - || page.frame_lengths.len() != count - || page.logical_offsets.len() != count + 1 - { - return Err(StreamError::Corruption( - "extent arrays have invalid lengths".into(), - )); - } - for offsets in page.logical_offsets.windows(2) { - if offsets[0] >= offsets[1] { - return Err(StreamError::Corruption( - "logical extents are empty, overlapping, or unordered".into(), - )); - } - } - for index in 0..count { - page.physical_offsets[index] - .checked_add(u64::from(page.frame_lengths[index])) - .ok_or_else(|| StreamError::Corruption("physical extent end overflows".into()))?; - } - Ok(()) -} diff --git a/lib/crowdb-chunk-stream/src/metrics.rs b/lib/crowdb-chunk-stream/src/metrics.rs index e87c43e68..5c2d3e3ee 100644 --- a/lib/crowdb-chunk-stream/src/metrics.rs +++ b/lib/crowdb-chunk-stream/src/metrics.rs @@ -20,8 +20,8 @@ pub struct StreamMetrics { pub(crate) max_queued_requests: AtomicU64, pub(crate) max_queued_bytes: AtomicU64, pub(crate) metadata_publications: AtomicU64, - pub(crate) extent_page_cache_hits: AtomicU64, - pub(crate) extent_page_cache_misses: AtomicU64, + pub(crate) record_cache_hits: AtomicU64, + pub(crate) record_cache_misses: AtomicU64, pub(crate) physical_read_requests: AtomicU64, } @@ -41,8 +41,8 @@ pub struct StreamMetricsSnapshot { pub max_queued_requests: u64, pub max_queued_bytes: u64, pub metadata_publications: u64, - pub extent_page_cache_hits: u64, - pub extent_page_cache_misses: u64, + pub record_cache_hits: u64, + pub record_cache_misses: u64, pub physical_read_requests: u64, } @@ -64,8 +64,8 @@ impl StreamMetrics { max_queued_requests: self.max_queued_requests.load(Ordering::Relaxed), max_queued_bytes: self.max_queued_bytes.load(Ordering::Relaxed), metadata_publications: self.metadata_publications.load(Ordering::Relaxed), - extent_page_cache_hits: self.extent_page_cache_hits.load(Ordering::Relaxed), - extent_page_cache_misses: self.extent_page_cache_misses.load(Ordering::Relaxed), + record_cache_hits: self.record_cache_hits.load(Ordering::Relaxed), + record_cache_misses: self.record_cache_misses.load(Ordering::Relaxed), physical_read_requests: self.physical_read_requests.load(Ordering::Relaxed), } } diff --git a/lib/crowdb-chunk-stream/src/mirror_shadow.rs b/lib/crowdb-chunk-stream/src/mirror_shadow.rs index f2fbce5c4..0381b1011 100644 --- a/lib/crowdb-chunk-stream/src/mirror_shadow.rs +++ b/lib/crowdb-chunk-stream/src/mirror_shadow.rs @@ -1,16 +1,31 @@ -use bytes::{Bytes, BytesMut}; -use crowdb_chunk_client::chunk::mirror_chunk_writer::STREAM_STRIP_BYTES; -use crowdb_protocol::common::ChunkId; +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded strip repair images assembled directly from immutable frame views. use crate::storage::MirrorStripImage; use crate::{Result, StreamError}; +use bytes::{Bytes, BytesMut}; +use crowdb_chunk_client::STREAM_STRIP_BYTES; +use crowdb_protocol::common::ChunkId; + +struct ShadowStrip { + start: u64, + bytes: BytesMut, +} + +struct PendingShadow { + chunk_id: ChunkId, + complete_end: u64, + strips: Vec<(u64, Bytes)>, +} #[derive(Default)] pub(crate) struct MirrorShadow { chunk_id: Option, - strip_start: u64, - retained: Option, - inflight: Option, + complete_end: u64, + retained: Vec, + pending: Option, } impl MirrorShadow { @@ -18,63 +33,121 @@ impl MirrorShadow { &mut self, chunk_id: ChunkId, physical_offset: u64, - data: &Bytes, + complete_end: u64, + views: &[Bytes], ) -> Result> { - if self.inflight.is_some() { + if self.pending.is_some() { return Err(StreamError::Internal( "mirror shadow has an unfinished write".into(), )); } - let strip_bytes = usize::try_from(STREAM_STRIP_BYTES) - .map_err(|_| StreamError::Internal("stream strip exceeds addressable memory".into()))?; - let mut copied = 0_usize; + let length = views + .iter() + .try_fold(0_u64, |sum, view| sum.checked_add(view.len() as u64)) + .ok_or_else(|| StreamError::InvalidRequest("stream write length overflows".into()))?; + let end = physical_offset + .checked_add(length) + .ok_or_else(|| StreamError::InvalidRequest("stream write offset overflows".into()))?; + if length == 0 + || complete_end != end + || complete_end <= physical_offset + || (self.chunk_id == Some(chunk_id) && self.complete_end != physical_offset) + { + return Err(StreamError::Corruption( + "mirror views differ from complete append bounds".into(), + )); + } + let mut view_index = 0; + let mut view_offset = 0; + let mut cursor = physical_offset; let mut images = Vec::new(); - while copied < data.len() { - let offset = physical_offset - .checked_add(copied as u64) - .ok_or_else(|| StreamError::InvalidRequest("stream write offset overflows".into()))?; - let strip_start = offset / STREAM_STRIP_BYTES * STREAM_STRIP_BYTES; - let block_offset = usize::try_from(offset - strip_start) - .map_err(|_| StreamError::Internal("mirror block offset overflows".into()))?; - let length = (strip_bytes - block_offset).min(data.len() - copied); - let view = data.slice(copied..copied + length); - let mut buffer = if self.chunk_id == Some(chunk_id) && self.strip_start == strip_start { - self.retained.take().ok_or_else(|| { - StreamError::Corruption("active mirror strip has no retained prefix".into()) - })? - } else { - if block_offset != 0 { - return Err(StreamError::Corruption( - "new mirror strip starts after its beginning".into(), - )); + while cursor < end { + let strip_start = cursor / STREAM_STRIP_BYTES * STREAM_STRIP_BYTES; + let block_offset = usize::try_from(cursor - strip_start).expect("bounded strip offset"); + let count = usize::try_from((end - cursor).min(STREAM_STRIP_BYTES - block_offset as u64)) + .expect("bounded strip image"); + let mut buffer = self.take_prefix(chunk_id, strip_start, block_offset)?; + let mut remaining = count; + while remaining > 0 { + let view = views + .get(view_index) + .ok_or_else(|| StreamError::Corruption("frame view chain ended early".into()))?; + let take = remaining.min(view.len() - view_offset); + buffer.extend_from_slice(&view[view_offset..view_offset + take]); + remaining -= take; + view_offset += take; + if view_offset == view.len() { + view_index += 1; + view_offset = 0; } - BytesMut::with_capacity(strip_bytes) - }; - if buffer.len() != block_offset { - return Err(StreamError::Corruption( - "mirror shadow does not match acknowledged cursor".into(), - )); } - buffer.extend_from_slice(&view); let full_image = buffer.freeze(); - self.chunk_id = Some(chunk_id); - self.strip_start = strip_start; - self.inflight = (block_offset + length < strip_bytes).then(|| full_image.clone()); images.push(MirrorStripImage { block_offset: block_offset as u64, - data: view, + data: full_image.slice(block_offset..), full_image, }); - copied += length; + cursor += count as u64; } + let strips = images + .iter() + .enumerate() + .filter_map(|(index, image)| { + let start = physical_offset / STREAM_STRIP_BYTES * STREAM_STRIP_BYTES + + index as u64 * STREAM_STRIP_BYTES; + (start + STREAM_STRIP_BYTES > complete_end).then(|| (start, image.full_image.clone())) + }) + .collect(); + self.pending = Some(PendingShadow { + chunk_id, + complete_end, + strips, + }); Ok(images) } + fn take_prefix(&mut self, chunk_id: ChunkId, start: u64, offset: usize) -> Result { + let buffer = if self.chunk_id == Some(chunk_id) { + self.retained + .iter() + .position(|strip| strip.start == start) + .map(|index| self.retained.remove(index).bytes) + } else { + None + }; + let buffer = match buffer { + Some(buffer) => buffer, + None if offset == 0 => { + BytesMut::with_capacity(usize::try_from(STREAM_STRIP_BYTES).expect("bounded strip")) + } + None => { + return Err(StreamError::Corruption( + "new mirror strip starts after its beginning".into(), + )) + } + }; + if buffer.len() != offset { + return Err(StreamError::Corruption( + "mirror shadow does not match synchronized cursor".into(), + )); + } + Ok(buffer) + } + pub(crate) fn finish(&mut self) { - self.retained = self.inflight.take().map(|image| { - image - .try_into_mut() - .unwrap_or_else(|shared| BytesMut::from(shared.as_ref())) - }); + if let Some(pending) = self.pending.take() { + self.chunk_id = Some(pending.chunk_id); + self.complete_end = pending.complete_end; + self.retained = pending + .strips + .into_iter() + .map(|(start, image)| ShadowStrip { + start, + bytes: image + .try_into_mut() + .unwrap_or_else(|shared| BytesMut::from(shared.as_ref())), + }) + .collect(); + } } } diff --git a/lib/crowdb-chunk-stream/src/perf.rs b/lib/crowdb-chunk-stream/src/perf.rs new file mode 100644 index 000000000..019ca6ac8 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/perf.rs @@ -0,0 +1,152 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Process-wide phase histograms, registered once and observed without locks. + +use std::sync::{Arc, OnceLock}; +use std::time::Instant; + +use crowdb_common::metrics::{HistogramSnapshot, LatencyHistogram, MetricsRegistry}; + +pub(crate) struct StreamPerfMetrics { + pub(crate) append: Arc, + pub(crate) queue: Arc, + pub(crate) capacity: Arc, + pub(crate) capacity_wait: Arc, + pub(crate) preparation: Arc, + pub(crate) successor_prepare: Arc, + pub(crate) successor_wait: Arc, + pub(crate) growth: Arc, + pub(crate) initialize: Arc, + pub(crate) seal: Arc, + pub(crate) seal_drain: Arc, + pub(crate) batch: Arc, + pub(crate) encode: Arc, + pub(crate) mirror: Arc, + pub(crate) cursor: Arc, + pub(crate) local_view: Arc, + pub(crate) checkpoint: Arc, + pub(crate) publish: Arc, + pub(crate) record_build: Arc, + pub(crate) record_get: Arc, + pub(crate) record_scan: Arc, + pub(crate) record_cas: Arc, + pub(crate) read: Arc, + pub(crate) read_plan: Arc, + pub(crate) read_io: Arc, +} + +fn histogram(name: &'static str) -> Arc { + Arc::new(LatencyHistogram::new(name.into())) +} + +pub(crate) fn perf() -> &'static StreamPerfMetrics { + static METRICS: OnceLock = OnceLock::new(); + METRICS.get_or_init(|| StreamPerfMetrics { + append: histogram("stream.append.e2e.lh"), + queue: histogram("stream.append.queue.lh"), + capacity: histogram("stream.append.capacity.lh"), + capacity_wait: histogram("stream.capacity.wait.lh"), + preparation: histogram("stream.capacity.prepare.lh"), + successor_prepare: histogram("stream.successor.prepare.lh"), + successor_wait: histogram("stream.successor.wait.lh"), + growth: histogram("stream.capacity.grow.lh"), + initialize: histogram("stream.capacity.initialize.lh"), + seal: histogram("stream.chunk.seal.lh"), + seal_drain: histogram("stream.chunk.seal_drain.lh"), + batch: histogram("stream.append.batch.lh"), + encode: histogram("stream.append.encode.lh"), + mirror: histogram("stream.append.mirror.lh"), + cursor: histogram("stream.checkpoint.cursor.lh"), + local_view: histogram("stream.append.local_view.lh"), + checkpoint: histogram("stream.checkpoint.lh"), + publish: histogram("stream.metadata.publish.lh"), + record_build: histogram("stream.metadata.record_build.lh"), + record_get: histogram("stream.metadata.record_get.lh"), + record_scan: histogram("stream.metadata.record_scan.lh"), + record_cas: histogram("stream.metadata.record_cas.lh"), + read: histogram("stream.read.e2e.lh"), + read_plan: histogram("stream.read.plan.lh"), + read_io: histogram("stream.read.io.lh"), + }) +} + +/// Cumulative process-wide latency distributions, including failed operations. +/// Values are nanoseconds; periodic perf flushes do not reset these snapshots. +/// Queue samples end when the worker takes each request. A later request can +/// wait behind another request's capacity preparation; the first request's own +/// preparation is recorded separately. Preparation overlaps ordinary appends. +/// Batch and metadata stages are nested, so their percentiles are not additive. +#[must_use] +pub fn stream_perf_metrics() -> Vec<(&'static str, HistogramSnapshot)> { + histograms() + .into_iter() + .map(|(name, histogram)| (name, histogram.snapshot_total())) + .collect() +} + +/// Include the process stream histograms in the service's periodic perf log. +/// Registration is a startup operation; observations only use atomic counters. +/// +/// # Panics +/// Panics if a stream metric name was registered with another histogram. +pub fn register_stream_perf_metrics(registry: &mut MetricsRegistry) { + for (_, histogram) in histograms() { + registry.register_shared_histogram(Arc::clone(histogram)); + } +} + +fn histograms() -> [(&'static str, &'static Arc); 25] { + let metrics = perf(); + [ + ("stream.append.e2e.lh", &metrics.append), + ("stream.append.queue.lh", &metrics.queue), + ("stream.append.capacity.lh", &metrics.capacity), + ("stream.capacity.wait.lh", &metrics.capacity_wait), + ("stream.capacity.prepare.lh", &metrics.preparation), + ("stream.successor.prepare.lh", &metrics.successor_prepare), + ("stream.successor.wait.lh", &metrics.successor_wait), + ("stream.capacity.grow.lh", &metrics.growth), + ("stream.capacity.initialize.lh", &metrics.initialize), + ("stream.chunk.seal.lh", &metrics.seal), + ("stream.chunk.seal_drain.lh", &metrics.seal_drain), + ("stream.append.batch.lh", &metrics.batch), + ("stream.append.encode.lh", &metrics.encode), + ("stream.append.mirror.lh", &metrics.mirror), + ("stream.checkpoint.cursor.lh", &metrics.cursor), + ("stream.append.local_view.lh", &metrics.local_view), + ("stream.checkpoint.lh", &metrics.checkpoint), + ("stream.metadata.publish.lh", &metrics.publish), + ("stream.metadata.record_build.lh", &metrics.record_build), + ("stream.metadata.record_get.lh", &metrics.record_get), + ("stream.metadata.record_scan.lh", &metrics.record_scan), + ("stream.metadata.record_cas.lh", &metrics.record_cas), + ("stream.read.e2e.lh", &metrics.read), + ("stream.read.plan.lh", &metrics.read_plan), + ("stream.read.io.lh", &metrics.read_io), + ] +} + +pub(crate) struct PhaseTimer<'a> { + histogram: &'a LatencyHistogram, + started: Instant, +} + +impl<'a> PhaseTimer<'a> { + pub(crate) fn start(histogram: &'a LatencyHistogram) -> Self { + Self { + histogram, + started: Instant::now(), + } + } +} + +impl Drop for PhaseTimer<'_> { + fn drop(&mut self) { + observe(self.histogram, self.started); + } +} + +pub(crate) fn observe(histogram: &LatencyHistogram, started: Instant) { + histogram.observe(started.elapsed().as_nanos().try_into().unwrap_or(u64::MAX)); +} diff --git a/lib/crowdb-chunk-stream/src/production.rs b/lib/crowdb-chunk-stream/src/production.rs index 9beccd529..51f552ffc 100644 --- a/lib/crowdb-chunk-stream/src/production.rs +++ b/lib/crowdb-chunk-stream/src/production.rs @@ -5,9 +5,10 @@ use std::sync::Arc; +use crossbeam_skiplist::SkipMap; use crowdb_chunk_client::{ChunkIoClient, ChunkReadPolicy}; use crowdb_kv_client::CrowdbKvClient; -use crowdb_protocol::chunk_stream::{StreamBinding, StreamName}; +use crowdb_protocol::chunk_stream::{StreamName, StreamRecord}; use crate::{ ChunkStream, KvStreamMetadataStore, KvStreamRegistry, ProductionStreamChunkStore, Result, @@ -20,6 +21,7 @@ pub struct ProductionStreamRuntime { registry: Arc, chunks: Arc, config: StreamConfig, + owners: SkipMap<(u64, StreamName), ChunkStream>, } impl ProductionStreamRuntime { @@ -70,6 +72,7 @@ impl ProductionStreamRuntime { kv, chunks, config, + owners: SkipMap::new(), }) } @@ -79,7 +82,7 @@ impl ProductionStreamRuntime { Arc::clone(&self.registry) } - /// Initializes metadata for a binding already activated by the control + /// Initializes metadata for a record already activated by the control /// plane. /// /// # Errors @@ -91,9 +94,9 @@ impl ProductionStreamRuntime { metadata_store_id: u64, writer_epoch: u64, ) -> Result { - let binding = self.active_binding(stream_name).await?; - let metadata = self.metadata(metadata_store_id, binding.metadata_group_id)?; - ChunkStream::create_registered( + let record = self.active_record(stream_name).await?; + let metadata = self.metadata(metadata_store_id, record.metadata_group_id)?; + let stream = ChunkStream::create_registered( stream_name, writer_epoch, self.config.clone(), @@ -101,10 +104,13 @@ impl ProductionStreamRuntime { metadata, self.chunks.clone(), ) - .await + .await?; + self.owners + .insert((metadata_store_id, stream_name), stream.clone()); + Ok(stream) } - /// Reopens the authoritative stream binding under a current writer epoch. + /// Reopens the authoritative stream record under a current writer epoch. /// /// # Errors /// @@ -115,9 +121,14 @@ impl ProductionStreamRuntime { metadata_store_id: u64, writer_epoch: u64, ) -> Result { - let binding = self.active_binding(stream_name).await?; - let metadata = self.metadata(metadata_store_id, binding.metadata_group_id)?; - ChunkStream::open( + if let Some(owner) = self.owners.get(&(metadata_store_id, stream_name)) { + if owner.value().writer_epoch() == writer_epoch && owner.value().writer_context().is_ok() { + return Ok(owner.value().clone()); + } + } + let record = self.active_record(stream_name).await?; + let metadata = self.metadata(metadata_store_id, record.metadata_group_id)?; + let stream = ChunkStream::open( stream_name, writer_epoch, self.config.clone(), @@ -125,7 +136,10 @@ impl ProductionStreamRuntime { metadata, self.chunks.clone(), ) - .await + .await?; + self.owners + .insert((metadata_store_id, stream_name), stream.clone()); + Ok(stream) } /// Opens an exact-epoch read-only stream view without changing writer @@ -140,8 +154,13 @@ impl ProductionStreamRuntime { metadata_store_id: u64, writer_epoch: u64, ) -> Result { - let binding = self.active_binding(stream_name).await?; - let metadata = self.metadata(metadata_store_id, binding.metadata_group_id)?; + let record = self.active_record(stream_name).await?; + if let Some(owner) = self.owners.get(&(metadata_store_id, stream_name)) { + if owner.value().writer_epoch() == writer_epoch && owner.value().writer_context().is_ok() { + return owner.value().capture_read_only().await; + } + } + let metadata = self.metadata(metadata_store_id, record.metadata_group_id)?; ChunkStream::open_read_only( stream_name, writer_epoch, @@ -157,39 +176,33 @@ impl ProductionStreamRuntime { /// Historical record authority remains the replay caller's responsibility. /// /// # Errors - /// Returns a binding, manifest, durable cursor or concurrent epoch error. + /// Returns a record, state, durable cursor or concurrent epoch error. pub async fn open_read_only_current( &self, stream_name: StreamName, metadata_store_id: u64, ) -> Result { - let binding = self.active_binding(stream_name).await?; - let metadata = self.metadata(metadata_store_id, binding.metadata_group_id)?; - let manifest = metadata - .load_current(stream_name) + let record = self.active_record(stream_name).await?; + let metadata = self.metadata(metadata_store_id, record.metadata_group_id)?; + let state = metadata + .record_store(record.metadata_group_id)? + .load_tail(stream_name) .await? - .ok_or_else(|| StreamError::InvalidRequest("read-only stream manifest does not exist".into()))?; - ChunkStream::open_read_only( - stream_name, - manifest.writer_epoch, - self.config.clone(), - self.registry.clone(), - metadata, - self.chunks.clone(), - ) - .await + .ok_or_else(|| StreamError::InvalidRequest("read-only stream state does not exist".into()))?; + self.open_read_only(stream_name, metadata_store_id, state.record().writer_epoch) + .await } - async fn active_binding(&self, stream_name: StreamName) -> Result { + async fn active_record(&self, stream_name: StreamName) -> Result { let registry: &dyn StreamRegistry = self.registry.as_ref(); - let binding = registry + let record = registry .load(stream_name) .await? - .ok_or_else(|| StreamError::InvalidRequest("stream binding does not exist".into()))?; - if binding.state != crowdb_protocol::chunk_stream::StreamBindingState::Active { - return Err(StreamError::InvalidRequest("stream binding is not active".into())); + .ok_or_else(|| StreamError::InvalidRequest("stream record does not exist".into()))?; + if record.state != crowdb_protocol::chunk_stream::StreamRecordState::Active { + return Err(StreamError::InvalidRequest("stream record is not active".into())); } - Ok(binding) + Ok(record) } fn metadata( diff --git a/lib/crowdb-chunk-stream/src/production_chunk.rs b/lib/crowdb-chunk-stream/src/production_chunk.rs index 8656dc1d6..4b7527735 100644 --- a/lib/crowdb-chunk-stream/src/production_chunk.rs +++ b/lib/crowdb-chunk-stream/src/production_chunk.rs @@ -12,25 +12,35 @@ use async_trait::async_trait; use bytes::Bytes; use crossbeam_skiplist::SkipMap; use crowdb_chunk_client::{ - ChunkAllocator, ChunkReadPolicy, ChunkReader, DiskWriter, FailedDiskList, MirrorChunkWriter, - MirrorStripFlow, ProtoLocation, + ChunkAllocator, ChunkReadPolicy, ChunkReader, DiskWriter, FailedDiskList, MirrorStripFlow, ProtoLocation, }; use crowdb_protocol::chunk_stream::{ActiveChunkDescriptor, StreamName}; use crowdb_protocol::chunkdb::rpc::{ - AdvanceChunkWriteRequest, AppendChunkRequest, Chunk, ChunkState, DeleteChunkRequest, QueryChunkRequest, - SealChunkRequest, Strip, StripType, + AdvanceChunkWriteRequest, Chunk, ChunkState, DeleteChunkRequest, QueryChunkRequest, SealChunkRequest, }; use crowdb_protocol::common::ChunkId; -use crowdb_protocol::frame::FrameMagic; use crate::{ - CursorAdvance, DurableCursor, MirrorStripImage, Result, StreamChunkStore, StreamError, TrimmedChunk, + CursorAdvance, DurableCursor, Result, StreamChunkStore, StreamError, StreamMirrorWrite, TrimmedChunk, }; +mod allocation; +mod authority; +mod cursor; +mod growth; +mod progress; +mod recovery; +mod sealing; +use progress::{admit_write, write_cursor}; + struct ChunkStateView { chunk: ArcSwap, modify_ts: AtomicU64, - cursor: AtomicU64, + persisted_cursor: AtomicU64, + synchronized_cursor: AtomicU64, + writer_started: AtomicBool, + fresh_grouped_allocation: AtomicBool, + initialized_capacity: AtomicU64, sealed: AtomicBool, last_checksum: AtomicU32, has_checksum: AtomicBool, @@ -48,6 +58,7 @@ pub struct ProductionStreamChunkStore { chunk_capacity_bytes: u64, failed_disks: Arc, chunks: SkipMap<(u64, u64), Arc>, + owners: SkipMap)>, } impl ProductionStreamChunkStore { @@ -118,6 +129,7 @@ impl ProductionStreamChunkStore { chunk_capacity_bytes, failed_disks: Arc::new(FailedDiskList::new(Duration::from_secs(60))), chunks: SkipMap::new(), + owners: SkipMap::new(), }) } @@ -147,7 +159,11 @@ impl ProductionStreamChunkStore { } let view = Arc::new(ChunkStateView { modify_ts: AtomicU64::new(chunk.modify_ts), - cursor: AtomicU64::new(chunk.acknowledged_cursor), + persisted_cursor: AtomicU64::new(chunk.acknowledged_cursor), + synchronized_cursor: AtomicU64::new(chunk.acknowledged_cursor), + writer_started: AtomicBool::new(false), + fresh_grouped_allocation: AtomicBool::new(false), + initialized_capacity: AtomicU64::new(0), sealed: AtomicBool::new(chunk.state == ChunkState::Sealed as i32), last_checksum: AtomicU32::new(0), has_checksum: AtomicBool::new(false), @@ -188,158 +204,86 @@ fn chunk_capacity(chunk: &Chunk) -> Result { } fn update_state(state: &ChunkStateView, chunk: Chunk) { - state.modify_ts.store(chunk.modify_ts, Ordering::Release); - state.cursor.store(chunk.acknowledged_cursor, Ordering::Release); + let incoming = Arc::new(chunk); + state.chunk.rcu(|current| { + if incoming.modify_ts >= current.modify_ts { + Arc::clone(&incoming) + } else { + Arc::clone(current) + } + }); + let chunk = state.chunk.load_full(); + state.modify_ts.fetch_max(chunk.modify_ts, Ordering::AcqRel); + state + .persisted_cursor + .fetch_max(chunk.acknowledged_cursor, Ordering::AcqRel); + state + .synchronized_cursor + .fetch_max(chunk.acknowledged_cursor, Ordering::AcqRel); state .sealed - .store(chunk.state == ChunkState::Sealed as i32, Ordering::Release); - state.chunk.store(Arc::new(chunk)); + .fetch_or(chunk.state == ChunkState::Sealed as i32, Ordering::AcqRel); } #[async_trait] impl StreamChunkStore for ProductionStreamChunkStore { + fn bind_writer(&self, name: StreamName, epoch: u64, session: Arc) { + self.owners.insert(name, (epoch, session)); + } + fn mirror_copies(&self) -> u32 { + self.mirror_copies + } + async fn current_capacity(&self, chunk_id: ChunkId) -> Result { + let state = self.state(chunk_id).await?; + chunk_capacity(&state.chunk.load()) + } async fn allocate_mirrored( &self, stream_name: StreamName, writer_epoch: u64, purpose: crowdb_protocol::chunk_stream::StreamPurpose, ) -> Result { - let writer = MirrorChunkWriter::allocate_with_copy_count( - Arc::clone(&self.allocator), - Arc::clone(&self.disk_writer), - stream_name, - writer_epoch, - self.writer_lease_ms, - self.mirror_copies, - purpose, - ) - .await - .map_err(io_error)?; - let chunk_id = writer.chunk_id(); - let capacity = writer.capacity(); - let cursor = writer.cursor(); - self.install(writer.chunk().clone())?; - Ok(ActiveChunkDescriptor { - chunk_id, - physical_start: 0, - logical_start: 0, - acknowledged_cursor: cursor, - capacity, - }) + self.allocate_active_chunk(stream_name, writer_epoch, purpose) + .await } async fn grow_mirrored( &self, - _stream_name: StreamName, + stream_name: StreamName, writer_epoch: u64, chunk_id: ChunkId, required_capacity: u64, ) -> Result> { - if required_capacity > self.chunk_capacity_bytes { - return Ok(None); - } - let state = self.state(chunk_id).await?; - for attempt in 0..2 { - let chunk = state.chunk.load_full(); - if chunk.writer_epoch != writer_epoch || state.sealed.load(Ordering::Acquire) { - return Err(StreamError::StaleWriter); - } - let capacity = chunk_capacity(&chunk)?; - if capacity >= required_capacity { - return Ok(Some(ActiveChunkDescriptor { - chunk_id, - physical_start: chunk.acknowledged_cursor, - logical_start: 0, - acknowledged_cursor: chunk.acknowledged_cursor, - capacity, - })); - } - let first = chunk - .strips - .first() - .ok_or_else(|| StreamError::Corruption("stream chunk has no mirror strip".into()))?; - let Some(Strip::MirrorStrip(mirror)) = &first.strip else { - return Err(StreamError::Corruption( - "stream chunk strip is not mirrored".into(), - )); - }; - let unit_count = mirror - .segments - .first() - .map(|segment| segment.unit_count) - .filter(|count| *count > 0) - .ok_or_else(|| { - StreamError::Corruption("stream mirror strip has no segment geometry".into()) - })?; - let strip_bytes = u64::from(first.capacity) * 1024; - let strip_count = u32::try_from((required_capacity - capacity).div_ceil(strip_bytes)) - .map_err(|_| StreamError::InvalidRequest("stream growth needs too many strips".into()))?; - let response = self - .allocator - .append_chunk(AppendChunkRequest { - chunk_id: Some(chunk_id), - modify_ts: chunk.modify_ts, - strip_size: unit_count, - strip_count, - strip_type: StripType::Mirror as i32, - data_num: 0, - code_num: 0, - copy_count: self.mirror_copies, - }) - .await - .map_err(io_error)?; - if let Some(current) = response.chunk { - update_state(&state, current); - if attempt == 0 { - continue; - } - return Err(StreamError::WriteStalled); - } - if response.strips.is_empty() { - return Err(StreamError::Corruption( - "stream chunk growth returned no strips".into(), - )); - } - let mut updated = (*chunk).clone(); - updated.strips.extend(response.strips); - updated.modify_ts = response.modify_ts; - updated.capacity = updated.strips.iter().map(|strip| strip.capacity).sum(); - let capacity = chunk_capacity(&updated)?; - update_state(&state, updated); - return Ok(Some(ActiveChunkDescriptor { - chunk_id, - physical_start: state.cursor.load(Ordering::Acquire), - logical_start: 0, - acknowledged_cursor: state.cursor.load(Ordering::Acquire), - capacity, - })); - } - unreachable!("two append attempts either return or fail") + self.grow_active_chunk(stream_name, writer_epoch, chunk_id, required_capacity) + .await } - async fn write_mirrors_with_images( + async fn write_mirrors( &self, - _stream_name: StreamName, + stream_name: StreamName, writer_epoch: u64, chunk_id: ChunkId, - physical_offset: u64, - data: Bytes, - images: &[MirrorStripImage], + write: StreamMirrorWrite<'_>, ) -> Result<()> { let state = self.state(chunk_id).await?; let mut chunk = (*state.chunk.load_full()).clone(); - if chunk.writer_epoch != writer_epoch - || state.sealed.load(Ordering::Acquire) - || state.cursor.load(Ordering::Acquire) != physical_offset - { + if StreamName::from_chunk_owner_key(&chunk.owner_key) != Some(stream_name) { return Err(StreamError::StaleWriter); } - let data_end = physical_offset - .checked_add(data.len() as u64) - .ok_or_else(|| StreamError::InvalidRequest("stream write cursor overflows".into()))?; + let data_end = admit_write(&state, &chunk, writer_epoch, &write, self.chunk_capacity_bytes)?; + let physical_offset = write.physical_offset; + let images = write.images; + let context = match write.authority { + Some(context) => { + context.check_io()?; + Some(context.clone()) + } + None => self.owner_context(stream_name, writer_epoch)?, + }; + let disk_writer = self.owned_disk_writer(context); let flow = MirrorStripFlow::with_shared_failures( Arc::clone(&self.allocator), - Arc::clone(&self.disk_writer), + disk_writer, Arc::clone(&self.failed_disks), writer_epoch, 3, @@ -364,33 +308,28 @@ impl StreamChunkStore for ProductionStreamChunkStore { .get(image_index) .ok_or_else(|| StreamError::InvalidRequest("stream mirror image is missing".into()))?; if image.block_offset != write_start - strip_start - || image.data != data.slice(copied..copied + count) + || image.data.len() != count + || image.full_image.len() + != usize::try_from(image.block_offset) + .unwrap_or(usize::MAX) + .saturating_add(count) + || image + .full_image + .get(usize::try_from(image.block_offset).unwrap_or(usize::MAX)..) + != Some(image.data.as_ref()) { return Err(StreamError::InvalidRequest( "stream mirror image does not match data".into(), )); } - let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { - return Err(StreamError::Corruption( - "stream chunk strip is not mirrored".into(), - )); - }; - let actual_copies = mirror.segments.len(); - let protected = if self.mirror_copies == 1 { - actual_copies == 1 - } else { - (2..=self.mirror_copies as usize).contains(&actual_copies) - }; - if !protected { - return Err(StreamError::Corruption( - "stream chunk mirror count differs from configuration".into(), - )); + progress::ensure_mirror_copies(&strip, self.mirror_copies)?; + if image_index == 0 { + progress::begin_write(&state)?; } let mut pending_advance = None; let result = flow .write( &mut chunk, - physical_offset, strip.strip_sequence, image.block_offset, image.data.clone(), @@ -405,88 +344,50 @@ impl StreamChunkStore for ProductionStreamChunkStore { crowdb_chunk_client::IoError::MetadataConflict(_) => StreamError::StaleWriter, other => io_error(other), })?; + let written = chunk + .strips + .iter() + .find(|current| current.strip_sequence == strip.strip_sequence) + .ok_or_else(|| StreamError::Corruption("written stream strip disappeared".into()))?; + progress::ensure_mirror_copies(written, self.mirror_copies)?; copied += count; image_index += 1; } - if copied != data.len() || image_index != images.len() { + if physical_offset.checked_add(copied as u64) != Some(data_end) || image_index != images.len() { return Err(StreamError::Corruption( "stream mirror images do not cover the write range".into(), )); } + state + .synchronized_cursor + .fetch_max(write.complete_end, Ordering::AcqRel); + state.writer_started.store(true, Ordering::Release); Ok(()) } async fn advance_cursor( &self, - _stream_name: StreamName, + stream_name: StreamName, writer_epoch: u64, chunk_id: ChunkId, expected_cursor: u64, new_cursor: u64, checksum: u32, ) -> Result { - let state = self.state(chunk_id).await?; - if state.chunk.load().writer_epoch != writer_epoch - || state.cursor.load(Ordering::Acquire) != expected_cursor - { - return Ok(CursorAdvance::DefinitelyNotCommitted); - } - let advance = self - .allocator - .advance_chunk_write(AdvanceChunkWriteRequest { + self.advance_active_cursor( + stream_name, + AdvanceChunkWriteRequest { chunk_id: Some(chunk_id), writer_epoch, - expected_modify_ts: state.modify_ts.load(Ordering::Acquire), + expected_modify_ts: 0, acknowledged_cursor: new_cursor, closed_strip_sequence: None, writer_lease_ms: self.writer_lease_ms, - }) - .await; - if let Err(error) = &advance { - tracing::warn!(?chunk_id, writer_epoch, expected_cursor, new_cursor, %error, "stream cursor advance failed; querying durable state"); - } - if let Ok(response) = advance { - let chunk = response - .chunk - .ok_or_else(|| StreamError::Corruption("cursor advance returned no chunk".into()))?; - if chunk.id != Some(chunk_id) - || chunk.writer_epoch != writer_epoch - || chunk.acknowledged_cursor != new_cursor - { - return Err(StreamError::Corruption( - "cursor advance returned inconsistent metadata".into(), - )); - } - update_state(&state, chunk); - state.last_checksum.store(checksum, Ordering::Release); - state.has_checksum.store(true, Ordering::Release); - return Ok(CursorAdvance::Committed); - } - let response = self - .allocator - .query_chunk(QueryChunkRequest { - chunk_id: Some(chunk_id), - }) - .await; - let Ok(response) = response else { - return Ok(CursorAdvance::Ambiguous); - }; - let Some(chunk) = response.chunk else { - return Ok(CursorAdvance::Ambiguous); - }; - if chunk.writer_epoch != writer_epoch { - return Err(StreamError::StaleWriter); - } - update_state(&state, chunk.clone()); - if chunk.acknowledged_cursor == new_cursor { - state.last_checksum.store(checksum, Ordering::Release); - state.has_checksum.store(true, Ordering::Release); - Ok(CursorAdvance::Committed) - } else if chunk.acknowledged_cursor == expected_cursor { - Ok(CursorAdvance::DefinitelyNotCommitted) - } else { - Ok(CursorAdvance::Ambiguous) - } + }, + expected_cursor, + checksum, + ) + .await } async fn durable_cursor(&self, chunk_id: ChunkId, writer_epoch: u64) -> Result { @@ -515,41 +416,89 @@ impl StreamChunkStore for ProductionStreamChunkStore { }) } + async fn recover_tail( + &self, + name: StreamName, + epoch: u64, + active: &ActiveChunkDescriptor, + ) -> Result { + self.recover_chunk_tail(name, epoch, active).await + } + async fn renew_liveness(&self, chunk_id: ChunkId, writer_epoch: u64) -> Result<()> { let state = self.state(chunk_id).await?; - let cursor = state.cursor.load(Ordering::Acquire); - let response = self - .allocator - .advance_chunk_write(AdvanceChunkWriteRequest { - chunk_id: Some(chunk_id), - writer_epoch, - expected_modify_ts: state.modify_ts.load(Ordering::Acquire), - acknowledged_cursor: cursor, - closed_strip_sequence: None, - writer_lease_ms: self.writer_lease_ms, - }) - .await - .map_err(io_error)?; - let chunk = response - .chunk - .ok_or_else(|| StreamError::Corruption("liveness renewal returned no chunk".into()))?; - if chunk.writer_epoch != writer_epoch || chunk.acknowledged_cursor != cursor { - return Err(StreamError::StaleWriter); + for _ in 0..3 { + let current = state.chunk.load_full(); + let name = + StreamName::from_chunk_owner_key(¤t.owner_key).ok_or(StreamError::StaleWriter)?; + self.owner_context(name, writer_epoch)?; + let result = self + .allocator + .advance_chunk_write(AdvanceChunkWriteRequest { + chunk_id: Some(chunk_id), + writer_epoch, + expected_modify_ts: current.modify_ts, + acknowledged_cursor: current.acknowledged_cursor, + closed_strip_sequence: None, + writer_lease_ms: self.writer_lease_ms, + }) + .await; + match result { + Ok(response) => { + let chunk = response.chunk.ok_or_else(|| { + StreamError::Corruption("liveness renewal returned no chunk".into()) + })?; + if chunk.writer_epoch != writer_epoch + || chunk.acknowledged_cursor < current.acknowledged_cursor + { + return Err(StreamError::StaleWriter); + } + update_state(&state, chunk); + return Ok(()); + } + Err(crowdb_chunk_client::IoError::MetadataConflict(_)) => { + let chunk = self + .allocator + .query_chunk(QueryChunkRequest { + chunk_id: Some(chunk_id), + }) + .await + .map_err(io_error)? + .chunk + .ok_or(StreamError::StaleWriter)?; + if chunk.writer_epoch != writer_epoch + || chunk.state != ChunkState::Active as i32 + || chunk.owner_key != current.owner_key + { + return Err(StreamError::StaleWriter); + } + update_state(&state, chunk); + } + Err(error) => return Err(io_error(error)), + } } - update_state(&state, chunk); - Ok(()) + Err(StreamError::ReadUnavailable( + "liveness layout kept changing".into(), + )) } async fn seal(&self, chunk_id: ChunkId, writer_epoch: u64, cursor: u64) -> Result<()> { + let _timer = crate::perf::PhaseTimer::start(&crate::perf::perf().seal); let state = self.state(chunk_id).await?; - if state.chunk.load().writer_epoch > writer_epoch || state.cursor.load(Ordering::Acquire) != cursor { + if state.chunk.load().writer_epoch > writer_epoch || write_cursor(&state) != cursor { return Err(StreamError::StaleWriter); } + let chunk = state.chunk.load_full(); + let name = StreamName::from_chunk_owner_key(&chunk.owner_key).ok_or(StreamError::StaleWriter)?; + if let Some(context) = self.owner_context(name, writer_epoch)? { + if chunk.state != ChunkState::Sealed as i32 { + self.drain_for_seal(&chunk, &context).await?; + } + } let response = self .allocator .seal_chunk(SealChunkRequest { chunk_id: Some(chunk_id), - seal_length: u32::try_from(cursor.div_ceil(1024)).unwrap_or(u32::MAX), seal_bytes: cursor, }) .await @@ -594,22 +543,64 @@ impl StreamChunkStore for ProductionStreamChunkStore { chunk_id: ChunkId, physical_offset: u64, length: usize, + synchronized_end: u64, ) -> Result { self.reader - .read_verified_frame( + .read_synchronized_stream_frame( chunk_id, physical_offset, u64::try_from(length) - .map_err(|_| StreamError::InvalidRequest("chunk frame length exceeds u64".into()))?, - FrameMagic::StreamV1, + .map_err(|_| StreamError::InvalidRequest("frame length exceeds u64".into()))?, + synchronized_end, ) .await .map_err(read_error) } - async fn release_trimmed(&self, chunk_id: ChunkId, _logical_end: u64) -> Result { - let state = self.state(chunk_id).await?; - let reclaimed_bytes = state.cursor.load(Ordering::Acquire); + async fn release_trimmed( + &self, + chunk_id: ChunkId, + _logical_end: u64, + physical_end: u64, + ) -> Result { + let response = match self + .allocator + .query_chunk(QueryChunkRequest { + chunk_id: Some(chunk_id), + }) + .await + { + Ok(response) => response, + Err(crowdb_chunk_client::IoError::ChunkNotFound(_)) => { + return Ok(TrimmedChunk { + chunk_id, + reclaimed_bytes: 0, + }) + } + Err(error) => return Err(io_error(error)), + }; + let Some(chunk) = response.chunk else { + return Ok(TrimmedChunk { + chunk_id, + reclaimed_bytes: 0, + }); + }; + if chunk.id != Some(chunk_id) { + return Err(StreamError::Corruption( + "chunk release query returned a different identity".into(), + )); + } + if chunk.state == ChunkState::Deleted as i32 { + return Ok(TrimmedChunk { + chunk_id, + reclaimed_bytes: 0, + }); + } + if chunk.state != ChunkState::Sealed as i32 || chunk.acknowledged_cursor != physical_end { + return Err(StreamError::Corruption( + "chunk release requires its exact sealed boundary".into(), + )); + } self.allocator .delete_chunk(DeleteChunkRequest { chunk_id: Some(chunk_id), @@ -618,7 +609,7 @@ impl StreamChunkStore for ProductionStreamChunkStore { .map_err(io_error)?; Ok(TrimmedChunk { chunk_id, - reclaimed_bytes, + reclaimed_bytes: physical_end, }) } } diff --git a/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs b/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs new file mode 100644 index 000000000..a096ea596 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/production_chunk/allocation.rs @@ -0,0 +1,170 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Fresh attributed Stream/Wal allocation before attachment publication. + +use crate::perf::{perf, PhaseTimer}; +use bytes::Bytes; +use crowdb_chunk_client::STREAM_STRIP_BYTES; +use crowdb_protocol::chunk_stream::StreamPurpose; +use crowdb_protocol::chunkdb::rpc::{AllocateChunkRequest, Chunk, ChunkState, ChunkStrip, Strip, StripType}; +use futures::{stream::FuturesUnordered, StreamExt}; + +use super::{ + io_error, progress, ActiveChunkDescriptor, Ordering, ProductionStreamChunkStore, Result, StreamError, + StreamName, +}; + +impl ProductionStreamChunkStore { + pub(super) async fn allocate_active_chunk( + &self, + stream_name: StreamName, + writer_epoch: u64, + purpose: StreamPurpose, + ) -> Result { + let context = self.owner_context(stream_name, writer_epoch)?; + if writer_epoch == 0 { + return Err(StreamError::InvalidRequest( + "stream allocation requires a writer epoch".into(), + )); + } + let response = self + .allocator + .allocate_chunk(AllocateChunkRequest { + chunk_id: None, + write_granularity: u32::try_from(STREAM_STRIP_BYTES / 1024).expect("bounded strip"), + strip_count: 1, + strip_type: StripType::Mirror as i32, + data_num: 0, + code_num: 0, + copy_count: self.mirror_copies, + chunk_type: purpose.chunk_type() as i32, + writer_epoch, + writer_lease_ms: self.writer_lease_ms, + owner_key: stream_name.chunk_owner_key(), + }) + .await + .map_err(io_error)?; + let chunk = response + .chunk + .ok_or_else(|| StreamError::Corruption("stream allocation returned no chunk".into()))?; + validate_allocation(&chunk, stream_name, writer_epoch, purpose)?; + progress::ensure_mirror_copies(&chunk.strips[0], self.mirror_copies)?; + let chunk_id = chunk.id.expect("validated allocation identity"); + if self.chunks.get(&(chunk_id.high, chunk_id.low)).is_some() { + return Err(StreamError::Corruption( + "allocator reused a known stream chunk identity".into(), + )); + } + if let Some(context) = &context { + context.check_io()?; + } + let state = self.install(chunk.clone())?; + self.initialize_strip(&chunk, &chunk.strips[0]).await?; + state + .initialized_capacity + .store(STREAM_STRIP_BYTES, Ordering::Release); + state.fresh_grouped_allocation.store(true, Ordering::Release); + Ok(ActiveChunkDescriptor { + chunk_id, + physical_start: 0, + logical_start: 0, + acknowledged_cursor: 0, + capacity: STREAM_STRIP_BYTES, + }) + } + + pub(super) async fn initialize_strips(&self, chunk: &Chunk, strips: &[ChunkStrip]) -> Result<()> { + // Await every submitted synchronization even if another strip failed. + let mut pending = FuturesUnordered::new(); + let mut strips = strips.iter(); + for strip in strips.by_ref().take(8) { + pending.push(self.initialize_strip(chunk, strip)); + } + let mut error = None; + while let Some(result) = pending.next().await { + if let Err(failure) = result { + error.get_or_insert(failure); + } + if let Some(strip) = strips.next() { + pending.push(self.initialize_strip(chunk, strip)); + } + } + error.map_or(Ok(()), Err) + } + + pub(super) async fn initialize_strip(&self, chunk: &Chunk, strip: &ChunkStrip) -> Result<()> { + let _timer = PhaseTimer::start(&perf().initialize); + progress::ensure_mirror_copies(strip, self.mirror_copies)?; + if u64::from(strip.capacity) * 1024 != STREAM_STRIP_BYTES { + return Err(StreamError::Corruption( + "stream strip has invalid initialization capacity".into(), + )); + } + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(StreamError::Corruption( + "fresh stream allocation is not mirrored".into(), + )); + }; + let unit_bytes = u64::from(strip.unit_kb) * 1024; + for (index, segment) in mirror.segments.iter().enumerate() { + if segment.disk_id.is_none() + || segment.owner_chunk != chunk.id + || segment.allocation_ts == 0 + || u64::from(segment.unit_count) * unit_bytes < STREAM_STRIP_BYTES + || segment + .unit_offset + .checked_add(u64::from(segment.unit_count)) + .and_then(|end| end.checked_mul(unit_bytes)) + .is_none() + || mirror.segments[..index].iter().any(|previous| { + previous.disk_id == segment.disk_id + && previous.zone_index == segment.zone_index + && previous.unit_offset < segment.unit_offset + u64::from(segment.unit_count) + && segment.unit_offset < previous.unit_offset + u64::from(previous.unit_count) + }) + { + return Err(StreamError::Corruption( + "fresh stream segment identity or geometry is invalid".into(), + )); + } + } + let strip_bytes = usize::try_from(STREAM_STRIP_BYTES).expect("bounded stream strip"); + let zeroes = Bytes::from(vec![0; strip_bytes]); + let name = StreamName::from_chunk_owner_key(&chunk.owner_key).ok_or(StreamError::StaleWriter)?; + let disk_writer = self.owned_disk_writer(self.owner_context(name, chunk.writer_epoch)?); + let writes = mirror.segments.iter().map(|segment| { + disk_writer.initialize_synchronized_at_byte_offset(segment, unit_bytes, 0, zeroes.clone()) + }); + let results = futures::future::join_all(writes).await; + for result in results { + result.map_err(io_error)?; + } + Ok(()) + } +} + +fn validate_allocation(chunk: &Chunk, name: StreamName, epoch: u64, purpose: StreamPurpose) -> Result<()> { + if !chunk.id.is_some_and(|id| purpose.matches(id)) + || chunk.chunk_type != purpose.chunk_type() as i32 + || chunk.writer_epoch != epoch + || StreamName::from_chunk_owner_key(&chunk.owner_key) != Some(name) + || chunk.state != ChunkState::Active as i32 + || chunk.modify_ts == 0 + || chunk.acknowledged_cursor != 0 + || chunk.strips.len() != 1 + || u64::from(chunk.capacity) * 1024 != STREAM_STRIP_BYTES + { + return Err(StreamError::Corruption( + "stream allocation has invalid identity, ownership, progress or capacity".into(), + )); + } + let strip = &chunk.strips[0]; + if strip.chunk_offset != 0 || u64::from(strip.capacity) * 1024 != STREAM_STRIP_BYTES || strip.unit_kb == 0 + { + return Err(StreamError::Corruption( + "stream allocation has invalid initial strip geometry".into(), + )); + } + Ok(()) +} diff --git a/lib/crowdb-chunk-stream/src/production_chunk/authority.rs b/lib/crowdb-chunk-stream/src/production_chunk/authority.rs new file mode 100644 index 000000000..3bc55661b --- /dev/null +++ b/lib/crowdb-chunk-stream/src/production_chunk/authority.rs @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bind physical submissions to the original local owner context. + +use super::ProductionStreamChunkStore; +use crate::{Result, StreamError, StreamName, StreamWriteContext}; +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_client::{DiskWriter, IoError}; +use crowdb_protocol::diskdb::rpc::Segment; +use std::sync::Arc; + +struct OwnedDiskWriter { + inner: Arc, + context: StreamWriteContext, +} + +impl ProductionStreamChunkStore { + pub(super) fn owner_context(&self, name: StreamName, epoch: u64) -> Result> { + self.owners + .get(&name) + .map(|entry| { + if entry.value().0 != epoch { + return Err(StreamError::StaleWriter); + } + entry.value().1.capture() + }) + .transpose() + } + + pub(super) fn owned_disk_writer(&self, context: Option) -> Arc { + match context { + Some(context) => Arc::new(OwnedDiskWriter { + inner: self.disk_writer.clone(), + context, + }), + None => self.disk_writer.clone(), + } + } +} + +#[async_trait] +impl DiskWriter for OwnedDiskWriter { + async fn write_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + self.write_synchronized_at_byte_offset(segment, unit_bytes, offset, data) + .await + } + async fn write( + &self, + _segment: &Segment, + _unit_bytes: u64, + _data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + Err(IoError::WriteFailed( + "owned stream IO requires synchronization".into(), + )) + } + + async fn write_synchronized_views_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Vec, + ) -> crowdb_chunk_client::Result<()> { + self.context.check_io().map_err(|error| authority_error(&error))?; + self.inner + .write_synchronized_views_until(segment, unit_bytes, offset, data, self.context.deadline()) + .await?; + self.context.check_ack().map_err(|error| authority_error(&error)) + } + + async fn write_synchronized_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + self.write_synchronized_views_at_byte_offset(segment, unit_bytes, offset, vec![data]) + .await + } + + async fn initialize_synchronized_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + self.context.check_io().map_err(|error| authority_error(&error))?; + self.inner + .initialize_synchronized_until(segment, unit_bytes, offset, data, self.context.deadline()) + .await?; + self.context.check_ack().map_err(|error| authority_error(&error)) + } +} + +fn authority_error(error: &StreamError) -> IoError { + IoError::MetadataConflict(error.to_string()) +} diff --git a/lib/crowdb-chunk-stream/src/production_chunk/cursor.rs b/lib/crowdb-chunk-stream/src/production_chunk/cursor.rs new file mode 100644 index 000000000..a40ec9b7f --- /dev/null +++ b/lib/crowdb-chunk-stream/src/production_chunk/cursor.rs @@ -0,0 +1,92 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Reconcile checkpoint publication across independent layout preparation. + +use crowdb_chunk_client::IoError; +use crowdb_protocol::chunkdb::rpc::{AdvanceChunkWriteRequest, Chunk, ChunkState, QueryChunkRequest}; + +use super::{update_state, ChunkStateView, Ordering, ProductionStreamChunkStore}; +use crate::{CursorAdvance, Result, StreamError, StreamName}; + +impl ProductionStreamChunkStore { + pub(super) async fn advance_active_cursor( + &self, + stream_name: StreamName, + mut request: AdvanceChunkWriteRequest, + expected_cursor: u64, + checksum: u32, + ) -> Result { + let chunk_id = request.chunk_id.expect("cursor request has a chunk"); + let state = self.state(chunk_id).await?; + validate_owner(&state.chunk.load(), stream_name, &request)?; + if state.persisted_cursor.load(Ordering::Acquire) != expected_cursor { + return Err(StreamError::StaleWriter); + } + for attempt in 0..2 { + request.expected_modify_ts = state.modify_ts.load(Ordering::Acquire); + let advance = self.allocator.advance_chunk_write(request.clone()).await; + if let Ok(response) = advance { + let chunk = response + .chunk + .ok_or_else(|| StreamError::Corruption("cursor advance returned no chunk".into()))?; + validate_owner(&chunk, stream_name, &request)?; + if chunk.acknowledged_cursor != request.acknowledged_cursor { + return Err(StreamError::Corruption( + "cursor advance returned inconsistent metadata".into(), + )); + } + return Ok(committed(&state, chunk, checksum)); + } + let definite_conflict = matches!(advance, Err(IoError::MetadataConflict(_))); + if let Err(error) = &advance { + if !definite_conflict { + tracing::warn!(?chunk_id, %error, "stream cursor advance failed; querying durable state"); + } + } + let response = self + .allocator + .query_chunk(QueryChunkRequest { + chunk_id: Some(chunk_id), + }) + .await; + let Ok(Some(chunk)) = response.map(|response| response.chunk) else { + return Ok(CursorAdvance::Ambiguous); + }; + validate_owner(&chunk, stream_name, &request)?; + update_state(&state, chunk.clone()); + if chunk.acknowledged_cursor == request.acknowledged_cursor { + return Ok(committed(&state, chunk, checksum)); + } + if attempt == 0 + && definite_conflict + && chunk.state == ChunkState::Active as i32 + && chunk.acknowledged_cursor == expected_cursor + && chunk.modify_ts > request.expected_modify_ts + { + // One preparation may have published a new layout revision. + // Only retry a definite rejection, never ambiguous data IO. + continue; + } + return Ok(CursorAdvance::Ambiguous); + } + unreachable!("one definite-conflict retry returns a cursor outcome") + } +} + +fn validate_owner(chunk: &Chunk, name: StreamName, request: &AdvanceChunkWriteRequest) -> Result<()> { + if chunk.id != request.chunk_id + || chunk.writer_epoch != request.writer_epoch + || StreamName::from_chunk_owner_key(&chunk.owner_key) != Some(name) + { + return Err(StreamError::StaleWriter); + } + Ok(()) +} + +fn committed(state: &ChunkStateView, chunk: Chunk, checksum: u32) -> CursorAdvance { + update_state(state, chunk); + state.last_checksum.store(checksum, Ordering::Release); + state.has_checksum.store(true, Ordering::Release); + CursorAdvance::Committed +} diff --git a/lib/crowdb-chunk-stream/src/production_chunk/growth.rs b/lib/crowdb-chunk-stream/src/production_chunk/growth.rs new file mode 100644 index 000000000..5250c405c --- /dev/null +++ b/lib/crowdb-chunk-stream/src/production_chunk/growth.rs @@ -0,0 +1,175 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Grow physical allocation only after initializing every newly attached strip. + +use super::{ + chunk_capacity, io_error, update_state, write_cursor, ActiveChunkDescriptor, ChunkId, ChunkStateView, + Ordering, ProductionStreamChunkStore, Result, StreamError, StreamName, +}; +use crate::perf::{perf, PhaseTimer}; +use crowdb_chunk_client::{IoError, STREAM_STRIP_BYTES}; +use crowdb_protocol::chunkdb::rpc::{ + AppendChunkRequest, AppendChunkResponse, Chunk, QueryChunkRequest, Strip, StripType, +}; + +const MIN_GROWTH_WINDOW_BYTES: u64 = 2 * STREAM_STRIP_BYTES; +const MAX_GROWTH_WINDOW_BYTES: u64 = 8 * STREAM_STRIP_BYTES; + +impl ProductionStreamChunkStore { + pub(super) async fn grow_active_chunk( + &self, + stream_name: StreamName, + writer_epoch: u64, + chunk_id: ChunkId, + required_capacity: u64, + ) -> Result> { + self.owner_context(stream_name, writer_epoch)?; + if required_capacity > self.chunk_capacity_bytes { + return Ok(None); + } + let state = self.state(chunk_id).await?; + for attempt in 0..2 { + let chunk = state.chunk.load_full(); + if chunk.writer_epoch != writer_epoch || state.sealed.load(Ordering::Acquire) { + return Err(StreamError::StaleWriter); + } + let capacity = chunk_capacity(&chunk)?; + if StreamName::from_chunk_owner_key(&chunk.owner_key) != Some(stream_name) + || state.initialized_capacity.load(Ordering::Acquire) != capacity + { + return Err(StreamError::InvalidRequest( + "stream growth requires initialized owned capacity".into(), + )); + } + if capacity >= required_capacity { + return Ok(Some(ActiveChunkDescriptor { + chunk_id, + physical_start: write_cursor(&state), + logical_start: 0, + acknowledged_cursor: write_cursor(&state), + capacity, + })); + } + // Small appends must not wait for a large initialization burst. + // Large appends amortize allocation over several following writes. + let window = required_capacity + .saturating_sub(write_cursor(&state)) + .saturating_mul(4) + .clamp(MIN_GROWTH_WINDOW_BYTES, MAX_GROWTH_WINDOW_BYTES) + .next_power_of_two(); + let target = required_capacity + .div_ceil(window) + .saturating_mul(window) + .min(self.chunk_capacity_bytes); + let growth_timer = PhaseTimer::start(&perf().growth); + let response = self.request_growth(&chunk, target, capacity).await?; + drop(growth_timer); + if let Some(current) = response.chunk { + update_state(&state, current); + if attempt == 0 { + continue; + } + return Err(StreamError::WriteStalled); + } + return self.finish_growth(&state, &chunk, response).await.map(Some); + } + unreachable!("two append attempts either return or fail") + } + + async fn finish_growth( + &self, + state: &ChunkStateView, + chunk: &Chunk, + response: AppendChunkResponse, + ) -> Result { + if response.strips.is_empty() { + return Err(StreamError::Corruption( + "stream chunk growth returned no strips".into(), + )); + } + let added = response.strips; + // Allocation may merge a concurrent cursor commit. Fetch the full + // record instead of stamping its new revision onto our old cursor, + // which could hide a delayed cursor response. + let updated = self + .allocator + .query_chunk(QueryChunkRequest { chunk_id: chunk.id }) + .await + .map_err(preparation_error)? + .chunk + .ok_or_else(|| StreamError::Corruption("grown chunk metadata is missing".into()))?; + if updated.id != chunk.id + || updated.writer_epoch != chunk.writer_epoch + || updated.owner_key != chunk.owner_key + { + return Err(StreamError::StaleWriter); + } + if updated.modify_ts < response.modify_ts + || updated.strips.get(chunk.strips.len()..) != Some(added.as_slice()) + { + return Err(StreamError::Corruption( + "grown layout changed before initialization".into(), + )); + } + let capacity = chunk_capacity(&updated)?; + update_state(state, updated.clone()); + self.initialize_strips(&updated, &added).await.map_err(|error| { + StreamError::InvalidRequest(format!("stream growth initialization is uncertain: {error}")) + })?; + state.initialized_capacity.store(capacity, Ordering::Release); + Ok(ActiveChunkDescriptor { + chunk_id: chunk.id.expect("owned chunk has an identity"), + physical_start: write_cursor(state), + logical_start: 0, + acknowledged_cursor: write_cursor(state), + capacity, + }) + } + + async fn request_growth( + &self, + chunk: &Chunk, + required_capacity: u64, + capacity: u64, + ) -> Result { + let first = chunk + .strips + .first() + .ok_or_else(|| StreamError::Corruption("stream chunk has no mirror strip".into()))?; + let Some(Strip::MirrorStrip(mirror)) = &first.strip else { + return Err(StreamError::Corruption( + "stream chunk strip is not mirrored".into(), + )); + }; + let unit_count = mirror + .segments + .first() + .map(|segment| segment.unit_count) + .filter(|count| *count > 0) + .ok_or_else(|| StreamError::Corruption("stream mirror strip has no segment geometry".into()))?; + let strip_bytes = u64::from(first.capacity) * 1024; + let strip_count = u32::try_from((required_capacity - capacity).div_ceil(strip_bytes)) + .map_err(|_| StreamError::InvalidRequest("stream growth needs too many strips".into()))?; + self.allocator + .append_chunk(AppendChunkRequest { + chunk_id: chunk.id, + modify_ts: chunk.modify_ts, + strip_size: unit_count, + strip_count, + strip_type: StripType::Mirror as i32, + data_num: 0, + code_num: 0, + copy_count: self.mirror_copies, + }) + .await + .map_err(preparation_error) + } +} + +fn preparation_error(error: IoError) -> StreamError { + match error { + IoError::AllocationFailed(message) => StreamError::ReadUnavailable(message), + other => io_error(other), + } +} diff --git a/lib/crowdb-chunk-stream/src/production_chunk/progress.rs b/lib/crowdb-chunk-stream/src/production_chunk/progress.rs new file mode 100644 index 000000000..0597f4d79 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/production_chunk/progress.rs @@ -0,0 +1,94 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Admission and physical progress are separate from persisted layout checkpoints. + +use crowdb_protocol::chunkdb::rpc::{Chunk, ChunkStrip, Strip}; +use crowdb_protocol::frame::{parse_header, FrameMagic}; +use std::sync::atomic::Ordering; + +use super::ChunkStateView; +use crate::{Result, StreamError, StreamMirrorWrite}; + +pub(super) fn ensure_mirror_copies(strip: &ChunkStrip, required: u32) -> Result<()> { + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(StreamError::Corruption( + "stream chunk strip is not mirrored".into(), + )); + }; + if mirror.segments.len() != required as usize { + return Err(StreamError::ReadUnavailable(format!( + "stream strip requires {required} synchronized replicas; layout has {}", + mirror.segments.len() + ))); + } + Ok(()) +} + +pub(super) fn write_cursor(state: &ChunkStateView) -> u64 { + state.synchronized_cursor.load(Ordering::Acquire) +} + +pub(super) fn admit_write( + state: &ChunkStateView, + chunk: &Chunk, + writer_epoch: u64, + write: &StreamMirrorWrite<'_>, + physical_limit: u64, +) -> Result { + if chunk.writer_epoch != writer_epoch + || state.sealed.load(Ordering::Acquire) + || write_cursor(state) != write.physical_offset + { + return Err(StreamError::StaleWriter); + } + let data_end = crate::storage::validate_write_bounds(*write, physical_limit)?; + if crate::storage::padded_end(data_end, physical_limit)? + > state.initialized_capacity.load(Ordering::Acquire) + { + return Err(StreamError::InvalidRequest( + "stream write requires more allocated capacity".into(), + )); + } + let mut prefix = [0; 15]; + let mut copied = 0; + for image in write.images { + let count = (prefix.len() - copied).min(image.data.len()); + prefix[copied..copied + count].copy_from_slice(&image.data[..count]); + copied += count; + if copied == prefix.len() { + break; + } + } + let header = + parse_header(&prefix[..copied]).map_err(|error| StreamError::InvalidRequest(error.to_string()))?; + if header.magic != FrameMagic::StreamV2 || copied < 15 || prefix[14] != 1 { + return Err(StreamError::InvalidRequest( + "stream writes require complete append groups".into(), + )); + } + if !state.writer_started.load(Ordering::Acquire) + && (write.physical_offset != 0 + || state.synchronized_cursor.load(Ordering::Acquire) != 0 + || !state.fresh_grouped_allocation.load(Ordering::Acquire)) + { + return Err(StreamError::InvalidRequest( + "first grouped write requires this owner's fresh allocation".into(), + )); + } + Ok(data_end) +} + +pub(super) fn begin_write(state: &ChunkStateView) -> Result<()> { + if !state.writer_started.load(Ordering::Acquire) + && state + .fresh_grouped_allocation + .compare_exchange(true, false, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err(StreamError::InvalidRequest( + "first stream write has already started or is not owned".into(), + )); + } + Ok(()) +} diff --git a/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs b/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs new file mode 100644 index 000000000..49d60f15d --- /dev/null +++ b/lib/crowdb-chunk-stream/src/production_chunk/recovery.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Recover the complete mirrored prefix beyond its persisted checkpoint. + +use super::{io_error, update_state, Ordering, ProductionStreamChunkStore}; +use crate::{ActiveChunkDescriptor, RecoveredStreamTail, Result, StreamError, StreamName}; +use async_trait::async_trait; +use crowdb_chunk_client::{replay_frame_prefix, select_frame_prefix, FrameExtentSink, MirroredFrameReaders}; +use crowdb_protocol::chunkdb::rpc::{ChunkState, QueryChunkRequest}; +use crowdb_protocol::frame::{ScannedFrameExtent, MAX_FRAME_SCAN_INPUT_BYTES}; +use std::{sync::Arc, time::Duration}; + +struct RecoveredExtents(Vec); + +#[async_trait] +impl FrameExtentSink for RecoveredExtents { + async fn append_extents(&mut self, extents: &[ScannedFrameExtent]) -> crowdb_chunk_client::Result<()> { + self.0.extend_from_slice(extents); + Ok(()) + } +} + +impl ProductionStreamChunkStore { + pub(super) async fn recover_chunk_tail( + &self, + name: StreamName, + epoch: u64, + active: &ActiveChunkDescriptor, + ) -> Result { + let context = self.owner_context(name, epoch)?.ok_or(StreamError::StaleWriter)?; + context.check_io()?; + let chunk = self + .allocator + .query_chunk(QueryChunkRequest { + chunk_id: Some(active.chunk_id), + }) + .await + .map_err(io_error)? + .chunk + .ok_or_else(|| StreamError::ReadUnavailable("recovery chunk is missing".into()))?; + if chunk.id != Some(active.chunk_id) + || chunk.writer_epoch > epoch + || chunk.owner_key != name.chunk_owner_key() + { + return Err(StreamError::StaleWriter); + } + let state = self.install(chunk.clone())?; + let deadline = tokio::time::Instant::from_std(context.deadline()) + .min(tokio::time::Instant::now() + Duration::from_secs(60)); + for strip in &chunk.strips { + let Some(crowdb_protocol::chunkdb::rpc::Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(StreamError::Corruption("recovery chunk is not mirrored".into())); + }; + for segment in &mirror.segments { + context.check_io()?; + self.disk_writer + .drain_segment(segment, u64::from(strip.unit_kb) * 1024, deadline.into_std()) + .await + .map_err(io_error)?; + } + } + let readers = + MirroredFrameReaders::new(Arc::new(chunk.clone()), name, self.disk_writer.clone(), deadline) + .map_err(|error| observation_error(&error))?; + let mut config = readers + .scan_config( + active.acknowledged_cursor, + active.logical_start, + chunk.acknowledged_cursor.max(active.acknowledged_cursor), + self.chunk_capacity_bytes, + ) + .map_err(|error| observation_error(&error))?; + if chunk.state == ChunkState::Sealed as i32 { + config.physical_limit = chunk.acknowledged_cursor; + } + let selection = select_frame_prefix( + readers.readers(), + config, + u32::try_from(MAX_FRAME_SCAN_INPUT_BYTES).expect("scan window fits u32"), + deadline, + ) + .await + .map_err(|error| observation_error(&error))?; + let mut extents = RecoveredExtents(Vec::new()); + let observed = replay_frame_prefix( + readers.readers()[selection.selected_index()].as_ref(), + selection.observation(), + deadline, + &mut extents, + ) + .await + .map_err(|error| observation_error(&error))?; + let refreshed = self + .allocator + .query_chunk(QueryChunkRequest { + chunk_id: Some(active.chunk_id), + }) + .await + .map_err(io_error)? + .chunk; + if refreshed.as_ref() != Some(&chunk) { + return Err(StreamError::StaleWriter); + } + context.check_io()?; + update_state(&state, chunk); + state + .synchronized_cursor + .fetch_max(observed.summary().physical_end, Ordering::AcqRel); + Ok(RecoveredStreamTail { + physical_end: observed.summary().physical_end, + logical_end: observed.summary().logical_end, + extents: extents.0, + }) + } +} + +fn observation_error(error: &crowdb_chunk_client::FrameObservationError) -> StreamError { + use crowdb_chunk_client::FrameObservationError as E; + match error { + E::InvalidLayout(_) + | E::InvalidRequest + | E::InvalidPrefix(_) + | E::ConflictingPrefixes { .. } + | E::ChangedPrefix => StreamError::Corruption(error.to_string()), + _ => StreamError::ReadUnavailable(error.to_string()), + } +} diff --git a/lib/crowdb-chunk-stream/src/production_chunk/sealing.rs b/lib/crowdb-chunk-stream/src/production_chunk/sealing.rs new file mode 100644 index 000000000..8e372f552 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/production_chunk/sealing.rs @@ -0,0 +1,45 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{io_error, ProductionStreamChunkStore}; +use crate::perf::{perf, PhaseTimer}; +use crate::{Result, StreamError, StreamWriteContext}; +use crowdb_protocol::chunkdb::rpc::{Chunk, Strip}; +use futures::{stream, StreamExt}; + +impl ProductionStreamChunkStore { + pub(super) async fn drain_for_seal(&self, chunk: &Chunk, context: &StreamWriteContext) -> Result<()> { + let _timer = PhaseTimer::start(&perf().seal_drain); + let mut segments = Vec::new(); + for strip in &chunk.strips { + let Some(Strip::MirrorStrip(mirror)) = &strip.strip else { + return Err(StreamError::Corruption( + "stream seal has non-mirror layout".into(), + )); + }; + segments.extend( + mirror + .segments + .iter() + .map(|segment| (*segment, u64::from(strip.unit_kb) * 1024)), + ); + } + // Each segment has its own ordered IO stream. Wait for every submitted + // drain, including after a peer fails, before returning to its lifecycle. + let results = stream::iter(segments) + .map(|(segment, unit_bytes)| async move { + context.check_io()?; + self.disk_writer + .drain_segment(&segment, unit_bytes, context.deadline()) + .await + .map_err(io_error) + }) + .buffer_unordered(8) + .collect::>() + .await; + for result in results { + result?; + } + context.check_io() + } +} diff --git a/lib/crowdb-chunk-stream/src/records.rs b/lib/crowdb-chunk-stream/src/records.rs new file mode 100644 index 000000000..9784eaeda --- /dev/null +++ b/lib/crowdb-chunk-stream/src/records.rs @@ -0,0 +1,193 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Ordered Stream chunk records over conditional KV operations. + +use crate::{Result, StreamError}; +use crowdb_kv_client::GetOutcome; +use crowdb_protocol::chunk_stream::{ + decode_stream_chunk_record, StreamChunkRecord, StreamChunkState, StreamMappingIndex, StreamName, +}; +use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}; +use std::sync::Arc; + +mod backend; +pub(crate) use backend::RecordBackend; +mod cache; +mod candidate; +mod collection; +mod pin; +mod publication; +pub use collection::StreamCollectionResult; +pub use pin::StreamReadPin; + +#[derive(Clone)] +pub struct StreamChunkRecordStore { + backend: Arc, + group_id: u64, + cache: Arc, +} + +#[derive(Clone, Debug)] +pub struct StreamChunkRecordSnapshot { + pub(crate) record: Arc, + pub(crate) index: Arc, + key: Vec, + revision: u64, +} + +impl StreamChunkRecordSnapshot { + #[must_use] + pub fn record(&self) -> &StreamChunkRecord { + &self.record + } + #[must_use] + pub const fn index(&self) -> &Arc { + &self.index + } +} + +impl StreamChunkRecordStore { + pub(crate) fn new(backend: Arc, group_id: u64) -> Result { + if group_id == 0 { + return Err(StreamError::InvalidRequest( + "Stream chunk records require a nonzero metadata group".into(), + )); + } + Ok(Self { + backend, + group_id, + cache: Arc::new(cache::RecordCache::default()), + }) + } + + #[must_use] + pub const fn group_id(&self) -> u64 { + self.group_id + } + + /// # Errors + /// Rejects IO, malformed records and incorrect key/value identities. + pub async fn load_tail(&self, id: StreamName) -> Result> { + let prefix = StreamChunkRecordKey::attached_prefix(id); + let records = self + .backend + .scan(&prefix, &[], 1, true) + .await + .map_err(storage_error)?; + if records.items.len() > 1 || records.items.len() != records.commit_slots.len() { + return Err(corrupt("tail scan shape")); + } + match records.items.into_iter().zip(records.commit_slots).next() { + Some(((key, value), revision)) => Self::decode(&key, &value, revision, id).map(Some), + None => Ok(None), + } + } + + /// # Errors + /// Rejects missing/corrupt retained coverage and unavailable KV queries. + pub async fn locate(&self, id: StreamName, offset: u64) -> Result { + self.locate_for_read(id, offset).await.map(|(record, _)| record) + } + + pub(crate) async fn locate_for_read( + &self, + id: StreamName, + offset: u64, + ) -> Result<(StreamChunkRecordSnapshot, bool)> { + if let Some(record) = self.cache.find(id, offset) { + return Ok((record, true)); + } + let prefix = StreamChunkRecordKey::attached_prefix(id); + let mut before = StreamChunkRecordKey { + stream_id: id, + order: StreamChunkRecordOrder::Attached(offset), + } + .to_bytes(); + // Reverse scan has an exclusive upper bound. No valid fixed-width key + // extends this key, so appending zero includes an exact-start match. + before.push(0); + let records = self + .backend + .scan(&prefix, &before, 1, true) + .await + .map_err(storage_error)?; + let Some(((key, value), revision)) = records.items.into_iter().zip(records.commit_slots).next() + else { + return Err(StreamError::ReadUnavailable( + "Stream chunk record is absent".into(), + )); + }; + let record = Self::decode(&key, &value, revision, id)?; + if record.record.logical_end.map_or(true, |end| offset >= end) + || record.record.state == StreamChunkState::Deleting + { + return Err(corrupt("logical offset is not covered by the chunk record")); + } + self.cache.insert(record.clone()); + Ok((record, false)) + } + + fn decode(key: &[u8], bytes: &[u8], revision: u64, id: StreamName) -> Result { + let key_value = StreamChunkRecordKey::from_bytes(key).map_err(|_| corrupt("record key"))?; + let record = decode_stream_chunk_record(bytes).map_err(codec_error)?; + if key_value.stream_id != id || record.stream_id != id || revision == 0 { + return Err(corrupt("record identity")); + } + match key_value.order { + StreamChunkRecordOrder::Attached(start) + if record.state != StreamChunkState::Candidate && record.logical_start == Some(start) => {} + StreamChunkRecordOrder::Candidate(sequence) + if record.state == StreamChunkState::Candidate && record.sequence == sequence => {} + _ => return Err(corrupt("record key/order differs from value")), + } + let index = Arc::new(record.validate().map_err(codec_error)?); + Ok(StreamChunkRecordSnapshot { + record: Arc::new(record), + index, + key: key.to_vec(), + revision, + }) + } + + async fn load_key(&self, key: &StreamChunkRecordKey) -> Result> { + let bytes = key.to_bytes(); + match self.backend.get(&bytes).await.map_err(storage_error)? { + GetOutcome::NotFound => Ok(None), + GetOutcome::Found { value, revision } => { + Self::decode(&bytes, &value, revision, key.stream_id).map(Some) + } + } + } +} + +fn record_key(record: &StreamChunkRecord) -> Result { + Ok(StreamChunkRecordKey { + stream_id: record.stream_id, + order: if record.state == StreamChunkState::Candidate { + StreamChunkRecordOrder::Candidate(record.sequence) + } else { + StreamChunkRecordOrder::Attached( + record + .logical_start + .ok_or_else(|| corrupt("attached start is absent"))?, + ) + }, + }) +} + +fn codec_error(error: impl std::fmt::Display) -> StreamError { + StreamError::Corruption(error.to_string()) +} +fn corrupt(reason: &str) -> StreamError { + StreamError::Corruption(format!("invalid Stream chunk record: {reason}")) +} +fn storage_error(error: crowdb_kv_client::Error) -> StreamError { + match error { + crowdb_kv_client::Error::CasBusy | crowdb_kv_client::Error::CasFailed { .. } => { + StreamError::Backpressure + } + crowdb_kv_client::Error::OutcomeUnknown => StreamError::WriteStalled, + other => StreamError::ReadUnavailable(format!("Stream chunk record KV operation failed: {other}")), + } +} diff --git a/lib/crowdb-chunk-stream/src/records/backend.rs b/lib/crowdb-chunk-stream/src/records/backend.rs new file mode 100644 index 000000000..b07f3b562 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/records/backend.rs @@ -0,0 +1,19 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use async_trait::async_trait; +use crowdb_kv_client::{BatchOp, GetOutcome, Result, ScanOutcome, WriteOutcome}; + +#[async_trait] +pub(crate) trait RecordBackend: Send + Sync { + async fn get(&self, key: &[u8]) -> Result; + async fn scan(&self, prefix: &[u8], after: &[u8], limit: u32, reverse: bool) -> Result; + async fn scan_keys(&self, prefix: &[u8], after: &[u8], limit: u32) -> Result { + let mut records = self.scan(prefix, after, limit, false).await?; + for (_, value) in &mut records.items { + *value = bytes::Bytes::new(); + } + Ok(records) + } + async fn batch(&self, ops: &[BatchOp], guard: &[u8], revision: u64) -> Result; +} diff --git a/lib/crowdb-chunk-stream/src/records/cache.rs b/lib/crowdb-chunk-stream/src/records/cache.rs new file mode 100644 index 000000000..0c670e332 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/records/cache.rs @@ -0,0 +1,44 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{StreamChunkRecordSnapshot, StreamName}; +use arc_swap::ArcSwapOption; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +const SLOTS: usize = 32; +const SLOT_BYTES: usize = 1024 * 1024; + +pub(super) struct RecordCache { + slots: [ArcSwapOption; SLOTS], + next: AtomicUsize, +} + +impl Default for RecordCache { + fn default() -> Self { + Self { + slots: std::array::from_fn(|_| ArcSwapOption::empty()), + next: AtomicUsize::new(0), + } + } +} + +impl RecordCache { + pub(super) fn find(&self, id: StreamName, offset: u64) -> Option { + self.slots.iter().find_map(|slot| { + let record = slot.load_full()?; + (record.record.stream_id == id + && record.record.logical_start.is_some_and(|start| start <= offset) + && record.record.logical_end.is_some_and(|end| offset < end)) + .then(|| record.as_ref().clone()) + }) + } + pub(super) fn insert(&self, record: StreamChunkRecordSnapshot) { + if record.record.mappings.encoded_len() + record.index.resident_bytes() > SLOT_BYTES { + return; + } + self.slots[self.next.fetch_add(1, Ordering::Relaxed) % SLOTS].store(Some(Arc::new(record))); + } +} diff --git a/lib/crowdb-chunk-stream/src/records/candidate.rs b/lib/crowdb-chunk-stream/src/records/candidate.rs new file mode 100644 index 000000000..f942b9378 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/records/candidate.rs @@ -0,0 +1,141 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{corrupt, publication::put, storage_error, StreamChunkRecordSnapshot, StreamChunkRecordStore}; +use crate::{ChunkId, Result, StreamError}; +use bytes::Bytes; +use crowdb_kv_client::{BatchOp, Error}; +use crowdb_protocol::chunk_stream::{StreamChunkRecord, StreamChunkState, StreamName}; +use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}; + +impl StreamChunkRecordStore { + /// Reserves the first chunk under a concrete revision before attachment. + /// # Errors + /// Rejects an initialized stream, superseding owner or conflicting reservation. + /// The previous reservation must be captured before acquiring writer authority. + pub async fn reserve_initial( + &self, + candidate: &StreamChunkRecord, + previous: Option<&StreamChunkRecordSnapshot>, + ) -> Result { + candidate.validate().map_err(super::codec_error)?; + if candidate.state != StreamChunkState::Candidate || candidate.sequence != 0 { + return Err(corrupt("initial reservation state or sequence")); + } + if self.load_tail(candidate.stream_id).await?.is_some() { + return Err(StreamError::StaleWriter); + } + let key = super::record_key(candidate)?; + if let Some(previous) = previous { + if previous.key != key.to_bytes() { + return Err(corrupt("initial reservation guard belongs to another stream")); + } + if previous.record.writer_epoch > candidate.writer_epoch { + return Err(StreamError::StaleWriter); + } + } + let revision = previous.map_or(0, |previous| previous.revision); + match self + .backend + .batch(&[put(candidate)?], &key.to_bytes(), revision) + .await + { + Ok(_) | Err(Error::CasFailed { .. } | Error::OutcomeUnknown) => { + let registered = self.load_key(&key).await?.ok_or(StreamError::StaleWriter)?; + if registered.record.as_ref() == candidate { + Ok(registered) + } else { + Err(StreamError::StaleWriter) + } + } + Err(error) => Err(storage_error(error)), + } + } + + /// # Errors + /// Rejects changed ownership, conflicting reservation identity and unavailable KV. + pub async fn register_candidate( + &self, + candidate: &StreamChunkRecord, + ) -> Result { + if candidate.state != StreamChunkState::Candidate { + return Err(corrupt("reservation state")); + } + candidate.validate().map_err(super::codec_error)?; + let tail = self + .load_tail(candidate.stream_id) + .await? + .ok_or(StreamError::WriteStalled)?; + if tail.record.writer_epoch != candidate.writer_epoch + || tail.record.owner_session != candidate.owner_session + || tail.record.closed + || tail.record.sequence.checked_add(1) != Some(candidate.sequence) + { + return Err(StreamError::StaleWriter); + } + let key = StreamChunkRecordKey { + stream_id: candidate.stream_id, + order: StreamChunkRecordOrder::Candidate(candidate.sequence), + }; + if let Some(current) = self.load_key(&key).await? { + return if current.record.as_ref() == candidate { + Ok(current) + } else { + Err(StreamError::Backpressure) + }; + } + self.fenced_batch(&tail, vec![put(candidate)?]).await?; + self.load_key(&key).await?.ok_or(StreamError::WriteStalled) + } + + /// # Errors + /// Rejects unavailable/corrupt record scans. + pub async fn candidates(&self, id: StreamName) -> Result> { + let records = self + .backend + .scan(&StreamChunkRecordKey::candidate_prefix(id), &[], 8, false) + .await + .map_err(storage_error)?; + if records.truncated || records.items.len() != records.commit_slots.len() { + return Err(corrupt("candidate count exceeds bound")); + } + records + .items + .into_iter() + .zip(records.commit_slots) + .map(|((key, value), revision)| Self::decode(&key, &value, revision, id)) + .collect() + } + + /// # Errors + /// Rejects stale owner scope or an unresolved conditional delete. + pub async fn remove_candidate(&self, id: StreamName, sequence: u64, chunk: ChunkId) -> Result<()> { + let key = StreamChunkRecordKey { + stream_id: id, + order: StreamChunkRecordOrder::Candidate(sequence), + }; + for _ in 0..16 { + let tail = self.load_tail(id).await?.ok_or(StreamError::WriteStalled)?; + let Some(current) = self.load_key(&key).await? else { + return Ok(()); + }; + if current.record.chunk_id != chunk { + return Err(StreamError::StaleWriter); + } + match self + .fenced_batch( + &tail, + vec![BatchOp::Delete { + key: Bytes::from(key.to_bytes()), + }], + ) + .await + { + Ok(()) => return Ok(()), + Err(StreamError::Backpressure | StreamError::WriteStalled) => tokio::task::yield_now().await, + Err(error) => return Err(error), + } + } + Err(StreamError::Backpressure) + } +} diff --git a/lib/crowdb-chunk-stream/src/records/collection.rs b/lib/crowdb-chunk-stream/src/records/collection.rs new file mode 100644 index 000000000..36f623cf4 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/records/collection.rs @@ -0,0 +1,126 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Logical-interval retention and retryable whole-chunk reclamation. + +use super::{ + corrupt, + pin::{now_ms, PIN_GRACE_MS}, + publication::put, + storage_error, StreamChunkRecordStore, +}; +use crate::{Result, StreamChunkStore, StreamError}; +use crowdb_kv_client::{BatchOp, GetOutcome}; +use crowdb_protocol::chunk_stream::{StreamChunkState, StreamName}; +use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}; + +#[derive(Default)] +pub struct StreamCollectionResult { + pub records: u64, + pub physical_bytes: u64, + pub next: Option, +} + +impl StreamChunkRecordStore { + /// # Errors + /// Rejects stale retention proofs, malformed spans and failed physical release. + /// A deleting record remains retryable until the chunk layer confirms release. + pub async fn collect( + &self, + stream: StreamName, + retained_generation: u64, + after: Option, + max_records: usize, + byte_budget: u64, + chunks: &dyn StreamChunkStore, + ) -> Result { + if max_records == 0 || max_records > 128 || byte_budget == 0 { + return Err(StreamError::InvalidRequest("Stream collection budget".into())); + } + let tail = self.load_tail(stream).await?.ok_or(StreamError::WriteStalled)?; + let pins = self.read_pins(stream).await?; + if self.prune_pins(&tail, &pins).await? { + return Ok(StreamCollectionResult { + next: after, + ..StreamCollectionResult::default() + }); + } + let after = after + .map(|start| { + StreamChunkRecordKey { + stream_id: stream, + order: StreamChunkRecordOrder::Attached(start), + } + .to_bytes() + }) + .unwrap_or_default(); + let records = self + .backend + .scan_keys( + &StreamChunkRecordKey::attached_prefix(stream), + &after, + u32::try_from(max_records) + .map_err(|_| StreamError::InvalidRequest("record budget overflow".into()))?, + ) + .await + .map_err(storage_error)?; + let more = records.truncated; + let mut result = StreamCollectionResult::default(); + let mut decoded_bytes = 0_u64; + for (key, _) in records.items { + if decoded_bytes >= byte_budget { + return Ok(result); + } + let GetOutcome::Found { value, revision } = + self.backend.get(&key).await.map_err(storage_error)? + else { + return Err(StreamError::Backpressure); + }; + decoded_bytes = decoded_bytes.saturating_add(value.len() as u64); + let snapshot = Self::decode(&key, &value, revision, stream)?; + let record = &snapshot.record; + let start = record + .logical_start + .ok_or_else(|| corrupt("collection start missing"))?; + let end = record + .logical_end + .ok_or_else(|| corrupt("collection end missing"))?; + result.next = Some(start); + if record.chunk_id == tail.record.chunk_id + || end > tail.record.trim_offset + || record.generation >= retained_generation + || !matches!( + record.state, + StreamChunkState::Sealed | StreamChunkState::Deleting + ) + { + continue; + } + if pins.iter().any(|pin| { + pin.expires.saturating_add(PIN_GRACE_MS) > now_ms() + && (pin.active == Some(record.chunk_id) || (pin.begin < end && pin.end > start)) + }) { + continue; + } + if record.state != StreamChunkState::Deleting { + let mut deleting = record.as_ref().clone(); + deleting.state = StreamChunkState::Deleting; + // A changed tail revision invalidates the entire pin observation. + self.fenced_batch(&tail, vec![put(&deleting)?]).await?; + } + let reclaimed = chunks + .release_trimmed(record.chunk_id, end, snapshot.index.physical_end()) + .await?; + let current = self.load_tail(stream).await?.ok_or(StreamError::WriteStalled)?; + self.fenced_batch(¤t, vec![BatchOp::Delete { key }]).await?; + result.records += 1; + result.physical_bytes = result.physical_bytes.saturating_add(reclaimed.reclaimed_bytes); + // Re-capture retention before another target because deletion changed its revision. + return Ok(result); + } + if !more { + result.next = None; + } + Ok(result) + } +} diff --git a/lib/crowdb-chunk-stream/src/records/pin.rs b/lib/crowdb-chunk-stream/src/records/pin.rs new file mode 100644 index 000000000..ed56ccd19 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/records/pin.rs @@ -0,0 +1,305 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Finite-reader interval leases for ordered Stream chunk records. + +use super::{corrupt, storage_error, StreamChunkRecordSnapshot, StreamChunkRecordStore}; +use crate::{ChunkId, Result, StreamError}; +use arc_swap::ArcSwap; +use bytes::Bytes; +use crowdb_kv_client::{BatchOp, GetOutcome}; +use crowdb_protocol::chunk_stream::{StreamName, StreamProcessLease}; +use serde::{Deserialize, Serialize}; +use std::{ + sync::Arc, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; +use tokio::sync::oneshot; + +const LEASE_MS: u64 = 60_000; +const RENEW_MS: u64 = 15_000; +pub(super) const PIN_GRACE_MS: u64 = 31_000; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct PinRecord { + pub(super) id: [u8; 16], + pub(super) stream: StreamName, + pub(super) begin: u64, + pub(super) end: u64, + pub(super) active: Option, + pub(super) expires: u64, +} + +pub struct StreamReadPin { + deadline: Arc>, + stop: Option>, + task: Option>, +} + +impl Drop for StreamReadPin { + fn drop(&mut self) { + self.stop.take(); + } +} + +impl StreamReadPin { + pub(crate) fn remaining(&self) -> Duration { + self.deadline.load().saturating_duration_since(Instant::now()) + } + + #[cfg(feature = "test-util")] + pub(crate) fn expire_for_tests(&self) { + self.deadline.store(Arc::new(Instant::now())); + } + pub(crate) async fn release(mut self) { + self.stop.take(); + if let Some(task) = self.task.take() { + let _ = task.await; + } + } + /// # Errors + /// Rejects an expired conservative local lease even if an old task resumes. + pub fn check(&self) -> Result<()> { + if Instant::now() >= **self.deadline.load() { + Err(StreamError::ReadUnavailable("Stream read lease expired".into())) + } else { + Ok(()) + } + } +} + +impl StreamChunkRecordStore { + /// # Errors + /// Rejects trimmed/stale captures, exhausted pin budgets and unavailable KV. + pub async fn pin_read(&self, captured: &StreamChunkRecordSnapshot) -> Result { + let id = StreamProcessLease::generate_boot_nonce() + .map_err(|error| StreamError::Internal(error.to_string()))?; + let started = Instant::now(); + let record = PinRecord { + id, + stream: captured.record.stream_id, + begin: captured.record.trim_offset, + end: captured + .record + .logical_end + .ok_or_else(|| corrupt("pin tail missing"))?, + active: Some(captured.record.chunk_id), + expires: now_ms().saturating_add(LEASE_MS), + }; + self.register_pin(&record).await?; + let deadline = Arc::new(ArcSwap::from_pointee( + started + Duration::from_millis(LEASE_MS - 1000), + )); + let (stop, receive) = oneshot::channel(); + let task = tokio::spawn(maintain(self.clone(), record, deadline.clone(), receive)); + let pin = StreamReadPin { + deadline, + stop: Some(stop), + task: Some(task), + }; + pin.check()?; + Ok(pin) + } + + async fn register_pin(&self, record: &PinRecord) -> Result<()> { + for _ in 0..16 { + let current = self + .load_tail(record.stream) + .await? + .ok_or(StreamError::WriteStalled)?; + if current.record.trim_offset > record.begin || current.record.logical_end < Some(record.end) { + return Err(StreamError::ReadUnavailable( + "captured Stream interval was retired".into(), + )); + } + let pins = self.read_pins(record.stream).await?; + let expired: Vec<_> = pins + .iter() + .filter(|pin| pin.expires.saturating_add(PIN_GRACE_MS) <= now_ms()) + .take(32) + .collect(); + if pins.len().saturating_sub(expired.len()) >= 512 { + return Err(StreamError::Backpressure); + } + let mut ops: Vec<_> = expired + .into_iter() + .map(|pin| BatchOp::Delete { + key: Bytes::from(pin_key(pin)), + }) + .collect(); + ops.push(pin_put(record)?); + match self.fenced_batch(¤t, ops).await { + Ok(()) => return Ok(()), + Err(StreamError::Backpressure) => tokio::task::yield_now().await, + Err(StreamError::WriteStalled) => { + if let GetOutcome::Found { value, .. } = + self.backend.get(&pin_key(record)).await.map_err(storage_error)? + { + let observed: PinRecord = + serde_json::from_slice(&value).map_err(|_| corrupt("read pin encoding"))?; + if &observed == record { + return Ok(()); + } + } + } + Err(error) => return Err(error), + } + } + Err(StreamError::Backpressure) + } + + pub(super) async fn prune_pins( + &self, + tail: &StreamChunkRecordSnapshot, + pins: &[PinRecord], + ) -> Result { + let ops: Vec<_> = pins + .iter() + .filter(|pin| pin.expires.saturating_add(PIN_GRACE_MS) <= now_ms()) + .take(32) + .map(|pin| BatchOp::Delete { + key: Bytes::from(pin_key(pin)), + }) + .collect(); + if ops.is_empty() { + return Ok(false); + } + self.fenced_batch(tail, ops).await?; + Ok(true) + } + + pub(super) async fn read_pins(&self, stream: StreamName) -> Result> { + let records = self + .backend + .scan(&pin_prefix(stream), &[], 512, false) + .await + .map_err(storage_error)?; + if records.truncated { + return Err(corrupt("read pin budget")); + } + records + .items + .into_iter() + .map(|(key, value)| { + if value.len() > 1024 { + return Err(corrupt("read pin length")); + } + let pin: PinRecord = + serde_json::from_slice(&value).map_err(|_| corrupt("read pin encoding"))?; + if pin.stream != stream || pin.begin > pin.end || key.as_ref() != pin_key(&pin) { + return Err(corrupt("read pin identity")); + } + Ok(pin) + }) + .collect() + } +} + +async fn maintain( + store: StreamChunkRecordStore, + mut pin: PinRecord, + deadline: Arc>, + mut stop: oneshot::Receiver<()>, +) { + let mut interval = Duration::from_millis(RENEW_MS); + let released = loop { + tokio::select! { + _ = &mut stop => break true, + () = tokio::time::sleep(interval) => {} + } + let started = Instant::now(); + if started >= **deadline.load() { + break false; + } + interval = Duration::from_millis(100); + let Some(tail) = store.load_tail(pin.stream).await.ok().flatten() else { + continue; + }; + match matches_pin(&store, &pin).await { + Ok(true) if pin.expires > now_ms() => {} + Err(_) => continue, + _ => break false, + } + let mut next = pin.clone(); + next.expires = now_ms().saturating_add(LEASE_MS); + let Ok(op) = pin_put(&next) else { break false }; + let committed = match store.fenced_batch(&tail, vec![op]).await { + Ok(()) => true, + Err(StreamError::WriteStalled) => matches_pin(&store, &next).await.unwrap_or(false), + Err(_) => false, + }; + if committed { + if Instant::now() >= **deadline.load() { + break false; + } + pin = next; + deadline.store(Arc::new(started + Duration::from_millis(LEASE_MS - 1000))); + interval = Duration::from_millis(RENEW_MS); + } + }; + if released { + release_pin(&store, &pin).await; + } else { + deadline.store(Arc::new(Instant::now())); + } +} + +async fn matches_pin(store: &StreamChunkRecordStore, pin: &PinRecord) -> Result { + match store.backend.get(&pin_key(pin)).await.map_err(storage_error)? { + GetOutcome::NotFound => Ok(false), + GetOutcome::Found { value, .. } => { + if value.len() > 1024 { + return Err(corrupt("read pin length")); + } + let observed: PinRecord = + serde_json::from_slice(&value).map_err(|_| corrupt("read pin encoding"))?; + Ok(&observed == pin) + } + } +} + +async fn release_pin(store: &StreamChunkRecordStore, pin: &PinRecord) { + for _ in 0..8 { + let Ok(Some(tail)) = store.load_tail(pin.stream).await else { + break; + }; + match store + .fenced_batch( + &tail, + vec![BatchOp::Delete { + key: Bytes::from(pin_key(pin)), + }], + ) + .await + { + Err(StreamError::Backpressure) => tokio::task::yield_now().await, + _ => break, + } + } +} + +pub(super) fn pin_prefix(stream: StreamName) -> Vec { + let mut key = vec![crowdb_protocol::key::CROWDB_KEY_MAGIC, 0, 0x1e]; + key.extend_from_slice(&stream.high.to_be_bytes()); + key.extend_from_slice(&stream.low.to_be_bytes()); + key +} +fn pin_key(pin: &PinRecord) -> Vec { + let mut key = pin_prefix(pin.stream); + key.extend_from_slice(&pin.id); + key +} +fn pin_put(pin: &PinRecord) -> Result { + Ok(BatchOp::Put { + key: Bytes::from(pin_key(pin)), + value: Bytes::from(serde_json::to_vec(pin).map_err(|_| corrupt("read pin encoding"))?), + }) +} +pub(super) fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) + }) +} diff --git a/lib/crowdb-chunk-stream/src/records/publication.rs b/lib/crowdb-chunk-stream/src/records/publication.rs new file mode 100644 index 000000000..c5a655006 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/records/publication.rs @@ -0,0 +1,199 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + codec_error, corrupt, record_key, storage_error, StreamChunkRecordSnapshot, StreamChunkRecordStore, +}; +use crate::{Result, StreamError}; +use bytes::Bytes; +use crowdb_kv_client::{BatchOp, Error}; +use crowdb_protocol::chunk_stream::{encode_stream_chunk_record, StreamChunkRecord, StreamChunkState}; +use crowdb_protocol::key::BinaryKey; + +impl StreamChunkRecordStore { + /// Atomically finalize an old tail and attach a prepared successor, or + /// checkpoint/control-update the current tail. No Stream identity is changed. + /// # Errors + /// Rejects wrong scope/progress, stale writers, conflicting revisions and unavailable KV. + pub async fn publish( + &self, + expected: Option<&StreamChunkRecordSnapshot>, + tail: StreamChunkRecord, + previous: Option, + candidate: Option<&StreamChunkRecordSnapshot>, + ) -> Result { + validate_transition(expected, &tail, previous.as_ref())?; + let guarded = expected + .or(candidate) + .ok_or_else(|| corrupt("missing attachment guard"))?; + if expected.is_none() { + if let Some(current) = self.load_tail(tail.stream_id).await? { + return if current.record.as_ref() == &tail { + Ok(current) + } else { + Err(StreamError::StaleWriter) + }; + } + } + let key = record_key(&tail)?.to_bytes(); + let value = encode_stream_chunk_record(&tail).map_err(codec_error)?; + let mut ops = Vec::with_capacity(3); + if let Some(previous) = &previous { + let old_key = record_key(previous)?.to_bytes(); + if old_key != key { + ops.push(put(previous)?); + } else if previous.logical_start != previous.logical_end { + return Err(corrupt("nonempty predecessor key reused")); + } + } + if let Some(candidate) = candidate { + if candidate.record.state != StreamChunkState::Candidate + || candidate.record.chunk_id != tail.chunk_id + || candidate.record.sequence != tail.sequence + || candidate.record.stream_id != tail.stream_id + || candidate.record.writer_epoch != tail.writer_epoch + || candidate.record.owner_session != tail.owner_session + { + return Err(corrupt("candidate identity differs from attachment")); + } + ops.push(BatchOp::Delete { + key: Bytes::copy_from_slice(&candidate.key), + }); + } + ops.push(BatchOp::Put { + key: Bytes::copy_from_slice(&key), + value: Bytes::from(value.clone()), + }); + let guard = guarded.key.as_slice(); + let mut revision = guarded.revision; + for attempt in 0_u32..8 { + match self.backend.batch(&ops, guard, revision).await { + Ok(outcome) => return Self::decode(&key, &value, outcome.revision, tail.stream_id), + Err(Error::CasBusy) => { + tokio::time::sleep(std::time::Duration::from_millis(1 << attempt.min(4))).await; + } + Err(Error::CasFailed { .. } | Error::OutcomeUnknown) => { + let current = self.load_tail(tail.stream_id).await?; + if let Some(current) = ¤t { + if current.record.as_ref() == &tail { + return Ok(current.clone()); + } + } + match (expected, current) { + (Some(expected), Some(current)) if current.record == expected.record => { + revision = current.revision; + if let Some(candidate) = candidate { + let registered = self.load_key(&record_key(&candidate.record)?).await?; + if registered.map_or(true, |registered| registered.record != candidate.record) + { + return Err(StreamError::StaleWriter); + } + } + } + (_, Some(current)) if current.record.writer_epoch >= tail.writer_epoch => { + return Err(StreamError::StaleWriter) + } + _ => return Err(StreamError::WriteStalled), + } + } + Err(error) => return Err(storage_error(error)), + } + } + Err(StreamError::Backpressure) + } + + pub(super) async fn fenced_batch( + &self, + expected: &StreamChunkRecordSnapshot, + mut ops: Vec, + ) -> Result<()> { + ops.push(put(&expected.record)?); + for attempt in 0_u32..8 { + match self.backend.batch(&ops, &expected.key, expected.revision).await { + Ok(_) => return Ok(()), + Err(Error::CasBusy) => { + tokio::time::sleep(std::time::Duration::from_millis(1 << attempt.min(4))).await; + } + Err(error) => return Err(storage_error(error)), + } + } + Err(StreamError::Backpressure) + } +} + +pub(super) fn put(record: &StreamChunkRecord) -> Result { + Ok(BatchOp::Put { + key: Bytes::from(record_key(record)?.to_bytes()), + value: Bytes::from(encode_stream_chunk_record(record).map_err(codec_error)?), + }) +} + +fn validate_transition( + expected: Option<&StreamChunkRecordSnapshot>, + tail: &StreamChunkRecord, + previous: Option<&StreamChunkRecord>, +) -> Result<()> { + tail.validate().map_err(codec_error)?; + if tail.state == StreamChunkState::Candidate || tail.state == StreamChunkState::Deleting { + return Err(corrupt("invalid tail state")); + } + let Some(expected) = expected else { + return if tail.generation == 1 + && tail.sequence == 0 + && tail.logical_start == Some(0) + && tail.logical_end == Some(0) + { + Ok(()) + } else { + Err(corrupt("initial tail bounds")) + }; + }; + let base = &expected.record; + if base.stream_id != tail.stream_id + || base.purpose != tail.purpose + || base.writer_epoch > tail.writer_epoch + || base.generation.checked_add(1) != Some(tail.generation) + || tail.trim_offset < base.trim_offset + || tail.logical_end < base.logical_end + || (base.closed && !tail.closed) + { + return Err(corrupt("tail transition regresses or changes scope")); + } + if base.chunk_id == tail.chunk_id { + if previous.is_some() || tail.sequence != base.sequence || tail.logical_start != base.logical_start { + return Err(corrupt("same-chunk transition changes attachment")); + } + validate_prefix(base, tail)?; + } else { + let previous = previous.ok_or_else(|| corrupt("missing finalized predecessor"))?; + previous.validate().map_err(codec_error)?; + if previous.chunk_id != base.chunk_id + || previous.logical_start != base.logical_start + || previous.stream_id != base.stream_id + || previous.purpose != base.purpose + || previous.sequence != base.sequence + || previous.writer_epoch != tail.writer_epoch + || previous.owner_session != tail.owner_session + || previous.generation != tail.generation + || previous.state != StreamChunkState::Sealed + || previous.logical_end != tail.logical_start + || previous.logical_end < base.logical_end + || base.sequence.checked_add(1) != Some(tail.sequence) + || base.closed + || tail.state != StreamChunkState::Active + { + return Err(corrupt("predecessor/successor continuity")); + } + validate_prefix(base, previous)?; + } + Ok(()) +} + +fn validate_prefix(base: &StreamChunkRecord, next: &StreamChunkRecord) -> Result<()> { + let base_lengths = base.mappings.lengths().map_err(codec_error)?; + let next_lengths = next.mappings.lengths().map_err(codec_error)?; + if !next_lengths.starts_with(&base_lengths) { + return Err(corrupt("checkpoint changes acknowledged append boundaries")); + } + Ok(()) +} diff --git a/lib/crowdb-chunk-stream/src/storage.rs b/lib/crowdb-chunk-stream/src/storage.rs index 3a01b2f70..9640a4f15 100644 --- a/lib/crowdb-chunk-stream/src/storage.rs +++ b/lib/crowdb-chunk-stream/src/storage.rs @@ -3,18 +3,18 @@ use async_trait::async_trait; use bytes::Bytes; -use crowdb_protocol::chunk_stream::{ - ActiveChunkDescriptor, StreamBinding, StreamExtentPage, StreamManifest, StreamName, -}; +use crowdb_protocol::chunk_stream::{ActiveChunkDescriptor, StreamName, StreamRecord}; use crowdb_protocol::common::ChunkId; use crowdb_protocol::frame::{parse_frame, FrameMagic}; use crate::Result; +mod unwritten; +pub(crate) use unwritten::{padded_end, validate_write_bounds}; + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum CursorAdvance { Committed, - DefinitelyNotCommitted, Ambiguous, } @@ -31,6 +31,12 @@ pub struct DurableCursor { pub sealed: bool, } +pub struct RecoveredStreamTail { + pub physical_end: u64, + pub logical_end: u64, + pub extents: Vec, +} + #[derive(Clone)] pub struct MirrorStripImage { pub block_offset: u64, @@ -38,38 +44,44 @@ pub struct MirrorStripImage { pub full_image: Bytes, } +/// Ordered bounded strip images and the exact complete data boundary of one batch. +/// Image data never extends beyond the complete boundary, including on retries. +#[derive(Clone, Copy)] +pub struct StreamMirrorWrite<'a> { + pub authority: Option<&'a crate::StreamWriteContext>, + pub physical_offset: u64, + pub complete_end: u64, + pub images: &'a [MirrorStripImage], +} + #[async_trait] pub trait StreamRegistry: Send + Sync { - async fn load(&self, stream_name: StreamName) -> Result>; - async fn create(&self, binding: StreamBinding) -> Result<()>; + async fn load(&self, stream_name: StreamName) -> Result>; + async fn create(&self, record: StreamRecord) -> Result<()>; + async fn claim_writer( + &self, + name: StreamName, + epoch: u64, + ) -> Result>; } #[async_trait] pub trait StreamMetadataStore: Send + Sync { - async fn load_current(&self, stream_name: StreamName) -> Result>; - async fn load_extent_page( - &self, - stream_name: StreamName, - writer_epoch: u64, - generation: u64, - page_index: u64, - ) -> Result>; - async fn publish( - &self, - expected: Option<(u64, u64)>, - manifest: StreamManifest, - extent_pages: Vec, - ) -> Result<()>; - async fn reclaim_extent_pages_before( - &self, - stream_name: StreamName, - retained_generation: u64, - max_pages: usize, - ) -> Result; + /// # Errors + /// Rejects an invalid or mismatched nonzero metadata group. + fn record_store(&self, metadata_group_id: u64) -> Result; } #[async_trait] pub trait StreamChunkStore: Send + Sync { + /// Required synchronized replica count for append traffic accounting. + fn mirror_copies(&self) -> u32; + fn bind_writer(&self, name: StreamName, epoch: u64, session: std::sync::Arc); + /// Current physical capacity from the chunk layer, independent of a cached + /// Stream checkpoint. + /// # Errors + /// Returns chunk metadata/availability errors. + async fn current_capacity(&self, chunk_id: ChunkId) -> Result; async fn allocate_mirrored( &self, stream_name: StreamName, @@ -79,6 +91,9 @@ pub trait StreamChunkStore: Send + Sync { /// Extends an active mirror chunk so `required_capacity` bytes can be /// addressed. `None` means that this store has reached the chunk's fixed /// logical limit and the caller must seal and roll over. + /// One preparation may overlap appends to the previously initialized + /// prefix. Return capacity only after every new replica is initialized; + /// layout publication must preserve concurrent writer progress. async fn grow_mirrored( &self, _stream_name: StreamName, @@ -88,14 +103,12 @@ pub trait StreamChunkStore: Send + Sync { ) -> Result> { Ok(None) } - async fn write_mirrors_with_images( + async fn write_mirrors( &self, stream_name: StreamName, writer_epoch: u64, chunk_id: ChunkId, - physical_offset: u64, - data: Bytes, - images: &[MirrorStripImage], + write: StreamMirrorWrite<'_>, ) -> Result<()>; async fn advance_cursor( &self, @@ -107,10 +120,16 @@ pub trait StreamChunkStore: Send + Sync { checksum: u32, ) -> Result; async fn durable_cursor(&self, chunk_id: ChunkId, writer_epoch: u64) -> Result; + /// Scan complete groups beyond the last checkpoint after excluding the old + /// producer. Failed IO cannot be interpreted as an empty physical suffix. + async fn recover_tail( + &self, + name: StreamName, + writer_epoch: u64, + active: &ActiveChunkDescriptor, + ) -> Result; /// Renews the one Active-chunk liveness task without advancing its cursor. - async fn renew_liveness(&self, _chunk_id: ChunkId, _writer_epoch: u64) -> Result<()> { - Ok(()) - } + async fn renew_liveness(&self, chunk_id: ChunkId, writer_epoch: u64) -> Result<()>; async fn seal(&self, chunk_id: ChunkId, writer_epoch: u64, cursor: u64) -> Result<()>; async fn read(&self, chunk_id: ChunkId, physical_offset: u64, length: usize) -> Result; /// Reads and validates a public stream frame before its payload is exposed. @@ -118,21 +137,39 @@ pub trait StreamChunkStore: Send + Sync { /// In-memory and test stores use the default parser. Production overrides /// this to let `ChunkReader` retry a corrupt serving mirror or reconstruct /// an EC stripe before returning the frame. + /// The complete-group bound comes from this read view's validated mappings; + /// it does not update a cached chunk cursor or persistent metadata. async fn read_verified_frame( &self, chunk_id: ChunkId, physical_offset: u64, length: usize, + synchronized_end: u64, ) -> Result { + if physical_offset + .checked_add(length as u64) + .map_or(true, |end| end > synchronized_end) + { + return Err(crate::StreamError::Corruption( + "frame exceeds captured group boundary".into(), + )); + } let frame = self.read(chunk_id, physical_offset, length).await?; let parsed = parse_frame(&frame, chunk_id) .map_err(|error| crate::StreamError::Corruption(format!("invalid stream frame: {error}")))?; - if parsed.header.magic != FrameMagic::StreamV1 { + if parsed.header.magic != FrameMagic::StreamV2 { return Err(crate::StreamError::Corruption( "stream extent has the wrong frame kind".into(), )); } Ok(frame) } - async fn release_trimmed(&self, chunk_id: ChunkId, logical_end: u64) -> Result; + /// Releases an exactly sealed whole chunk. Active chunks or a different + /// physical end must fail before deletion; already released IDs are idempotent. + async fn release_trimmed( + &self, + chunk_id: ChunkId, + logical_end: u64, + physical_end: u64, + ) -> Result; } diff --git a/lib/crowdb-chunk-stream/src/storage/unwritten.rs b/lib/crowdb-chunk-stream/src/storage/unwritten.rs new file mode 100644 index 000000000..5aa097dd3 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/storage/unwritten.rs @@ -0,0 +1,33 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Reserve an unwritten header inside initialized physical capacity. +use crowdb_protocol::frame::FRAME_HEADER_PREFIX_BYTES; + +use super::StreamMirrorWrite; +use crate::{Result, StreamError}; + +pub(crate) fn padded_end(complete_end: u64, physical_limit: u64) -> Result { + let remaining = physical_limit + .checked_sub(complete_end) + .ok_or_else(|| StreamError::InvalidRequest("complete append exceeds physical chunk limit".into()))?; + Ok(complete_end + remaining.min(FRAME_HEADER_PREFIX_BYTES as u64)) +} + +pub(crate) fn validate_write_bounds(write: StreamMirrorWrite<'_>, physical_limit: u64) -> Result { + let length = write + .images + .iter() + .try_fold(0_u64, |sum, image| sum.checked_add(image.data.len() as u64)) + .ok_or_else(|| StreamError::InvalidRequest("stream image length overflows".into()))?; + let end = write + .physical_offset + .checked_add(length) + .ok_or_else(|| StreamError::InvalidRequest("stream image end overflows".into()))?; + if write.complete_end <= write.physical_offset || end != write.complete_end || end > physical_limit { + return Err(StreamError::InvalidRequest( + "stream images must cover only their complete append range".into(), + )); + } + Ok(end) +} diff --git a/lib/crowdb-chunk-stream/src/stream.rs b/lib/crowdb-chunk-stream/src/stream.rs index 0c8198ef3..10ed8aa5f 100644 --- a/lib/crowdb-chunk-stream/src/stream.rs +++ b/lib/crowdb-chunk-stream/src/stream.rs @@ -7,26 +7,44 @@ use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use arc_swap::ArcSwap; use bytes::{Bytes, BytesMut}; -use crowdb_protocol::chunk_stream::{ - StreamBinding, StreamBindingState, StreamExtentPage, StreamExtentPageFence, StreamManifest, StreamName, -}; +use crowdb_protocol::chunk_stream::{ActiveChunkDescriptor, StreamName, StreamRecord, StreamRecordState}; use crowdb_protocol::common::ChunkId; -use crowdb_protocol::frame::{ - encode_frames, framed_physical_length, parse_frame, FrameMagic, FRAME_HEADER_PREFIX_BYTES, -}; +use crowdb_protocol::frame::FrameLayout; use tokio::sync::{mpsc, oneshot}; -use crate::extent_cache::ExtentPageCache; -use crate::metadata::{resolve_extent, validate_manifest}; use crate::metrics::{StreamMetrics, StreamMetricsSnapshot}; use crate::mirror_shadow::MirrorShadow; -use crate::storage::{CursorAdvance, StreamChunkStore, StreamMetadataStore, StreamRegistry}; +use crate::storage::{StreamChunkStore, StreamMetadataStore, StreamRegistry}; use crate::{Result, StreamError}; - +use crate::{StreamChunkRecordSnapshot, StreamChunkRecordStore}; + +mod append; +mod capacity; +mod capacity_plan; +#[cfg(feature = "test-util")] +pub use capacity_plan::capacity_target_for_tests; +mod checkpoint; +mod collection; mod liveness; mod observation; -mod recovery; +mod open; +mod preparation; +mod publication; +mod state; +use state::{StreamViewState, WriterProgress}; +mod read; +mod successor; +mod view; +mod worker; pub use observation::StreamMetadataObservation; +pub use read::StreamReader; +use view::ReadView; +pub use view::StreamViewSource; + +use crate::perf::{observe, perf}; +use worker::run_worker; + +use capacity::rollover; #[derive(Clone, Debug)] pub struct StreamConfig { @@ -36,7 +54,6 @@ pub struct StreamConfig { pub batch_requests: usize, pub batch_bytes: usize, pub max_append_bytes: usize, - pub extent_page_entries: usize, pub read_window_bytes: usize, pub read_request_bytes: usize, pub read_concurrency: usize, @@ -54,7 +71,6 @@ impl Default for StreamConfig { batch_requests: 64, batch_bytes: 1024 * 1024, max_append_bytes: 64 * 1024 * 1024, - extent_page_entries: 1_024, read_window_bytes: 8 * 1024 * 1024, read_request_bytes: 1024 * 1024, read_concurrency: 8, @@ -73,7 +89,6 @@ impl StreamConfig { || self.batch_requests == 0 || self.batch_bytes == 0 || self.max_append_bytes == 0 - || self.extent_page_entries == 0 || self.read_window_bytes == 0 || self.read_request_bytes == 0 || self.read_concurrency == 0 @@ -114,6 +129,9 @@ struct PhysicalRead { frame_length: usize, payload_start: usize, length: usize, + group_start: u64, + group_length: u64, + ordinal: u32, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -124,6 +142,7 @@ pub enum ReadHint { struct AppendRequest { data: Bytes, + authority: crate::StreamWriteContext, enqueued_at: Instant, completion: oneshot::Sender>, } @@ -134,41 +153,46 @@ impl AppendRequest { } fn physical_len(&self) -> Result { - usize::try_from( - framed_physical_length( - u64::try_from(self.data.len()) - .map_err(|_| StreamError::InvalidRequest("append length exceeds u64".into()))?, - ) - .map_err(|error| StreamError::InvalidRequest(error.to_string()))?, - ) - .map_err(|_| StreamError::InvalidRequest("framed append exceeds address space".into())) + let length = FrameLayout::StreamGroupV2 + .physical_length(self.data.len() as u64) + .map_err(|error| StreamError::InvalidRequest(error.to_string()))?; + usize::try_from(length) + .map_err(|_| StreamError::InvalidRequest("framed append exceeds address space".into())) } } enum Command { Append(AppendRequest), + Capture(oneshot::Sender>>), Trim { offset: u64, completion: oneshot::Sender>, }, Close(oneshot::Sender>), + Release(oneshot::Sender>), + Reclaim { + retained_generation: u64, + max_records: usize, + completion: oneshot::Sender>, + }, } #[derive(Clone)] pub struct ChunkStream { + authority: Option>, stream_name: StreamName, writer_epoch: u64, config: Arc, - metadata: Arc, + metadata: StreamChunkRecordStore, chunks: Arc, sender: mpsc::Sender, queued_bytes: Arc, queued_requests: Arc, - tail: Arc, - manifest: Arc>, + state: Arc>, + read_view: Arc>, + view_source: StreamViewSource, closed: Arc, metrics: Arc, - extent_cache: Arc, } #[derive(Clone)] @@ -177,24 +201,37 @@ struct Extent { logical_start: u64, logical_end: u64, frame_start: u64, - frame_length: u32, } struct WorkerState { + authority: Option>, stream_name: StreamName, writer_epoch: u64, config: Arc, - metadata: Arc, + metadata: StreamChunkRecordStore, chunks: Arc, - manifest_view: Arc>, + published_view: Arc>, + read_view: Arc>, tail_view: Arc, queued_bytes: Arc, queued_requests: Arc, closed_view: Arc, - manifest: StreamManifest, + progress: WriterProgress, + snapshot: StreamChunkRecordSnapshot, + collection_after: Option, + owner_session: [u8; 16], extents: Vec, mirror_shadow: MirrorShadow, + checkpoint: Option, + capacity_preparation: Option, + capacity_failure: Option<(ChunkId, StreamError)>, + capacity_exhausted: Option, + capacity_plan: capacity_plan::CapacityPlan, + successor: Option, + liveness_failed: Arc, + liveness_changed: Arc, stalled: bool, + last_unavailable_log: Option, metrics: Arc, } @@ -207,10 +244,14 @@ enum BatchFailure { } impl ChunkStream { + #[cfg(feature = "test-util")] + pub fn expire_read_lease_for_tests(&self) { + self.read_view.load().pin.expire_for_tests(); + } /// Durable chunk purpose of this stream. #[must_use] pub fn purpose(&self) -> crowdb_protocol::chunk_stream::StreamPurpose { - self.manifest.load().purpose + self.state.load().purpose } #[must_use] @@ -218,192 +259,6 @@ impl ChunkStream { self.stream_name } - /// Creates and publishes a new registered stream. - /// - /// # Errors - /// - /// Returns a typed storage, validation, fencing, or duplicate-name error. - pub async fn create( - binding: StreamBinding, - writer_epoch: u64, - config: StreamConfig, - registry: Arc, - metadata: Arc, - chunks: Arc, - ) -> Result { - config.validate()?; - if binding.state != StreamBindingState::Active - || binding.metadata_group_id == 0 - || binding.binding_generation == 0 - || writer_epoch == 0 - { - return Err(StreamError::InvalidRequest( - "stream binding or writer epoch is invalid".into(), - )); - } - if registry.load(binding.stream_name).await?.is_some() { - return Err(StreamError::InvalidRequest("stream already exists".into())); - } - registry.create(binding.clone()).await?; - Self::initialize(binding, writer_epoch, config, metadata, chunks).await - } - - /// Initializes stream metadata for an active binding already published by - /// the control plane. - /// - /// # Errors - /// - /// Returns a typed error if the binding is absent or inactive, metadata - /// already exists, or initial metadata publication fails. - pub async fn create_registered( - stream_name: StreamName, - writer_epoch: u64, - config: StreamConfig, - registry: Arc, - metadata: Arc, - chunks: Arc, - ) -> Result { - config.validate()?; - let binding = registry - .load(stream_name) - .await? - .ok_or_else(|| StreamError::InvalidRequest("stream binding does not exist".into()))?; - if binding.state != StreamBindingState::Active - || binding.metadata_group_id == 0 - || binding.binding_generation == 0 - || writer_epoch == 0 - { - return Err(StreamError::InvalidRequest( - "stream binding or writer epoch is invalid".into(), - )); - } - if metadata.load_current(stream_name).await?.is_some() { - return Err(StreamError::InvalidRequest( - "stream metadata already exists".into(), - )); - } - Self::initialize(binding, writer_epoch, config, metadata, chunks).await - } - - async fn initialize( - binding: StreamBinding, - writer_epoch: u64, - config: StreamConfig, - metadata: Arc, - chunks: Arc, - ) -> Result { - let manifest = StreamManifest { - purpose: binding.purpose, - stream_name: binding.stream_name, - metadata_group_id: binding.metadata_group_id, - writer_epoch, - generation: 1, - trim_offset: 0, - sealed_tail: 0, - active: None, - extent_pages: Vec::new(), - previous_generation: None, - closed: false, - }; - metadata.publish(None, manifest.clone(), Vec::new()).await?; - Self::start(config, metadata, chunks, manifest, Vec::new(), &[]) - } - - /// Opens an existing stream and recovers its durable active cursor. - /// - /// # Errors - /// - /// Returns a typed storage, corruption, or stale-writer error. - pub async fn open( - stream_name: StreamName, - writer_epoch: u64, - config: StreamConfig, - registry: Arc, - metadata: Arc, - chunks: Arc, - ) -> Result { - config.validate()?; - let binding = registry - .load(stream_name) - .await? - .ok_or_else(|| StreamError::InvalidRequest("stream binding does not exist".into()))?; - if binding.state != StreamBindingState::Active { - return Err(StreamError::InvalidRequest("stream binding is not active".into())); - } - let mut manifest = metadata - .load_current(stream_name) - .await? - .ok_or_else(|| StreamError::Corruption("stream manifest does not exist".into()))?; - if manifest.purpose != binding.purpose { - return Err(StreamError::Corruption( - "stream binding purpose differs from manifest".into(), - )); - } - if manifest.metadata_group_id != binding.metadata_group_id || manifest.writer_epoch > writer_epoch { - return Err(StreamError::StaleWriter); - } - let pages = load_extent_pages(metadata.as_ref(), &manifest).await?; - validate_manifest(&manifest, &pages)?; - let expected = (manifest.writer_epoch, manifest.generation); - let mut needs_publish = manifest.writer_epoch < writer_epoch; - let mut rotate_active = false; - let mut extents = collect_extents(&pages); - let mut tail = manifest.sealed_tail; - if let Some(active) = &mut manifest.active { - let durable = chunks.durable_cursor(active.chunk_id, writer_epoch).await?; - if durable.offset < active.physical_start || durable.offset > active.capacity { - return Err(StreamError::Corruption( - "recovered active cursor is outside chunk bounds".into(), - )); - } - if durable.offset < active.acknowledged_cursor { - return Err(StreamError::Corruption( - "recovered active cursor precedes its published cursor".into(), - )); - } - tail = recovery::recover_active_frames(chunks.as_ref(), active, durable.offset, &mut extents) - .await?; - if !durable.sealed { - chunks.seal(active.chunk_id, writer_epoch, durable.offset).await?; - } - active.acknowledged_cursor = durable.offset; - rotate_active = true; - } - manifest.writer_epoch = writer_epoch; - if rotate_active { - manifest.active.take(); - manifest.sealed_tail = tail; - let mut successor = chunks - .allocate_mirrored(stream_name, writer_epoch, manifest.purpose) - .await?; - successor.logical_start = tail; - manifest.active = Some(successor); - needs_publish = true; - } - if needs_publish { - manifest.previous_generation = Some(manifest.generation); - manifest.generation = manifest - .generation - .checked_add(1) - .ok_or_else(|| StreamError::Corruption("manifest generation overflows".into()))?; - let adopted_pages = build_extent_pages( - &extents, - manifest.stream_name, - writer_epoch, - manifest.generation, - config.extent_page_entries, - ); - manifest.extent_pages = fences_for(&adopted_pages); - validate_manifest(&manifest, &adopted_pages)?; - metadata - .publish(Some(expected), manifest.clone(), adopted_pages) - .await?; - } - let stream = Self::start(config, metadata, chunks, manifest, extents, &pages)?; - stream.tail.store(tail, Ordering::Release); - Ok(stream) - } - /// Opens one immutable view at the exact current writer epoch. /// /// The handle observes the durable active cursor without sealing, rotating, @@ -421,122 +276,149 @@ impl ChunkStream { chunks: Arc, ) -> Result { config.validate()?; - let binding = registry + let record = registry .load(stream_name) .await? - .ok_or_else(|| StreamError::InvalidRequest("stream binding does not exist".into()))?; - if binding.state != StreamBindingState::Active || writer_epoch == 0 { + .ok_or_else(|| StreamError::InvalidRequest("stream record does not exist".into()))?; + if record.state != StreamRecordState::Active || writer_epoch == 0 { return Err(StreamError::InvalidRequest( - "stream binding or reader epoch is invalid".into(), + "stream record or reader epoch is invalid".into(), )); } - let mut manifest = metadata - .load_current(stream_name) + let metadata = metadata.record_store(record.metadata_group_id)?; + let snapshot = metadata + .load_tail(stream_name) .await? - .ok_or_else(|| StreamError::Corruption("stream manifest does not exist".into()))?; - if manifest.purpose != binding.purpose { + .ok_or_else(|| StreamError::Corruption("Stream tail chunk record does not exist".into()))?; + let published = StreamViewState::new(snapshot.clone(), metadata.group_id()); + if published.purpose != record.purpose { return Err(StreamError::Corruption( - "stream binding purpose differs from manifest".into(), + "stream record purpose differs from published".into(), )); } - if manifest.metadata_group_id != binding.metadata_group_id || manifest.writer_epoch != writer_epoch { + if published.metadata_group_id != record.metadata_group_id || published.writer_epoch != writer_epoch { return Err(StreamError::StaleWriter); } - let pages = load_extent_pages(metadata.as_ref(), &manifest).await?; - validate_manifest(&manifest, &pages)?; - if let Some(active) = &mut manifest.active { - let durable = chunks.durable_cursor(active.chunk_id, writer_epoch).await?; - if durable.offset < active.physical_start || durable.offset > active.capacity { - return Err(StreamError::Corruption( - "observed active cursor is outside chunk bounds".into(), - )); - } - if durable.offset < active.acknowledged_cursor { - return Err(StreamError::Corruption( - "observed active cursor precedes the published cursor".into(), - )); - } - // A concurrent append can advance the physical cursor before its - // extent pages are published. Keep this reader at the captured - // manifest's acknowledged boundary. - } - validate_manifest(&manifest, &pages)?; - let stream = Self::start( - config, - metadata, - chunks, - manifest, - collect_extents(&pages), - &pages, - )?; + let active = state::load_active(&snapshot, chunks.as_ref()).await?; + let mut stream = Self::start(config, metadata, chunks, snapshot, active, None).await?; stream.closed.store(true, Ordering::Release); + stream.view_source = StreamViewSource::PersistedCheckpoint; Ok(stream) } - fn start( + async fn start( config: StreamConfig, - metadata: Arc, + metadata: StreamChunkRecordStore, chunks: Arc, - manifest: StreamManifest, - extents: Vec, - pages: &[StreamExtentPage], + snapshot: StreamChunkRecordSnapshot, + active: Option, + authority: Option>, ) -> Result { + let published = StreamViewState::new(snapshot.clone(), metadata.group_id()); let (sender, receiver) = mpsc::channel(config.queue_requests); let config = Arc::new(config); let queued_bytes = Arc::new(AtomicU64::new(0)); let queued_requests = Arc::new(AtomicUsize::new(0)); - let tail = Arc::new(AtomicU64::new(durable_tail(&manifest)?)); - let manifest_view = Arc::new(ArcSwap::from_pointee(manifest.clone())); - let closed = Arc::new(AtomicBool::new(manifest.closed)); + let tail = Arc::new(AtomicU64::new(published.published_tail)); + let published_view = Arc::new(ArcSwap::from_pointee(published)); + let read_view = Arc::new(ArcSwap::from_pointee( + ReadView::new(metadata.clone(), published_view.load_full()).await?, + )); + let closed = Arc::new(AtomicBool::new(published_view.load().closed)); let metrics = Arc::new(StreamMetrics::default()); - let extent_cache = Arc::new(ExtentPageCache::new(&manifest, pages)); + metrics + .metadata_publications + .store(u64::from(authority.is_some()), Ordering::Relaxed); + perf(); let state = WorkerState { - stream_name: manifest.stream_name, - writer_epoch: manifest.writer_epoch, + authority: authority.clone(), + stream_name: published_view.load().stream_name, + writer_epoch: published_view.load().writer_epoch, config: Arc::clone(&config), - metadata: Arc::clone(&metadata), + metadata: metadata.clone(), chunks: Arc::clone(&chunks), - manifest_view: Arc::clone(&manifest_view), + published_view: Arc::clone(&published_view), + read_view: Arc::clone(&read_view), tail_view: Arc::clone(&tail), queued_bytes: Arc::clone(&queued_bytes), queued_requests: Arc::clone(&queued_requests), closed_view: Arc::clone(&closed), - manifest, - extents, + progress: WriterProgress::new(snapshot.record(), active), + owner_session: snapshot.record().owner_session, + snapshot, + collection_after: None, + extents: Vec::new(), mirror_shadow: MirrorShadow::default(), + checkpoint: None, + capacity_preparation: None, + capacity_failure: None, + capacity_exhausted: None, + capacity_plan: capacity_plan::CapacityPlan::default(), + successor: None, + liveness_failed: Arc::new(AtomicBool::new(false)), + liveness_changed: Arc::new(tokio::sync::Notify::new()), stalled: false, + last_unavailable_log: None, metrics: Arc::clone(&metrics), }; - tokio::spawn(run_worker(state, receiver)); + if authority.is_some() { + tokio::spawn(run_worker(state, receiver)); + } Ok(Self { - stream_name: state_name(&manifest_view), - writer_epoch: state_epoch(&manifest_view), + authority, + stream_name: published_view.load().stream_name, + writer_epoch: published_view.load().writer_epoch, config, metadata, chunks, sender, queued_bytes, queued_requests, - tail, - manifest: manifest_view, + state: Arc::clone(&published_view), + read_view, + view_source: StreamViewSource::LiveOwner, closed, metrics, - extent_cache, }) } - /// Returns the durable end of the published read snapshot. - /// - /// # Panics - /// - /// Panics if an internally installed manifest violates the validated - /// active-cursor bounds. + pub(crate) fn writer_context(&self) -> Result { + self.authority + .as_ref() + .ok_or(StreamError::WriteStalled)? + .capture() + } + + /// Stop admission, drain and checkpoint before durably releasing this owner. + /// # Errors + /// Returns a fencing, checkpoint or release error; an error is not a release proof. + pub async fn release_writer(&self) -> Result<()> { + if self + .authority + .as_ref() + .is_some_and(|authority| authority.is_released()) + { + return Ok(()); + } + if self.authority.is_none() { + return Err(StreamError::WriteStalled); + } + self.closed.store(true, Ordering::Release); + let (completion, result) = oneshot::channel(); + self.sender + .send(Command::Release(completion)) + .await + .map_err(|_| StreamError::WriteStalled)?; + result.await.map_err(|_| StreamError::WriteStalled)? + } + + /// Returns the synchronized logical end visible to this handle. #[must_use] pub fn tail(&self) -> u64 { - durable_tail(&self.manifest.load()).expect("validated stream manifest has a durable tail") + self.read_view.load().end() } - /// Appends one logical request after its mirror data and cursor are durable. + /// Appends one logical request after every required mirror is synchronized. /// /// # Errors /// @@ -545,6 +427,7 @@ impl ChunkStream { if self.closed.load(Ordering::Acquire) { return Err(StreamError::WriteStalled); } + let authority = self.writer_context()?; let length = buffers .iter() .try_fold(0_usize, |total, bytes| total.checked_add(bytes.len())) @@ -572,15 +455,13 @@ impl ChunkStream { } self.record_queue_high_watermarks(); self.metrics.submitted.fetch_add(1, Ordering::Relaxed); - let mut data = BytesMut::with_capacity(length); - for bytes in buffers { - data.extend_from_slice(bytes); - } + let data = owned_payload(buffers, length); let (completion, result) = oneshot::channel(); if self .sender .try_send(Command::Append(AppendRequest { - data: data.freeze(), + data, + authority, enqueued_at: Instant::now(), completion, })) @@ -617,11 +498,8 @@ impl ChunkStream { "chunk-bound append exceeds maximum chunk capacity".into(), )); } - let mut data = BytesMut::with_capacity(length); - for bytes in buffers { - data.extend_from_slice(bytes); - } - self.enqueue_append(data.freeze()).await + let data = owned_payload(buffers, length); + self.enqueue_append(data).await } /// Enqueues several chunk-bound records in input order and returns their @@ -652,6 +530,7 @@ impl ChunkStream { if self.closed.load(Ordering::Acquire) { return Err(StreamError::WriteStalled); } + let authority = self.writer_context()?; let length = data.len(); if length == 0 || length > self.config.max_append_bytes { return Err(StreamError::InvalidRequest("append length is invalid".into())); @@ -668,6 +547,7 @@ impl ChunkStream { .sender .try_send(Command::Append(AppendRequest { data, + authority, enqueued_at: Instant::now(), completion, })) @@ -699,30 +579,45 @@ impl ChunkStream { /// Returns the immutable metadata generation currently backing reads. #[must_use] - pub fn manifest_generation(&self) -> u64 { - self.manifest.load().generation + pub fn record_generation(&self) -> u64 { + self.state.load().generation } - /// Reclaims immutable extent pages older than a caller-proven retained - /// manifest generation. + /// Reclaims Stream chunk records older than a caller-proven retained + /// record generation. /// /// # Errors /// /// Returns an error if the watermark could remove the current generation, /// the pass bound is zero, or metadata deletion fails. - pub async fn reclaim_metadata_before(&self, retained_generation: u64, max_pages: usize) -> Result { - let current = self.manifest.load().generation; - if retained_generation == 0 || retained_generation > current || max_pages == 0 { + pub async fn reclaim_metadata_before(&self, retained_generation: u64, max_records: usize) -> Result { + let current = self.state.load().generation; + if retained_generation == 0 || retained_generation > current || max_records == 0 { return Err(StreamError::InvalidRequest( "metadata reclaim watermark is outside retained generations".into(), )); } - self.metadata - .reclaim_extent_pages_before(self.stream_name, retained_generation, max_pages) + if self + .authority + .as_ref() + .is_some_and(|authority| authority.is_released()) + { + return Ok(0); + } + self.writer_context()?; + let (completion, result) = oneshot::channel(); + self.sender + .send(Command::Reclaim { + retained_generation, + max_records, + completion, + }) .await + .map_err(|_| StreamError::WriteStalled)?; + result.await.map_err(|_| StreamError::WriteStalled)? } - /// Seals the active chunk and publishes a closed manifest. + /// Seals the active chunk and publishes a closed state. /// /// # Errors /// @@ -740,181 +635,6 @@ impl ChunkStream { .await .map_err(|_| StreamError::Internal("close worker stopped".into()))? } - - /// Reads an exact retained durable logical range. - /// - /// # Errors - /// - /// Returns an error for an invalid range, corrupt metadata, or unavailable data. - pub async fn read_at(&self, offset: u64, length: usize) -> Result { - let segments = self.read_at_with_provenance(offset, length).await?; - let mut output = BytesMut::with_capacity(length); - for segment in segments { - output.extend_from_slice(&segment.data); - } - Ok(output.freeze()) - } - - /// Reads an exact range while retaining the physical chunk identity for - /// every returned logical segment. - /// - /// # Errors - /// - /// Returns an error for an invalid range, corrupt metadata, or unavailable data. - pub async fn read_at_with_provenance(&self, offset: u64, length: usize) -> Result> { - let end = offset - .checked_add(length as u64) - .ok_or_else(|| StreamError::InvalidRequest("read range overflows".into()))?; - let manifest = self.manifest.load_full(); - if offset < manifest.trim_offset || end > durable_tail(&manifest)? { - return Err(StreamError::InvalidRequest( - "read is outside retained durable range".into(), - )); - } - let mut cursor = offset; - let mut reads = Vec::new(); - while cursor < end { - let fence_index = find_extent_fence(&manifest, cursor)?; - let fence = &manifest.extent_pages[fence_index]; - let page = self.load_extent_page(&manifest, fence).await?; - let location = resolve_extent(&page, cursor)?; - let read_len = usize::try_from(location.available.min(end - cursor)) - .map_err(|_| StreamError::InvalidRequest("read length exceeds addressable range".into()))?; - let read_len = read_len.min(self.config.read_request_bytes); - reads.push(PhysicalRead { - chunk_id: location.chunk_id, - logical_start: cursor, - frame_start: location.frame_offset, - frame_length: usize::try_from(location.frame_length).map_err(|_| { - StreamError::Corruption("stream frame length exceeds address space".into()) - })?, - payload_start: usize::try_from(location.payload_offset).map_err(|_| { - StreamError::Corruption("stream frame payload offset exceeds address space".into()) - })?, - length: read_len, - }); - cursor += read_len as u64; - } - let output = self.read_physical_ranges(&reads).await?; - let output_len = output - .iter() - .fold(0_usize, |total, segment| total.saturating_add(segment.data.len())); - if output_len != length { - return Err(StreamError::Corruption( - "chunk reads returned an unexpected length".into(), - )); - } - self.metrics - .read_bytes - .fetch_add(length as u64, Ordering::Relaxed); - Ok(output) - } - - async fn read_physical_ranges(&self, reads: &[PhysicalRead]) -> Result> { - self.metrics - .physical_read_requests - .fetch_add(reads.len() as u64, Ordering::Relaxed); - let mut output = vec![None; reads.len()]; - let mut tasks = tokio::task::JoinSet::new(); - let mut next = 0_usize; - while next < reads.len() || !tasks.is_empty() { - while next < reads.len() && tasks.len() < self.config.read_concurrency { - let index = next; - let read = reads[index]; - let stream = self.clone(); - tasks.spawn(async move { - let frame = stream - .read_chunk_with_watchdog( - read.chunk_id, - read.frame_start, - read.frame_length, - read.logical_start, - ) - .await?; - let parsed = parse_frame(&frame, read.chunk_id) - .map_err(|error| StreamError::Corruption(format!("invalid stream frame: {error}")))?; - if parsed.header.magic != FrameMagic::StreamV1 { - return Err(StreamError::Corruption( - "stream extent has the wrong frame kind".into(), - )); - } - let payload_end = read.payload_start.checked_add(read.length).ok_or_else(|| { - StreamError::Corruption("stream frame payload range overflows".into()) - })?; - let data = Bytes::copy_from_slice( - parsed - .payload - .get( - read.payload_start - FRAME_HEADER_PREFIX_BYTES - ..payload_end - FRAME_HEADER_PREFIX_BYTES, - ) - .ok_or_else(|| { - StreamError::Corruption("stream extent exceeds frame payload".into()) - })?, - ); - Ok::<_, StreamError>(( - index, - ReadSegment { - chunk_id: read.chunk_id, - logical_start: read.logical_start, - data, - }, - )) - }); - next += 1; - } - let completed = tasks - .join_next() - .await - .ok_or_else(|| StreamError::Internal("physical read task set ended early".into()))? - .map_err(|error| StreamError::Internal(format!("physical read task failed: {error}")))??; - output[completed.0] = Some(completed.1); - } - output - .into_iter() - .map(|segment| { - segment.ok_or_else(|| StreamError::Internal("physical read result is missing".into())) - }) - .collect() - } - - /// Creates a bounded sequential reader ending at the current durable tail. - /// - /// # Errors - /// - /// Returns an error if `offset` is outside the retained durable range. - pub fn read_from(&self, offset: u64) -> Result { - self.reader(offset, ReadHint::ToEnd) - } - - /// Creates a seekable reader with a finite or captured-tail hint. - /// - /// # Errors - /// - /// Returns an error if `offset` or its hint is outside the retained range. - pub fn reader(&self, offset: u64, hint: ReadHint) -> Result { - let manifest = self.manifest.load(); - let tail = durable_tail(&manifest)?; - if offset < manifest.trim_offset || offset > tail { - return Err(StreamError::InvalidRequest( - "read cursor is outside retained range".into(), - )); - } - let end = match hint { - ReadHint::ToEnd => tail, - ReadHint::Bytes(length) => offset - .checked_add(length) - .ok_or_else(|| StreamError::InvalidRequest("reader hint overflows".into()))? - .min(tail), - }; - Ok(StreamReader { - stream: self.clone(), - offset, - end, - pending: None, - }) - } - #[must_use] pub fn writer_epoch(&self) -> u64 { self.writer_epoch @@ -934,927 +654,11 @@ impl ChunkStream { .max_queued_bytes .fetch_max(self.queued_bytes.load(Ordering::Acquire), Ordering::Relaxed); } - - async fn load_extent_page( - &self, - manifest: &StreamManifest, - fence: &StreamExtentPageFence, - ) -> Result> { - if let Some(page) = - self.extent_cache - .get(manifest.writer_epoch, manifest.generation, fence.page_index) - { - self.metrics - .extent_page_cache_hits - .fetch_add(1, Ordering::Relaxed); - return Ok(page); - } - self.metrics - .extent_page_cache_misses - .fetch_add(1, Ordering::Relaxed); - let page = self - .metadata - .load_extent_page( - manifest.stream_name, - manifest.writer_epoch, - manifest.generation, - fence.page_index, - ) - .await? - .ok_or_else(|| StreamError::Corruption("referenced extent page is missing".into()))?; - validate_page_fence(manifest, fence, &page)?; - Ok(self.extent_cache.insert(page)) - } - - async fn read_chunk_with_watchdog( - &self, - chunk_id: ChunkId, - physical_offset: u64, - length: usize, - logical_offset: u64, - ) -> Result { - let read = self.chunks.read_verified_frame(chunk_id, physical_offset, length); - tokio::pin!(read); - let started = Instant::now(); - loop { - tokio::select! { - result = &mut read => return result, - () = tokio::time::sleep(self.config.watchdog_interval) => { - self.metrics.watchdog_observations.fetch_add(1, Ordering::Relaxed); - tracing::warn!( - stream_high = self.stream_name.high, - stream_low = self.stream_name.low, - writer_epoch = self.writer_epoch, - logical_offset, - read_bytes = length, - operation_age_ms = started.elapsed().as_millis(), - stage = "chunk_read", - "chunk-stream read window remains in flight" - ); - } - } - } - } -} - -type PrefetchResult = tokio::task::JoinHandle)>>; - -pub struct StreamReader { - stream: ChunkStream, - offset: u64, - end: u64, - pending: Option, -} - -impl StreamReader { - /// Moves the reader within its captured retained range. - /// - /// # Errors - /// - /// Returns an error when `offset` is outside the captured range. - pub fn seek(&mut self, offset: u64) -> Result<()> { - let trim = self.stream.manifest.load().trim_offset; - if offset < trim || offset > self.end { - return Err(StreamError::InvalidRequest( - "seek is outside the reader range".into(), - )); - } - if let Some(pending) = self.pending.take() { - pending.abort(); - } - self.offset = offset; - Ok(()) - } - - /// Returns the next bounded logical window, or `None` at the captured tail. - /// - /// # Errors - /// - /// Returns a typed metadata or chunk-read error. - pub async fn next(&mut self) -> Result> { - let Some(segments) = self.next_with_provenance().await? else { - return Ok(None); - }; - let length = segments.iter().map(|segment| segment.data.len()).sum(); - let mut bytes = BytesMut::with_capacity(length); - for segment in segments { - bytes.extend_from_slice(&segment.data); - } - Ok(Some(bytes.freeze())) - } - - /// Returns the next bounded logical window split by physical chunk. - /// - /// # Errors - /// - /// Returns a typed metadata, task, or chunk-read error. - pub async fn next_with_provenance(&mut self) -> Result>> { - if self.offset == self.end { - return Ok(None); - } - self.start_prefetch()?; - let pending = self - .pending - .take() - .ok_or_else(|| StreamError::Internal("reader prefetch was not installed".into()))?; - let (offset, segments) = pending - .await - .map_err(|error| StreamError::Internal(format!("reader prefetch task failed: {error}")))??; - if offset != self.offset { - return Err(StreamError::Internal( - "reader prefetch completed for a stale offset".into(), - )); - } - let length = segments.iter().try_fold(0_u64, |length, segment| { - length.checked_add(segment.data.len() as u64) - }); - self.offset = self - .offset - .checked_add(length.ok_or_else(|| StreamError::Corruption("reader length overflows".into()))?) - .ok_or_else(|| StreamError::Corruption("reader offset overflows".into()))?; - self.start_prefetch()?; - Ok(Some(segments)) - } - - fn start_prefetch(&mut self) -> Result<()> { - if self.pending.is_some() || self.offset == self.end { - return Ok(()); - } - let offset = self.offset; - let length = usize::try_from((self.end - offset).min(self.stream.config.read_window_bytes as u64)) - .map_err(|_| StreamError::InvalidRequest("read window exceeds addressable range".into()))?; - let stream = self.stream.clone(); - self.pending = Some(tokio::spawn(async move { - stream - .read_at_with_provenance(offset, length) - .await - .map(|segments| (offset, segments)) - })); - Ok(()) - } -} - -async fn run_worker(mut state: WorkerState, mut receiver: mpsc::Receiver) { - let mut pending = None; - let mut liveness = tokio::time::interval(state.config.liveness_interval); - liveness.tick().await; - loop { - let command = if let Some(command) = pending.take() { - command - } else { - tokio::select! { - command = receiver.recv() => match command { - Some(command) => command, - None => break, - }, - _ = liveness.tick() => { - if let Some(chunk_id) = state.manifest.active.as_ref().map(|active| active.chunk_id) { - liveness::maintain(&mut state, chunk_id).await; - } - continue; - } - } - }; - match command { - Command::Append(first) => { - process_append_batch(&mut state, first, &mut receiver, &mut pending).await; - } - Command::Trim { offset, completion } => { - let _ = completion.send(process_trim(&mut state, offset).await); - } - Command::Close(completion) => { - let result = process_close(&mut state).await; - let _ = completion.send(result); - break; - } - } - } - state.closed_view.store(true, Ordering::Release); -} - -async fn process_append_batch( - state: &mut WorkerState, - first: AppendRequest, - receiver: &mut mpsc::Receiver, - pending: &mut Option, -) { - if state.stalled || state.manifest.closed { - finish_failed(state, vec![first], &StreamError::WriteStalled); - return; - } - let first_len = match first.physical_len() { - Ok(length) => length, - Err(error) => { - finish_failed(state, vec![first], &error); - return; - } - }; - if let Err(error) = prepare_active_for_append(state, first_len).await { - state.stalled = true; - finish_failed(state, vec![first], &error); - return; - } - - let mut requests = vec![first]; - let mut bytes = first_len; - while requests.len() < state.config.batch_requests && bytes < state.config.batch_bytes { - match receiver.try_recv() { - Ok(Command::Append(request)) - if bytes - .checked_add(request.physical_len().unwrap_or(usize::MAX)) - .is_some_and(|total| total <= state.config.batch_bytes && fits_active(state, total)) => - { - bytes += request.physical_len().expect("checked above"); - requests.push(request); - } - Ok(command) => { - *pending = Some(command); - break; - } - Err(_) => break, - } - } - let result = write_with_recovery(state, &requests, bytes).await; - match result { - Ok(ranges) => { - for (request, range) in requests.into_iter().zip(ranges) { - state - .queued_bytes - .fetch_sub(request.logical_len() as u64, Ordering::AcqRel); - state.queued_requests.fetch_sub(1, Ordering::AcqRel); - state.metrics.completed.fetch_add(1, Ordering::Relaxed); - state - .metrics - .logical_append_bytes - .fetch_add(request.logical_len() as u64, Ordering::Relaxed); - let _ = request.completion.send(Ok(range)); - } - } - Err(BatchFailure::MirrorWrite(error) | BatchFailure::Other(error)) => { - state.stalled = true; - finish_failed(state, requests, &error); - } - } -} - -async fn prepare_active_for_append(state: &mut WorkerState, first_len: usize) -> Result<()> { - loop { - if state.manifest.active.is_none() { - match ensure_active(state).await { - Ok(()) => {} - Err( - error @ (StreamError::StaleWriter - | StreamError::Corruption(_) - | StreamError::InvalidRequest(_)), - ) => { - return Err(error); - } - Err(error) => { - tracing::warn!(%error, "chunk-stream allocation remains unavailable"); - tokio::time::sleep(Duration::from_millis(100)).await; - continue; - } - } - } - if fits_active(state, first_len) { - return Ok(()); - } - match grow_or_rollover(state, first_len).await { - Ok(()) if fits_active(state, first_len) => return Ok(()), - Ok(()) => return Err(StreamError::Corruption("new chunk cannot fit the append".into())), - Err( - error @ (StreamError::StaleWriter - | StreamError::Corruption(_) - | StreamError::InvalidRequest(_)), - ) => { - return Err(error); - } - Err(error) => { - tracing::warn!(%error, "chunk-stream growth remains unavailable"); - match rollover(state).await { - Ok(()) if fits_active(state, first_len) => return Ok(()), - Err(StreamError::StaleWriter) => return Err(StreamError::StaleWriter), - _ => tokio::time::sleep(Duration::from_millis(100)).await, - } - } - } - } -} - -async fn write_with_recovery( - state: &mut WorkerState, - requests: &[AppendRequest], - bytes: usize, -) -> std::result::Result, BatchFailure> { - let mut rotated = false; - loop { - match write_batch_with_watchdog(state, requests, bytes).await { - Err(BatchFailure::MirrorWrite( - error @ (StreamError::StaleWriter - | StreamError::Corruption(_) - | StreamError::InvalidRequest(_)), - )) => return Err(BatchFailure::Other(error)), - Err( - BatchFailure::MirrorWrite(error) - | BatchFailure::Other(error @ StreamError::DefinitelyNotCommitted(_)), - ) => { - if rotated { - return Err(BatchFailure::Other(error)); - } - tracing::warn!( - stream_high = state.stream_name.high, - stream_low = state.stream_name.low, - writer_epoch = state.writer_epoch, - %error, - "chunk-stream append will rotate after an uncommitted write" - ); - rollover(state).await.map_err(BatchFailure::Other)?; - rotated = true; - } - Err(BatchFailure::Other(error)) => { - if matches!(rotate_externally_sealed_active(state).await, Ok(true)) { - continue; - } - return Err(BatchFailure::Other(error)); - } - success => return success, - } - } -} - -async fn write_batch_with_watchdog( - state: &mut WorkerState, - requests: &[AppendRequest], - bytes: usize, -) -> std::result::Result, BatchFailure> { - let interval = state.config.watchdog_interval; - let metrics = Arc::clone(&state.metrics); - let stream_name = state.stream_name; - let writer_epoch = state.writer_epoch; - let logical_begin = state.tail_view.load(Ordering::Acquire); - let logical_end = logical_begin.saturating_add(bytes as u64); - let oldest_queue_age = requests - .first() - .map_or(Duration::ZERO, |request| request.enqueued_at.elapsed()); - metrics.batches.fetch_add(1, Ordering::Relaxed); - metrics - .batch_requests - .fetch_add(requests.len() as u64, Ordering::Relaxed); - let write = write_batch(state, requests, bytes); - tokio::pin!(write); - let started = Instant::now(); - loop { - tokio::select! { - result = &mut write => return result, - () = tokio::time::sleep(interval) => { - metrics.watchdog_observations.fetch_add(1, Ordering::Relaxed); - tracing::warn!( - stream_high = stream_name.high, - stream_low = stream_name.low, - writer_epoch, - logical_begin, - logical_end, - batch_requests = requests.len(), - batch_bytes = bytes, - operation_age_ms = started.elapsed().as_millis(), - oldest_queue_age_ms = oldest_queue_age.as_millis(), - stage = "append_durability", - "chunk-stream append batch remains in flight" - ); - } - } - } -} - -async fn write_batch( - state: &mut WorkerState, - requests: &[AppendRequest], - bytes: usize, -) -> std::result::Result, BatchFailure> { - let active = state - .manifest - .active - .as_ref() - .ok_or_else(|| BatchFailure::Other(StreamError::Internal("append has no active chunk".into())))? - .clone(); - let expected_cursor = active.acknowledged_cursor; - let mut new_cursor = expected_cursor; - let mut staging = BytesMut::with_capacity(bytes); - let logical_begin = state.tail_view.load(Ordering::Acquire); - let mut logical_cursor = logical_begin; - let mut extents = Vec::new(); - let mut ranges = Vec::with_capacity(requests.len()); - for request in requests { - let frames = encode_frames( - FrameMagic::StreamV1, - active.chunk_id, - &request.data, - unix_time_ms(), - ) - .map_err(|error| BatchFailure::Other(StreamError::InvalidRequest(error.to_string())))?; - for frame in frames { - let frame_length = u32::try_from(frame.len()).map_err(|_| { - BatchFailure::Other(StreamError::InvalidRequest("stream frame exceeds u32".into())) - })?; - let payload_length = u64::from(frame_length) - .checked_sub((FRAME_HEADER_PREFIX_BYTES + crowdb_protocol::frame::FRAME_FOOTER_BYTES) as u64) - .ok_or_else(|| { - BatchFailure::Other(StreamError::Corruption("stream frame length underflows".into())) - })?; - extents.push(Extent { - chunk_id: active.chunk_id, - logical_start: logical_cursor, - logical_end: logical_cursor.saturating_add(payload_length), - frame_start: new_cursor, - frame_length, - }); - logical_cursor = logical_cursor.saturating_add(payload_length); - new_cursor = new_cursor.saturating_add(u64::from(frame_length)); - staging.extend_from_slice(&frame); - } - ranges.push(AppendRange { - stream_name: state.stream_name, - chunk_id: Some(active.chunk_id), - begin: logical_cursor.saturating_sub(request.logical_len() as u64), - end: logical_cursor, - }); - } - let staging = staging.freeze(); - let images = state - .mirror_shadow - .stage(active.chunk_id, expected_cursor, &staging) - .map_err(BatchFailure::Other)?; - let write = state - .chunks - .write_mirrors_with_images( - state.stream_name, - state.writer_epoch, - active.chunk_id, - expected_cursor, - staging.clone(), - &images, - ) - .await; - drop(images); - state.mirror_shadow.finish(); - if let Err(error) = write { - return Err(BatchFailure::MirrorWrite(error)); - } - state - .metrics - .physical_append_bytes - .fetch_add((bytes as u64).saturating_mul(3), Ordering::Relaxed); - resolve_cursor_advance( - state, - active.chunk_id, - expected_cursor, - new_cursor, - crc32fast::hash(&staging), - ) - .await - .map_err(BatchFailure::Other)?; - - state.extents.extend(extents); - if let Some(active) = &mut state.manifest.active { - active.acknowledged_cursor = new_cursor; - active.physical_start = new_cursor; - active.logical_start = logical_cursor; - } - state.manifest.sealed_tail = logical_cursor; - state.tail_view.store(logical_cursor, Ordering::Release); - publish_state(state).await.map_err(BatchFailure::Other)?; - Ok(ranges) -} - -async fn resolve_cursor_advance( - state: &mut WorkerState, - chunk_id: ChunkId, - expected_cursor: u64, - new_cursor: u64, - checksum: u32, -) -> Result<()> { - let outcome = state - .chunks - .advance_cursor( - state.stream_name, - state.writer_epoch, - chunk_id, - expected_cursor, - new_cursor, - checksum, - ) - .await; - match outcome { - Ok(CursorAdvance::Committed) => Ok(()), - Ok(CursorAdvance::DefinitelyNotCommitted) => Err(StreamError::DefinitelyNotCommitted( - "durable cursor did not advance".into(), - )), - Ok(CursorAdvance::Ambiguous) | Err(_) => loop { - match state.chunks.durable_cursor(chunk_id, state.writer_epoch).await { - Ok(durable) if durable.offset == expected_cursor => { - return Err(StreamError::DefinitelyNotCommitted( - "ambiguous write proved absent".into(), - )); - } - Ok(durable) - if durable.offset == new_cursor && durable.last_advance_checksum == Some(checksum) => - { - return Ok(()); - } - Ok(durable) if durable.offset == new_cursor => { - return Err(StreamError::Corruption( - "committed cursor has an unexpected checksum".into(), - )); - } - Ok(durable) => { - return Err(StreamError::Corruption(format!( - "durable cursor is outside the append bounds: chunk={chunk_id:?} epoch={} expected={expected_cursor} new={new_cursor} durable={}", - state.writer_epoch, durable.offset - ))); - } - Err(StreamError::StaleWriter) => return Err(StreamError::StaleWriter), - Err(error) => { - tracing::warn!(%error, "chunk-stream cursor resolution remains unavailable"); - tokio::time::sleep(Duration::from_millis(100)).await; - } - } - }, - } -} - -async fn ensure_active(state: &mut WorkerState) -> Result<()> { - if state.manifest.active.is_some() { - return Ok(()); - } - let mut active = state - .chunks - .allocate_mirrored(state.stream_name, state.writer_epoch, state.manifest.purpose) - .await?; - active.logical_start = state.tail_view.load(Ordering::Acquire); - if active.acknowledged_cursor < active.physical_start || active.acknowledged_cursor > active.capacity { - return Err(StreamError::Corruption( - "allocated chunk cursor is invalid".into(), - )); - } - state.manifest.active = Some(active); - publish_state(state).await -} - -async fn rollover(state: &mut WorkerState) -> Result<()> { - let active = state - .manifest - .active - .as_ref() - .ok_or_else(|| StreamError::Internal("rollover has no active chunk".into()))?; - let active = active.clone(); - let durable = state - .chunks - .durable_cursor(active.chunk_id, state.writer_epoch) - .await?; - if durable.offset != active.acknowledged_cursor { - return Err(StreamError::Corruption( - "rollover cursor differs from the acknowledged cursor".into(), - )); - } - if !durable.sealed { - state - .chunks - .seal(active.chunk_id, state.writer_epoch, active.acknowledged_cursor) - .await?; - } - let tail = state.tail_view.load(Ordering::Acquire); - state.manifest.sealed_tail = tail; - let mut successor = state - .chunks - .allocate_mirrored(state.stream_name, state.writer_epoch, state.manifest.purpose) - .await?; - successor.logical_start = tail; - state.manifest.active = Some(successor); - publish_state(state).await?; - state.metrics.rollovers.fetch_add(1, Ordering::Relaxed); - Ok(()) -} - -async fn grow_or_rollover(state: &mut WorkerState, bytes: usize) -> Result<()> { - let active = state - .manifest - .active - .as_ref() - .ok_or_else(|| StreamError::Internal("active chunk is missing".into()))?; - let required_capacity = active - .acknowledged_cursor - .checked_add( - u64::try_from(bytes) - .map_err(|_| StreamError::InvalidRequest("append length exceeds addressable range".into()))?, - ) - .ok_or_else(|| StreamError::InvalidRequest("append cursor overflows".into()))?; - if let Some(mut grown) = state - .chunks - .grow_mirrored( - state.stream_name, - state.writer_epoch, - active.chunk_id, - required_capacity, - ) - .await? - { - grown.physical_start = active.physical_start; - grown.logical_start = active.logical_start; - grown.acknowledged_cursor = active.acknowledged_cursor; - if grown.capacity < required_capacity { - return Err(StreamError::Corruption( - "grown chunk capacity does not cover the requested append".into(), - )); - } - state.manifest.active = Some(grown); - publish_state(state).await?; - return Ok(()); - } - rollover(state).await -} - -async fn rotate_externally_sealed_active(state: &mut WorkerState) -> Result { - let Some(active) = state.manifest.active.clone() else { - return Ok(false); - }; - let durable = state - .chunks - .durable_cursor(active.chunk_id, state.writer_epoch) - .await?; - if !durable.sealed { - return Ok(false); - } - if durable.offset != active.acknowledged_cursor { - return Err(StreamError::Corruption( - "externally sealed active chunk has an unexpected durable cursor".into(), - )); - } - - let tail = state.tail_view.load(Ordering::Acquire); - state.manifest.active = None; - state.manifest.sealed_tail = tail; - let mut successor = state - .chunks - .allocate_mirrored(state.stream_name, state.writer_epoch, state.manifest.purpose) - .await?; - successor.logical_start = tail; - state.manifest.active = Some(successor); - publish_state(state).await?; - state.stalled = false; - state.metrics.rollovers.fetch_add(1, Ordering::Relaxed); - Ok(true) -} - -async fn process_trim(state: &mut WorkerState, offset: u64) -> Result { - if state.stalled { - return Err(StreamError::WriteStalled); - } - let tail = state.tail_view.load(Ordering::Acquire); - if offset < state.manifest.trim_offset || offset > tail { - return Err(StreamError::InvalidRequest( - "trim offset regresses or exceeds durable tail".into(), - )); - } - if offset != state.manifest.trim_offset { - state.manifest.trim_offset = offset; - publish_state(state).await?; - } - let mut reclaimed = 0_u64; - let mut removed = 0_usize; - for extent in &state.extents { - if extent.logical_end > offset || (removed > 0 && reclaimed >= state.config.gc_bytes_per_pass) { - break; - } - reclaimed = reclaimed.saturating_add( - state - .chunks - .release_trimmed(extent.chunk_id, extent.logical_end) - .await? - .reclaimed_bytes, - ); - removed += 1; - } - if removed > 0 { - state.extents.drain(..removed); - publish_state(state).await?; - } - state - .metrics - .reclaimed_bytes - .fetch_add(reclaimed, Ordering::Relaxed); - Ok(reclaimed) -} - -async fn process_close(state: &mut WorkerState) -> Result<()> { - if state.stalled { - return Err(StreamError::WriteStalled); - } - if state.manifest.closed { - return Ok(()); - } - if let Some(active) = state.manifest.active.take() { - state - .chunks - .seal(active.chunk_id, state.writer_epoch, active.acknowledged_cursor) - .await?; - let tail = state.tail_view.load(Ordering::Acquire); - state.manifest.sealed_tail = tail; - } - state.manifest.closed = true; - publish_state(state).await?; - state.closed_view.store(true, Ordering::Release); - Ok(()) -} - -async fn publish_state(state: &mut WorkerState) -> Result<()> { - let expected = state.manifest.generation; - state.manifest.previous_generation = Some(expected); - state.manifest.generation = expected + 1; - let pages = build_extent_pages( - &state.extents, - state.stream_name, - state.writer_epoch, - state.manifest.generation, - state.config.extent_page_entries, - ); - state.manifest.extent_pages = fences_for(&pages); - loop { - match state - .metadata - .publish( - Some((state.writer_epoch, expected)), - state.manifest.clone(), - pages.clone(), - ) - .await - { - Ok(()) => break, - Err(error) => { - match state.metadata.load_current(state.stream_name).await { - Ok(Some(current)) if current == state.manifest => break, - Ok(Some(current)) - if current.writer_epoch == state.writer_epoch && current.generation == expected => - { - if matches!(error, StreamError::InvalidRequest(_) | StreamError::Corruption(_)) { - state.stalled = true; - return Err(error); - } - } - Ok(Some(current)) if current.writer_epoch > state.writer_epoch => { - state.stalled = true; - return Err(StreamError::StaleWriter); - } - Ok(_) => { - state.stalled = true; - return Err(StreamError::Corruption( - "stream manifest head differs from both prior and candidate state".into(), - )); - } - Err(load_error) => { - tracing::warn!(%load_error, "chunk-stream manifest publication remains unresolved"); - } - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - } - } - state - .metrics - .metadata_publications - .fetch_add(1, Ordering::Relaxed); - state.manifest_view.store(Arc::new(state.manifest.clone())); - Ok(()) -} - -fn fences_for(pages: &[StreamExtentPage]) -> Vec { - pages - .iter() - .map(|page| StreamExtentPageFence { - page_index: page.page_index, - first_logical: page.logical_offsets[0], - end_logical: *page - .logical_offsets - .last() - .expect("built extent page is nonempty"), - }) - .collect() -} - -fn build_extent_pages( - extents: &[Extent], - stream_name: StreamName, - writer_epoch: u64, - generation: u64, - page_entries: usize, -) -> Vec { - extents - .chunks(page_entries) - .map(|chunk| { - let mut logical_offsets = Vec::with_capacity(chunk.len() + 1); - logical_offsets.push(chunk[0].logical_start); - for extent in chunk { - logical_offsets.push(extent.logical_end); - } - StreamExtentPage { - stream_name, - writer_epoch, - generation, - page_index: chunk[0].logical_start, - chunk_ids: chunk.iter().map(|extent| extent.chunk_id).collect(), - logical_offsets, - physical_offsets: chunk.iter().map(|extent| extent.frame_start).collect(), - frame_lengths: chunk.iter().map(|extent| extent.frame_length).collect(), - } - }) - .collect() -} - -async fn load_extent_pages( - metadata: &dyn StreamMetadataStore, - manifest: &StreamManifest, -) -> Result> { - let mut pages = Vec::with_capacity(manifest.extent_pages.len()); - for fence in &manifest.extent_pages { - pages.push( - metadata - .load_extent_page( - manifest.stream_name, - manifest.writer_epoch, - manifest.generation, - fence.page_index, - ) - .await? - .ok_or_else(|| StreamError::Corruption("referenced extent page is missing".into()))?, - ); - } - Ok(pages) -} - -fn collect_extents(pages: &[StreamExtentPage]) -> Vec { - let mut extents = Vec::new(); - for page in pages { - for index in 0..page.chunk_ids.len() { - extents.push(Extent { - chunk_id: page.chunk_ids[index], - logical_start: page.logical_offsets[index], - logical_end: page.logical_offsets[index + 1], - frame_start: page.physical_offsets[index], - frame_length: page.frame_lengths[index], - }); - } - } - extents -} - -fn find_extent_fence(manifest: &StreamManifest, offset: u64) -> Result { - let index = manifest - .extent_pages - .partition_point(|fence| fence.end_logical <= offset); - let fence = manifest - .extent_pages - .get(index) - .ok_or_else(|| StreamError::Corruption(format!( - "sealed offset is not covered by extent directory: offset={offset}, sealed_tail={}, trim={}, generation={}, epoch={}, fences={}", - manifest.sealed_tail, manifest.trim_offset, manifest.generation, manifest.writer_epoch, manifest.extent_pages.len() - )))?; - if offset < fence.first_logical { - return Err(StreamError::Corruption( - "extent directory has a logical gap".into(), - )); - } - Ok(index) -} - -fn validate_page_fence( - manifest: &StreamManifest, - fence: &StreamExtentPageFence, - page: &StreamExtentPage, -) -> Result<()> { - if page.stream_name != manifest.stream_name - || page.writer_epoch != manifest.writer_epoch - || page.generation != manifest.generation - || page.page_index != fence.page_index - || page.logical_offsets.first() != Some(&fence.first_logical) - || page.logical_offsets.last() != Some(&fence.end_logical) - { - return Err(StreamError::Corruption( - "extent page fence or identity mismatch".into(), - )); - } - Ok(()) -} - -fn fits_active(state: &WorkerState, bytes: usize) -> bool { - state.manifest.active.as_ref().is_some_and(|active| { - active - .acknowledged_cursor - .checked_add(bytes as u64) - .is_some_and(|end| end <= active.capacity) - }) } fn finish_failed(state: &WorkerState, requests: Vec, error: &StreamError) { for request in requests { + observe(&perf().append, request.enqueued_at); state .queued_bytes .fetch_sub(request.logical_len() as u64, Ordering::AcqRel); @@ -1864,6 +668,17 @@ fn finish_failed(state: &WorkerState, requests: Vec, error: &Stre } } +fn owned_payload(buffers: &[Bytes], length: usize) -> Bytes { + if let [only] = buffers { + return only.clone(); + } + let mut data = BytesMut::with_capacity(length); + for bytes in buffers { + data.extend_from_slice(bytes); + } + data.freeze() +} + fn reserve_bytes(counter: &AtomicU64, limit: u64, bytes: u64) -> Result<()> { let mut current = counter.load(Ordering::Acquire); loop { @@ -1893,30 +708,6 @@ fn reserve_requests(counter: &AtomicUsize, limit: usize) -> Result<()> { } } -fn durable_tail(manifest: &StreamManifest) -> Result { - match &manifest.active { - Some(active) => { - let durable = active - .acknowledged_cursor - .checked_sub(active.physical_start) - .ok_or_else(|| StreamError::Corruption("active cursor precedes its physical start".into()))?; - active - .logical_start - .checked_add(durable) - .ok_or_else(|| StreamError::Corruption("active tail overflows".into())) - } - None => Ok(manifest.sealed_tail), - } -} - -fn state_name(manifest: &ArcSwap) -> StreamName { - manifest.load().stream_name -} - -fn state_epoch(manifest: &ArcSwap) -> u64 { - manifest.load().writer_epoch -} - fn unix_time_ms() -> u64 { SystemTime::now() .duration_since(UNIX_EPOCH) diff --git a/lib/crowdb-chunk-stream/src/stream/append.rs b/lib/crowdb-chunk-stream/src/stream/append.rs new file mode 100644 index 000000000..e58b23dfc --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/append.rs @@ -0,0 +1,183 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Ordered frame assembly, mirror synchronization and local progress publication. + +use super::capacity::rollover; +use super::worker::rotate_externally_sealed_active; +use super::{AppendRange, AppendRequest, BatchFailure, WorkerState}; +use crate::perf::{perf, PhaseTimer}; +use crate::{StreamError, StreamMirrorWrite}; +use std::sync::{atomic::Ordering, Arc}; +use std::time::{Duration, Instant}; + +mod encoding; + +pub(super) async fn write_with_recovery( + state: &mut WorkerState, + requests: &[AppendRequest], + bytes: usize, +) -> std::result::Result, BatchFailure> { + let mut rotated = false; + loop { + match write_batch_with_watchdog(state, requests, bytes).await { + Err(BatchFailure::MirrorWrite( + error @ (StreamError::StaleWriter + | StreamError::Corruption(_) + | StreamError::InvalidRequest(_)), + )) => return Err(BatchFailure::Other(error)), + Err(BatchFailure::MirrorWrite(error)) => { + if rotated { + return Err(BatchFailure::Other(error)); + } + tracing::warn!( + stream_high = state.stream_name.high, + stream_low = state.stream_name.low, + writer_epoch = state.writer_epoch, + %error, + "chunk-stream append will rotate after an uncommitted write" + ); + rollover(state).await.map_err(BatchFailure::Other)?; + rotated = true; + } + Err(BatchFailure::Other(error)) => { + if matches!(rotate_externally_sealed_active(state).await, Ok(true)) { + continue; + } + return Err(BatchFailure::Other(error)); + } + success => return success, + } + } +} + +async fn write_batch_with_watchdog( + state: &mut WorkerState, + requests: &[AppendRequest], + bytes: usize, +) -> std::result::Result, BatchFailure> { + let interval = state.config.watchdog_interval; + let metrics = Arc::clone(&state.metrics); + let stream_name = state.stream_name; + let writer_epoch = state.writer_epoch; + let logical_begin = state.tail_view.load(Ordering::Acquire); + let logical_end = logical_begin.saturating_add(bytes as u64); + let oldest_queue_age = requests + .first() + .map_or(Duration::ZERO, |request| request.enqueued_at.elapsed()); + metrics.batches.fetch_add(1, Ordering::Relaxed); + metrics + .batch_requests + .fetch_add(requests.len() as u64, Ordering::Relaxed); + let _batch_timer = PhaseTimer::start(&perf().batch); + let write = write_batch(state, requests); + tokio::pin!(write); + let started = Instant::now(); + loop { + tokio::select! { + result = &mut write => return result, + () = tokio::time::sleep(interval) => { + metrics.watchdog_observations.fetch_add(1, Ordering::Relaxed); + tracing::warn!( + stream_high = stream_name.high, + stream_low = stream_name.low, + writer_epoch, + logical_begin, + logical_end, + batch_requests = requests.len(), + batch_bytes = bytes, + operation_age_ms = started.elapsed().as_millis(), + oldest_queue_age_ms = oldest_queue_age.as_millis(), + stage = "append_durability", + "chunk-stream append batch remains in flight" + ); + } + } + } +} + +async fn write_batch( + state: &mut WorkerState, + requests: &[AppendRequest], +) -> std::result::Result, BatchFailure> { + for request in requests { + request.authority.check_io().map_err(BatchFailure::Other)?; + } + let active = state + .progress + .active + .as_ref() + .ok_or_else(|| BatchFailure::Other(StreamError::Internal("append has no active chunk".into())))? + .clone(); + let encode_timer = PhaseTimer::start(&perf().encode); + let expected_cursor = active.acknowledged_cursor; + let encoded = encoding::encode_requests( + state.stream_name, + &active, + state.tail_view.load(Ordering::Acquire), + requests, + ) + .map_err(BatchFailure::Other)?; + let new_cursor = encoded.physical_end; + let logical_cursor = encoded.logical_end; + let images = state + .mirror_shadow + .stage(active.chunk_id, expected_cursor, new_cursor, &encoded.views) + .map_err(BatchFailure::Other)?; + drop(encode_timer); + let mirror_timer = PhaseTimer::start(&perf().mirror); + let mirror_started = Instant::now(); + let write = state + .chunks + .write_mirrors( + state.stream_name, + state.writer_epoch, + active.chunk_id, + StreamMirrorWrite { + authority: requests + .iter() + .min_by_key(|request| request.authority.deadline()) + .map(|request| &request.authority), + physical_offset: expected_cursor, + complete_end: new_cursor, + images: &images, + }, + ) + .await; + let physical_bytes = images.iter().map(|image| image.data.len() as u64).sum::(); + drop(mirror_timer); + drop(images); + state.mirror_shadow.finish(); + if let Err(error) = write { + return Err(BatchFailure::MirrorWrite(error)); + } + state + .capacity_plan + .record_get(new_cursor - expected_cursor, mirror_started.elapsed()); + state.metrics.physical_append_bytes.fetch_add( + physical_bytes.saturating_mul(u64::from(state.chunks.mirror_copies())), + Ordering::Relaxed, + ); + for request in requests { + request.authority.check_ack().map_err(BatchFailure::Other)?; + } + super::checkpoint::record( + state, + active.chunk_id, + expected_cursor, + new_cursor, + &encoded.views, + ); + + let _view_timer = PhaseTimer::start(&perf().local_view); + state.read_view.load().append(&encoded.extents, logical_cursor); + state.extents.extend(encoded.extents); + if let Some(active) = &mut state.progress.active { + active.acknowledged_cursor = new_cursor; + active.physical_start = new_cursor; + active.logical_start = logical_cursor; + } + state.progress.logical_tail = logical_cursor; + state.tail_view.store(logical_cursor, Ordering::Release); + Ok(encoded.ranges) +} diff --git a/lib/crowdb-chunk-stream/src/stream/append/encoding.rs b/lib/crowdb-chunk-stream/src/stream/append/encoding.rs new file mode 100644 index 000000000..9526bd8f7 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/append/encoding.rs @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One complete frame group and compact location per append request. + +use super::super::{unix_time_ms, AppendRange, AppendRequest, Extent}; +use crate::{ActiveChunkDescriptor, Result, StreamError, StreamName}; +use bytes::Bytes; +use crowdb_protocol::frame::encode_append_group_views; + +pub(super) struct EncodedBatch { + pub views: Vec, + pub physical_end: u64, + pub logical_end: u64, + pub extents: Vec, + pub ranges: Vec, +} + +pub(super) fn encode_requests( + stream_name: StreamName, + active: &ActiveChunkDescriptor, + logical_begin: u64, + requests: &[AppendRequest], +) -> Result { + let mut physical = active.acknowledged_cursor; + let mut logical = logical_begin; + let mut views = Vec::new(); + let mut extents = Vec::with_capacity(requests.len()); + let mut ranges = Vec::with_capacity(requests.len()); + for request in requests { + let group = encode_append_group_views(active.chunk_id, physical, &request.data, unix_time_ms()) + .map_err(|error| StreamError::InvalidRequest(error.to_string()))?; + let physical_length = u32::try_from(request.physical_len()?) + .map_err(|_| StreamError::InvalidRequest("append group exceeds u32".into()))?; + let logical_end = logical + .checked_add(request.logical_len() as u64) + .ok_or_else(|| StreamError::InvalidRequest("stream logical tail overflows".into()))?; + extents.push(Extent { + chunk_id: active.chunk_id, + logical_start: logical, + logical_end, + frame_start: physical, + }); + ranges.push(AppendRange { + stream_name, + chunk_id: Some(active.chunk_id), + begin: logical, + end: logical_end, + }); + logical = logical_end; + physical = physical + .checked_add(u64::from(physical_length)) + .ok_or_else(|| StreamError::InvalidRequest("stream physical tail overflows".into()))?; + for view in group { + views.push(view); + } + } + Ok(EncodedBatch { + views, + physical_end: physical, + logical_end: logical, + extents, + ranges, + }) +} diff --git a/lib/crowdb-chunk-stream/src/stream/capacity.rs b/lib/crowdb-chunk-stream/src/stream/capacity.rs new file mode 100644 index 000000000..581d15423 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/capacity.rs @@ -0,0 +1,167 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Allocate, grow and rotate the worker-owned active chunk. + +use super::publication::publish_state; +use super::{preparation, successor}; +use super::{Duration, Ordering, Result, StreamError, WorkerState}; +use crate::perf::{perf, PhaseTimer}; + +pub(super) async fn prepare_active_for_append(state: &mut WorkerState, first_len: usize) -> Result<()> { + preparation::poll_ready(state).await?; + if fits_active(state, first_len) { + return Ok(()); + } + let _wait = PhaseTimer::start(&perf().capacity_wait); + preparation::drain(state).await?; + loop { + if state.progress.active.is_none() { + match ensure_active(state).await { + Ok(()) => {} + Err( + error @ (StreamError::StaleWriter + | StreamError::Corruption(_) + | StreamError::InvalidRequest(_)), + ) => { + return Err(error); + } + Err(error) => { + state.log_unavailable("allocate", &error); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + } + } + if fits_active(state, first_len) { + return Ok(()); + } + match grow_or_rollover(state, first_len).await { + Ok(()) if fits_active(state, first_len) => return Ok(()), + Ok(()) => return Err(StreamError::Corruption("new chunk cannot fit the append".into())), + Err( + error @ (StreamError::StaleWriter + | StreamError::Corruption(_) + | StreamError::InvalidRequest(_)), + ) => { + return Err(error); + } + Err(error) => { + state.log_unavailable("grow", &error); + match rollover(state).await { + Ok(()) if fits_active(state, first_len) => return Ok(()), + Err(StreamError::StaleWriter) => return Err(StreamError::StaleWriter), + _ => tokio::time::sleep(Duration::from_millis(100)).await, + } + } + } + } +} + +async fn ensure_active(state: &mut WorkerState) -> Result<()> { + if state.progress.active.is_some() { + return Ok(()); + } + let mut active = state + .chunks + .allocate_mirrored(state.stream_name, state.writer_epoch, state.progress.purpose) + .await?; + active.logical_start = state.tail_view.load(Ordering::Acquire); + if active.acknowledged_cursor < active.physical_start || active.acknowledged_cursor > active.capacity { + return Err(StreamError::Corruption( + "allocated chunk cursor is invalid".into(), + )); + } + state.progress.active = Some(active); + publish_state(state).await?; + successor::start(state); + Ok(()) +} + +pub(super) async fn rollover(state: &mut WorkerState) -> Result<()> { + preparation::drain(state).await?; + let active = state + .progress + .active + .as_ref() + .ok_or_else(|| StreamError::Internal("rollover has no active chunk".into()))?; + let active = active.clone(); + state + .chunks + .seal(active.chunk_id, state.writer_epoch, active.acknowledged_cursor) + .await?; + state.checkpoint = None; + let tail = state.tail_view.load(Ordering::Acquire); + state.progress.logical_tail = tail; + let mut successor = successor::take(state).await?; + successor.logical_start = tail; + state.progress.active = Some(successor); + publish_state(state).await?; + successor::start(state); + state.metrics.rollovers.fetch_add(1, Ordering::Relaxed); + Ok(()) +} + +async fn grow_or_rollover(state: &mut WorkerState, bytes: usize) -> Result<()> { + if grow_active(state, bytes).await? { + return Ok(()); + } + rollover(state).await?; + // Production allocates only the first mirror strip. The waiting append + // may need several strips even in an otherwise empty successor. + if fits_active(state, bytes) || grow_active(state, bytes).await? { + return Ok(()); + } + Err(StreamError::Corruption("new chunk cannot fit the append".into())) +} + +async fn grow_active(state: &mut WorkerState, bytes: usize) -> Result { + let active = state + .progress + .active + .as_ref() + .ok_or_else(|| StreamError::Internal("active chunk is missing".into()))?; + let required_capacity = active + .acknowledged_cursor + .checked_add( + u64::try_from(bytes) + .map_err(|_| StreamError::InvalidRequest("append length exceeds addressable range".into()))?, + ) + .ok_or_else(|| StreamError::InvalidRequest("append cursor overflows".into()))?; + if required_capacity > state.config.chunk_capacity_bytes { + return Ok(false); + } + let required_capacity = crate::storage::padded_end(required_capacity, state.config.chunk_capacity_bytes)?; + if let Some(mut grown) = state + .chunks + .grow_mirrored( + state.stream_name, + state.writer_epoch, + active.chunk_id, + required_capacity, + ) + .await? + { + grown.physical_start = active.physical_start; + grown.logical_start = active.logical_start; + grown.acknowledged_cursor = active.acknowledged_cursor; + if grown.capacity < required_capacity { + return Err(StreamError::Corruption( + "grown chunk capacity does not cover the requested append".into(), + )); + } + state.progress.active = Some(grown); + return Ok(true); + } + Ok(false) +} + +pub(super) fn fits_active(state: &WorkerState, bytes: usize) -> bool { + state.progress.active.as_ref().is_some_and(|active| { + active + .acknowledged_cursor + .checked_add(bytes as u64) + .and_then(|end| crate::storage::padded_end(end, state.config.chunk_capacity_bytes).ok()) + .is_some_and(|end| end <= active.capacity) + }) +} diff --git a/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs b/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs new file mode 100644 index 000000000..bf9ce1026 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/capacity_plan.rs @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded lookahead from synchronized write bandwidth and preparation latency. + +use std::time::Duration; + +const MIN_AHEAD: u64 = 2 * 1024 * 1024; +const INITIAL_PREPARATION: Duration = Duration::from_millis(20); + +#[derive(Default)] +pub(super) struct CapacityPlan { + bytes_per_second: u64, + preparation: Duration, +} + +impl CapacityPlan { + pub(super) fn record_get(&mut self, bytes: u64, elapsed: Duration) { + let rate = + u64::try_from(u128::from(bytes) * 1_000_000_000 / elapsed.as_nanos().max(1)).unwrap_or(u64::MAX); + // React immediately to acceleration; decay gradually after a slow write. + // Idle time, queueing and foreground capacity waits are not write samples. + self.bytes_per_second = rate.max(self.bytes_per_second.saturating_mul(7) / 8); + } + + pub(super) fn record_preparation(&mut self, elapsed: Duration) { + self.preparation = elapsed.max(self.preparation.mul_f64(0.875)); + } + + pub(super) fn target(&self, cursor: u64, capacity: u64, limit: u64, batch_bytes: u64) -> Option { + if capacity >= limit { + return None; + } + // Cover twice the observed preparation time, including allocation and every + // replica's initialization. Bound speculative capacity independently of + // stream length and leave room for the batch already being submitted. + let latency = self.preparation.max(INITIAL_PREPARATION).as_nanos(); + let ahead = u64::try_from( + u128::from(self.bytes_per_second) + .saturating_mul(latency) + .saturating_mul(2) + / 1_000_000_000, + ) + .unwrap_or(u64::MAX) + .clamp(MIN_AHEAD, limit.max(MIN_AHEAD)); + if capacity.saturating_sub(cursor) > ahead.saturating_add(batch_bytes) { + return None; + } + Some( + cursor + .saturating_add(ahead.saturating_mul(2)) + .saturating_add(batch_bytes) + .max(capacity.saturating_add(1)) + .min(limit), + ) + } +} + +#[cfg(feature = "test-util")] +#[must_use] +pub fn capacity_target_for_tests( + writes: &[(u64, Duration)], + preparation: Duration, + cursor: u64, + capacity: u64, + limit: u64, + batch_bytes: u64, +) -> Option { + let mut plan = CapacityPlan::default(); + for (bytes, elapsed) in writes { + plan.record_get(*bytes, *elapsed); + } + plan.record_preparation(preparation); + plan.target(cursor, capacity, limit, batch_bytes) +} diff --git a/lib/crowdb-chunk-stream/src/stream/checkpoint.rs b/lib/crowdb-chunk-stream/src/stream/checkpoint.rs new file mode 100644 index 000000000..54310b839 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/checkpoint.rs @@ -0,0 +1,111 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Persist one ordered prefix only when publishing a chunk record checkpoint. + +use super::WorkerState; +use crate::perf::{perf, PhaseTimer}; +use crate::{ChunkId, CursorAdvance, Result, StreamError}; +use bytes::Bytes; +use std::time::Duration; + +pub(super) struct PendingCheckpoint { + chunk_id: ChunkId, + expected: u64, + end: u64, + checksum: crc32fast::Hasher, +} + +pub(super) fn record(state: &mut WorkerState, chunk_id: ChunkId, expected: u64, end: u64, views: &[Bytes]) { + let pending = state.checkpoint.get_or_insert_with(|| PendingCheckpoint { + chunk_id, + expected, + end: expected, + checksum: crc32fast::Hasher::new(), + }); + assert_eq!((pending.chunk_id, pending.end), (chunk_id, expected)); + for view in views { + pending.checksum.update(view); + } + pending.end = end; +} + +pub(super) async fn flush(state: &mut WorkerState) -> Result<()> { + let Some(pending) = &state.checkpoint else { + return Ok(()); + }; + resolve_cursor_advance( + state, + pending.chunk_id, + pending.expected, + pending.end, + pending.checksum.clone().finalize(), + ) + .await?; + state.checkpoint = None; + Ok(()) +} + +async fn resolve_cursor_advance( + state: &mut WorkerState, + chunk_id: ChunkId, + expected_cursor: u64, + new_cursor: u64, + checksum: u32, +) -> Result<()> { + let _timer = PhaseTimer::start(&perf().cursor); + loop { + state + .authority + .as_ref() + .ok_or(StreamError::WriteStalled)? + .check()?; + let outcome = state + .chunks + .advance_cursor( + state.stream_name, + state.writer_epoch, + chunk_id, + expected_cursor, + new_cursor, + checksum, + ) + .await; + match outcome { + Ok(CursorAdvance::Committed) => return Ok(()), + Err( + error @ (StreamError::StaleWriter + | StreamError::Corruption(_) + | StreamError::InvalidRequest(_)), + ) => return Err(error), + _ => {} + } + match state.chunks.durable_cursor(chunk_id, state.writer_epoch).await { + Ok(durable) if durable.offset == expected_cursor => { + // The synchronized data write already completed. An older + // metadata checkpoint cannot prove that append absent. + } + Ok(durable) + if durable.offset == new_cursor && durable.last_advance_checksum == Some(checksum) => + { + return Ok(()); + } + Ok(durable) if durable.offset == new_cursor => { + return Err(StreamError::Corruption( + "committed cursor has an unexpected checksum".into(), + )); + } + Ok(durable) => { + return Err(StreamError::Corruption(format!( + "durable cursor is outside the append bounds: chunk={chunk_id:?} epoch={} expected={expected_cursor} new={new_cursor} durable={}", + state.writer_epoch, durable.offset + ))); + } + Err(StreamError::StaleWriter) => return Err(StreamError::StaleWriter), + Err(error) => { + tracing::warn!(%error, "chunk-stream cursor resolution remains unavailable"); + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} diff --git a/lib/crowdb-chunk-stream/src/stream/collection.rs b/lib/crowdb-chunk-stream/src/stream/collection.rs new file mode 100644 index 000000000..7344a9ef8 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/collection.rs @@ -0,0 +1,54 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded ordered-record reclamation, serialized with owner lifecycle commands. + +use super::WorkerState; +use crate::{Result, StreamError}; +use std::sync::atomic::Ordering; + +pub(super) async fn reclaim_metadata(state: &mut WorkerState, retained: u64, budget: usize) -> Result { + state + .authority + .as_ref() + .ok_or(StreamError::WriteStalled)? + .check()?; + if retained == 0 || retained > state.snapshot.record().generation || budget == 0 { + return Err(StreamError::InvalidRequest( + "Stream retention horizon or budget".into(), + )); + } + let result = state + .metadata + .collect( + state.stream_name, + retained, + state.collection_after, + budget.min(128), + state.config.gc_bytes_per_pass, + state.chunks.as_ref(), + ) + .await?; + state.collection_after = result.next; + state + .metrics + .reclaimed_bytes + .fetch_add(result.physical_bytes, Ordering::Relaxed); + Ok(result.records) +} + +pub(super) async fn reclaim_chunks(state: &mut WorkerState) -> Result { + let result = state + .metadata + .collect( + state.stream_name, + state.snapshot.record().generation, + state.collection_after, + 32, + state.config.gc_bytes_per_pass, + state.chunks.as_ref(), + ) + .await?; + state.collection_after = result.next; + Ok(result.physical_bytes) +} diff --git a/lib/crowdb-chunk-stream/src/stream/liveness.rs b/lib/crowdb-chunk-stream/src/stream/liveness.rs index 1659ddfd0..e8ea19deb 100644 --- a/lib/crowdb-chunk-stream/src/stream/liveness.rs +++ b/lib/crowdb-chunk-stream/src/stream/liveness.rs @@ -1,75 +1,100 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{rollover, ChunkId, Duration, StreamError, WorkerState}; +use super::{rollover, Duration, WorkerState}; -pub(super) async fn maintain(state: &mut WorkerState, chunk_id: ChunkId) { - if state.stalled || state.manifest.closed { - return; +pub(super) struct Renewal(tokio::task::JoinHandle<()>); +impl Drop for Renewal { + fn drop(&mut self) { + self.0.abort(); } - for attempts in 1..=3 { - if !owns_head(state, chunk_id).await { - return; - } - match state.chunks.renew_liveness(chunk_id, state.writer_epoch).await { - Ok(()) => return, - Err(error) => { - tracing::warn!(stream_high = state.stream_name.high, - stream_low = state.stream_name.low, writer_epoch = state.writer_epoch, - attempts, %error, "chunk-stream idle liveness renewal failed"); - if matches!(error, StreamError::StaleWriter) || attempts == 3 { - if owns_head(state, chunk_id).await { - rotate(state).await; - } +} + +pub(super) fn start(state: &WorkerState) -> Renewal { + let chunks = state.chunks.clone(); + let view = state.published_view.clone(); + let authority = state.authority.clone().expect("writer worker has authority"); + let failed = state.liveness_failed.clone(); + let changed = state.liveness_changed.clone(); + let closed = state.closed_view.clone(); + let interval = state.config.liveness_interval; + let epoch = state.writer_epoch; + Renewal(tokio::spawn(async move { + let mut ticks = tokio::time::interval(interval); + ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + ticks.tick().await; + loop { + ticks.tick().await; + if closed.load(std::sync::atomic::Ordering::Acquire) || authority.check().is_err() { + break; + } + let state = view.load_full(); + let active = state.snapshot.record(); + if active.state != crowdb_protocol::chunk_stream::StreamChunkState::Active { + continue; + } + for attempt in 0..3 { + if authority.check().is_err() { return; } - tokio::time::sleep(Duration::from_millis(100)).await; + match chunks.renew_liveness(active.chunk_id, epoch).await { + Ok(()) => break, + Err(error) => { + tracing::warn!(%error, attempt, "stream chunk liveness renewal failed"); + if attempt == 2 || matches!(error, crate::StreamError::StaleWriter) { + failed.store(true, std::sync::atomic::Ordering::Release); + changed.notify_one(); + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } } } - } + })) } -async fn owns_head(state: &mut WorkerState, chunk_id: ChunkId) -> bool { - match state.metadata.load_current(state.stream_name).await { - Ok(Some(current)) - if current.writer_epoch == state.writer_epoch - && current.generation == state.manifest.generation - && !current.closed - && current - .active - .as_ref() - .is_some_and(|active| active.chunk_id == chunk_id) => - { - true - } - Ok(_) => { - state.stalled = true; - tracing::info!( - stream_high = state.stream_name.high, - stream_low = state.stream_name.low, - writer_epoch = state.writer_epoch, - "chunk-stream idle writer no longer owns manifest head" - ); - false - } - Err(error) => { - tracing::warn!(%error, "chunk-stream idle authority observation unavailable"); - false - } +pub(super) async fn maintain(state: &mut WorkerState) { + if !state + .liveness_failed + .swap(false, std::sync::atomic::Ordering::AcqRel) + { + return; } -} - -async fn rotate(state: &mut WorkerState) { - match rollover(state).await { - Ok(()) => {} - Err( - error @ (StreamError::StaleWriter | StreamError::Corruption(_) | StreamError::InvalidRequest(_)), - ) => { + loop { + if state + .authority + .as_ref() + .map_or(true, |authority| authority.check().is_err()) + { state.stalled = true; - tracing::warn!(%error, "chunk-stream idle rollover cannot continue safely"); + return; } - Err(error) => { - tracing::warn!(%error, "chunk-stream idle rollover remains unavailable"); + let Some(active) = state.progress.active.as_ref() else { + return; + }; + match state + .chunks + .renew_liveness(active.chunk_id, state.writer_epoch) + .await + { + Ok(()) => return, + Err(crate::StreamError::StaleWriter) => { + if let Err(error) = rollover(state).await { + state.stalled = true; + tracing::warn!(%error, "stream cannot rotate its retired active chunk"); + } + return; + } + Err(error @ (crate::StreamError::Corruption(_) | crate::StreamError::InvalidRequest(_))) => { + state.stalled = true; + tracing::warn!(%error, "stream liveness validation failed"); + return; + } + Err(error) => state.log_unavailable("renew_liveness", &error), } + // No payload has been submitted. Backpressure until the metadata owner + // can confirm this tail, within the unchanged local owner deadline. + tokio::time::sleep(Duration::from_millis(100)).await; } } diff --git a/lib/crowdb-chunk-stream/src/stream/observation.rs b/lib/crowdb-chunk-stream/src/stream/observation.rs index ae2799c1e..3b68f1f54 100644 --- a/lib/crowdb-chunk-stream/src/stream/observation.rs +++ b/lib/crowdb-chunk-stream/src/stream/observation.rs @@ -1,9 +1,11 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Bounded views of the already-published stream manifest; no chunk reads. +//! Bounded views of the already-published stream state; no chunk reads. -use crate::{ActiveChunkDescriptor, Result, StreamError, StreamExtentPageFence}; +use crate::ChunkId; +use crate::{Result, StreamError}; +use crowdb_protocol::chunk_stream::{StreamChunkState, StreamName}; use super::ChunkStream; @@ -13,16 +15,26 @@ pub struct StreamMetadataObservation { pub writer_epoch: u64, pub metadata_group_id: u64, pub trim_offset: u64, - pub sealed_tail: u64, + pub published_tail: u64, pub closed: bool, - pub active: Option, - pub extent_pages: Vec, + pub chunk: StreamChunkRecordObservation, pub offset: usize, pub next_offset: Option, } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StreamChunkRecordObservation { + pub stream_id: StreamName, + pub chunk_id: ChunkId, + pub state: StreamChunkState, + pub logical_start: u64, + pub logical_end: u64, + pub mapping_count: u32, + pub mapping_bytes: usize, +} + impl ChunkStream { - /// Copies at most 100 extent-page fences from one published manifest. + /// Copies the published tail `StreamChunkRecord` without reading chunk data. /// /// # Errors /// Rejects continuations without a generation, stale generations and invalid offsets. @@ -31,27 +43,37 @@ impl ChunkStream { generation: Option, offset: usize, ) -> Result { - let manifest = self.manifest.load(); - if generation.is_some_and(|value| value != manifest.generation) { + let state = self.state.load(); + if generation.is_some_and(|value| value != state.generation) { return Err(StreamError::StaleWriter); } - if (offset > 0 && generation.is_none()) || offset > manifest.extent_pages.len() { + if offset != 0 { return Err(StreamError::InvalidRequest( - "invalid extent index continuation".into(), + "the tail chunk record has no continuation".into(), )); } - let end = offset.saturating_add(100).min(manifest.extent_pages.len()); Ok(StreamMetadataObservation { - generation: manifest.generation, - writer_epoch: manifest.writer_epoch, - metadata_group_id: manifest.metadata_group_id, - trim_offset: manifest.trim_offset, - sealed_tail: manifest.sealed_tail, - closed: manifest.closed, - active: manifest.active.clone(), - extent_pages: manifest.extent_pages[offset..end].to_vec(), + generation: state.generation, + writer_epoch: state.writer_epoch, + metadata_group_id: state.metadata_group_id, + trim_offset: state.trim_offset, + published_tail: state.published_tail, + closed: state.closed, + chunk: StreamChunkRecordObservation { + stream_id: state.stream_name, + chunk_id: state.snapshot.record().chunk_id, + state: state.snapshot.record().state, + logical_start: state + .snapshot + .record() + .logical_start + .ok_or_else(|| StreamError::Corruption("attached record lacks logical start".into()))?, + logical_end: state.published_tail, + mapping_count: state.snapshot.record().mappings.count(), + mapping_bytes: state.snapshot.record().mappings.encoded_len(), + }, offset, - next_offset: (end < manifest.extent_pages.len()).then_some(end), + next_offset: None, }) } } diff --git a/lib/crowdb-chunk-stream/src/stream/open.rs b/lib/crowdb-chunk-stream/src/stream/open.rs new file mode 100644 index 000000000..352404c08 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/open.rs @@ -0,0 +1,308 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Creation and ownership-fenced recovery of writer handles. + +use super::{ + Arc, ChunkStream, Result, StreamChunkStore, StreamConfig, StreamError, StreamMetadataStore, StreamName, + StreamRecord, StreamRecordState, StreamRegistry, +}; +use crate::perf::{perf, PhaseTimer}; + +impl ChunkStream { + /// Creates and publishes a new registered stream. + /// + /// # Errors + /// + /// Returns a typed storage, validation, fencing, or duplicate-name error. + pub async fn create( + record: StreamRecord, + writer_epoch: u64, + config: StreamConfig, + registry: Arc, + metadata: Arc, + chunks: Arc, + ) -> Result { + config.validate()?; + if record.state != StreamRecordState::Active + || record.metadata_group_id == 0 + || record.record_version != 1 + || record.created_at_ms == 0 + || writer_epoch == 0 + { + return Err(StreamError::InvalidRequest( + "stream record or writer epoch is invalid".into(), + )); + } + if registry.load(record.stream_name).await?.is_some() { + return Err(StreamError::InvalidRequest("stream already exists".into())); + } + registry.create(record.clone()).await?; + Self::initialize(record, writer_epoch, config, registry, metadata, chunks).await + } + + /// Initializes stream metadata for an active record already published by + /// the control plane. + /// + /// # Errors + /// + /// Returns a typed error if the record is absent or inactive, metadata + /// already exists, or initial metadata publication fails. + pub async fn create_registered( + stream_name: StreamName, + writer_epoch: u64, + config: StreamConfig, + registry: Arc, + metadata: Arc, + chunks: Arc, + ) -> Result { + config.validate()?; + let record = registry + .load(stream_name) + .await? + .ok_or_else(|| StreamError::InvalidRequest("stream record does not exist".into()))?; + if record.state != StreamRecordState::Active + || record.metadata_group_id == 0 + || record.record_version != 1 + || record.created_at_ms == 0 + || writer_epoch == 0 + { + return Err(StreamError::InvalidRequest( + "stream record or writer epoch is invalid".into(), + )); + } + if metadata + .record_store(record.metadata_group_id)? + .load_tail(stream_name) + .await? + .is_some() + { + return Err(StreamError::InvalidRequest( + "stream metadata already exists".into(), + )); + } + Self::initialize(record, writer_epoch, config, registry, metadata, chunks).await + } + + async fn initialize( + record: StreamRecord, + writer_epoch: u64, + config: StreamConfig, + registry: Arc, + metadata: Arc, + chunks: Arc, + ) -> Result { + let metadata = metadata.record_store(record.metadata_group_id)?; + let previous = metadata + .candidates(record.stream_name) + .await? + .into_iter() + .find(|candidate| candidate.record().sequence == 0); + let authority = registry.claim_writer(record.stream_name, writer_epoch).await?; + chunks.bind_writer(record.stream_name, writer_epoch, authority.clone()); + let initialized = async { + let active = chunks + .allocate_mirrored(record.stream_name, writer_epoch, record.purpose) + .await?; + authority.check()?; + let record = crowdb_protocol::chunk_stream::StreamChunkRecord { + stream_id: record.stream_name, + purpose: record.purpose, + chunk_id: active.chunk_id, + state: crowdb_protocol::chunk_stream::StreamChunkState::Active, + logical_start: Some(0), + logical_end: Some(0), + sequence: 0, + writer_epoch, + generation: 1, + owner_session: owner_session()?, + trim_offset: 0, + closed: false, + mappings: crowdb_protocol::chunk_stream::StreamMappings::default(), + }; + let _publication = PhaseTimer::start(&perf().publish); + let mut candidate = record.clone(); + candidate.state = crowdb_protocol::chunk_stream::StreamChunkState::Candidate; + candidate.logical_start = None; + candidate.logical_end = None; + candidate.generation = 0; + let candidate = metadata.reserve_initial(&candidate, previous.as_ref()).await?; + authority.check()?; + let snapshot = metadata.publish(None, record, None, Some(&candidate)).await?; + authority.check()?; + Self::start( + config, + metadata, + chunks, + snapshot, + Some(active), + Some(authority.clone()), + ) + .await + } + .await; + if initialized.is_err() { + // No data has been admitted; retain chunk expiry fences for abandoned allocations. + if let Err(error) = authority.release().await { + tracing::warn!(%error, "failed initialization owner release is unconfirmed"); + } + } + initialized + } + + /// Opens an existing stream and recovers its durable active cursor. + /// + /// # Errors + /// + /// Returns a typed storage, corruption, or stale-writer error. + pub async fn open( + stream_name: StreamName, + writer_epoch: u64, + config: StreamConfig, + registry: Arc, + metadata: Arc, + chunks: Arc, + ) -> Result { + config.validate()?; + let record = registry + .load(stream_name) + .await? + .ok_or_else(|| StreamError::InvalidRequest("stream record does not exist".into()))?; + if record.state != StreamRecordState::Active { + return Err(StreamError::InvalidRequest("stream record is not active".into())); + } + let metadata = metadata.record_store(record.metadata_group_id)?; + let snapshot = metadata + .load_tail(stream_name) + .await? + .ok_or_else(|| StreamError::Corruption("Stream tail record does not exist".into()))?; + let base = snapshot.record(); + if base.purpose != record.purpose { + return Err(StreamError::Corruption( + "Stream purpose differs from identity record".into(), + )); + } + if base.writer_epoch > writer_epoch { + return Err(StreamError::StaleWriter); + } + let authority = registry.claim_writer(stream_name, writer_epoch).await?; + chunks.bind_writer(stream_name, writer_epoch, authority.clone()); + let recovered = async { + // Claiming may wait for the old writer to drain and publish its + // final checkpoint. Recover from that completed publication. + let snapshot = metadata + .load_tail(stream_name) + .await? + .ok_or_else(|| StreamError::Corruption("Stream tail record does not exist".into()))?; + if snapshot.record().purpose != record.purpose { + return Err(StreamError::Corruption( + "Stream purpose differs from identity record".into(), + )); + } + if snapshot.record().writer_epoch > writer_epoch { + return Err(StreamError::StaleWriter); + } + Self::recover_owned( + writer_epoch, + config, + metadata, + chunks, + snapshot, + authority.clone(), + ) + .await + } + .await; + if recovered.is_err() { + if let Err(error) = authority.release().await { + tracing::warn!(%error, "failed recovery owner release is unconfirmed"); + } + } + recovered + } + + async fn recover_owned( + writer_epoch: u64, + config: StreamConfig, + metadata: crate::StreamChunkRecordStore, + chunks: Arc, + snapshot: crate::StreamChunkRecordSnapshot, + authority: Arc, + ) -> Result { + let base = snapshot.record(); + let stream_name = base.stream_id; + let session = owner_session()?; + let mut completed = base.clone(); + let mut active = super::state::load_active(&snapshot, chunks.as_ref()).await?; + let mut previous = None; + if let Some(descriptor) = active.take() { + let recovered = chunks + .recover_tail(stream_name, writer_epoch, &descriptor) + .await?; + let mut lengths = base + .mappings + .lengths() + .map_err(|error| StreamError::Corruption(error.to_string()))?; + for extent in recovered.extents { + lengths.push( + u32::try_from(extent.logical_length) + .map_err(|_| StreamError::Corruption("recovered append length".into()))?, + ); + } + chunks + .seal(descriptor.chunk_id, writer_epoch, recovered.physical_end) + .await?; + completed.logical_end = Some(recovered.logical_end); + completed.mappings = crowdb_protocol::chunk_stream::StreamMappings::new(&lengths) + .map_err(|error| StreamError::Corruption(error.to_string()))?; + completed.state = crowdb_protocol::chunk_stream::StreamChunkState::Sealed; + completed.writer_epoch = writer_epoch; + completed.owner_session = session; + let mut successor = chunks + .allocate_mirrored(stream_name, writer_epoch, base.purpose) + .await?; + successor.logical_start = recovered.logical_end; + let mut next = completed.clone(); + next.chunk_id = successor.chunk_id; + next.state = crowdb_protocol::chunk_stream::StreamChunkState::Active; + next.logical_start = Some(recovered.logical_end); + next.sequence = base + .sequence + .checked_add(1) + .ok_or_else(|| StreamError::Corruption("chunk sequence overflow".into()))?; + next.mappings = crowdb_protocol::chunk_stream::StreamMappings::default(); + previous = Some(completed); + completed = next; + active = Some(successor); + } + completed.writer_epoch = writer_epoch; + completed.owner_session = session; + completed.generation = base + .generation + .checked_add(1) + .ok_or_else(|| StreamError::Corruption("Stream generation overflow".into()))?; + if let Some(previous) = &mut previous { + previous.generation = completed.generation; + } + authority.check()?; + let _publication = PhaseTimer::start(&perf().publish); + let snapshot = metadata + .publish(Some(&snapshot), completed, previous, None) + .await?; + super::successor::reclaim_excluded(&metadata, chunks.as_ref(), stream_name, writer_epoch).await?; + Self::start( + config, + metadata, + chunks, + snapshot, + active, + Some(authority.clone()), + ) + .await + } +} + +fn owner_session() -> Result<[u8; 16]> { + crowdb_protocol::chunk_stream::StreamProcessLease::generate_boot_nonce() + .map_err(|error| StreamError::Internal(error.to_string())) +} diff --git a/lib/crowdb-chunk-stream/src/stream/preparation.rs b/lib/crowdb-chunk-stream/src/stream/preparation.rs new file mode 100644 index 000000000..60b5ed538 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/preparation.rs @@ -0,0 +1,146 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One bounded strip preparation overlaps writes to the initialized prefix. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::task::JoinHandle; + +use super::WorkerState; +use crate::perf::{perf, PhaseTimer}; +use crate::{ActiveChunkDescriptor, ChunkId, Result, StreamError}; + +pub(super) struct CapacityPreparation { + chunk_id: ChunkId, + task: JoinHandle<(Result>, Duration)>, +} + +pub(super) fn start(state: &mut WorkerState, batch_bytes: usize) { + let Some(active) = state.progress.active.as_ref() else { + return; + }; + if state.capacity_preparation.is_some() + || state.capacity_exhausted == Some(active.chunk_id) + || state + .capacity_failure + .as_ref() + .is_some_and(|(id, _)| *id == active.chunk_id) + { + return; + } + let Some(required) = state.capacity_plan.target( + active.acknowledged_cursor, + active.capacity, + state.config.chunk_capacity_bytes, + batch_bytes as u64, + ) else { + return; + }; + let chunks = Arc::clone(&state.chunks); + let name = state.stream_name; + let epoch = state.writer_epoch; + let chunk_id = active.chunk_id; + state.capacity_preparation = Some(CapacityPreparation { + chunk_id, + task: tokio::spawn(async move { + let _timer = PhaseTimer::start(&perf().preparation); + let started = Instant::now(); + let result = chunks.grow_mirrored(name, epoch, chunk_id, required).await; + (result, started.elapsed()) + }), + }); +} + +pub(super) async fn poll_ready(state: &mut WorkerState) -> Result<()> { + if state + .capacity_preparation + .as_ref() + .is_some_and(|preparation| preparation.task.is_finished()) + { + complete(state).await; + } + // A failed initialization cannot make the new suffix writable. The old + // initialized prefix remains usable unless authority itself was lost. + if let Some((id, StreamError::StaleWriter)) = &state.capacity_failure { + if state + .progress + .active + .as_ref() + .is_some_and(|active| active.chunk_id == *id) + { + return Err(StreamError::StaleWriter); + } + } + Ok(()) +} + +pub(super) async fn drain(state: &mut WorkerState) -> Result<()> { + complete(state).await; + if matches!(state.capacity_failure, Some((_, StreamError::ReadUnavailable(_)))) { + // Allocation availability may recover. The ordinary slow path can + // retry it; uncertain initialization is InvalidRequest and stays fatal. + state.capacity_failure = None; + } + if let Some((id, error)) = &state.capacity_failure { + if state + .progress + .active + .as_ref() + .is_some_and(|active| active.chunk_id == *id) + { + return Err(error.clone()); + } + } + Ok(()) +} + +async fn complete(state: &mut WorkerState) { + let Some(preparation) = state.capacity_preparation.take() else { + return; + }; + let chunk_id = preparation.chunk_id; + let result = match preparation.task.await { + Ok((result, elapsed)) => { + state.capacity_plan.record_preparation(elapsed); + result + } + Err(error) => Err(StreamError::Internal(format!( + "capacity preparation task failed: {error}" + ))), + }; + if let Err(error) = install(state, chunk_id, result) { + tracing::warn!(?chunk_id, %error, "capacity preparation failed; retaining only initialized capacity"); + state.capacity_failure = Some((chunk_id, error)); + } +} + +fn install( + state: &mut WorkerState, + chunk_id: ChunkId, + result: Result>, +) -> Result<()> { + let active = state + .progress + .active + .as_mut() + .filter(|active| active.chunk_id == chunk_id) + .ok_or_else(|| StreamError::Internal("capacity preparation outlived its active chunk".into()))?; + let Some(prepared) = result? else { + state.capacity_exhausted = Some(chunk_id); + return Ok(()); + }; + if prepared.chunk_id != chunk_id + || prepared.capacity < active.capacity + || prepared.capacity > state.config.chunk_capacity_bytes + { + return Err(StreamError::Corruption( + "prepared capacity has invalid chunk bounds".into(), + )); + } + // The append worker has advanced since preparation began. Only capacity + // belongs to this result; its captured cursors must never replace progress. + active.capacity = prepared.capacity; + Ok(()) +} diff --git a/lib/crowdb-chunk-stream/src/stream/publication.rs b/lib/crowdb-chunk-stream/src/stream/publication.rs new file mode 100644 index 000000000..57d4a1464 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/publication.rs @@ -0,0 +1,143 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Publish only the changed tail chunk record at ordered lifecycle barriers. + +use super::{state::StreamViewState, ReadView, WorkerState}; +use crate::perf::{perf, PhaseTimer}; +use crate::{Result, StreamError}; +use crowdb_protocol::chunk_stream::{StreamChunkRecord, StreamChunkState, StreamMappings}; +use std::sync::{atomic::Ordering, Arc}; +use std::time::Duration; + +pub(super) async fn publish_state(state: &mut WorkerState) -> Result<()> { + state + .authority + .as_ref() + .ok_or(StreamError::WriteStalled)? + .check()?; + super::checkpoint::flush(state).await?; + let _timer = PhaseTimer::start(&perf().publish); + let build = PhaseTimer::start(&perf().record_build); + let (tail, previous) = build_records(state)?; + drop(build); + let candidate = if previous.is_some() { + state + .metadata + .candidates(state.stream_name) + .await? + .into_iter() + .find(|candidate| candidate.record().chunk_id == tail.chunk_id) + } else { + None + }; + let snapshot = loop { + state + .authority + .as_ref() + .ok_or(StreamError::WriteStalled)? + .check()?; + match state + .metadata + .publish( + Some(&state.snapshot), + tail.clone(), + previous.clone(), + candidate.as_ref(), + ) + .await + { + Ok(snapshot) => break snapshot, + Err( + error @ (StreamError::Corruption(_) + | StreamError::InvalidRequest(_) + | StreamError::StaleWriter), + ) => { + state.stalled = true; + return Err(error); + } + Err(error) => { + state.log_unavailable("chunk_record_publish", &error); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + }; + state.progress.generation = snapshot.record().generation; + state.snapshot = snapshot; + state.extents.clear(); + let published = Arc::new(StreamViewState::new( + state.snapshot.clone(), + state.metadata.group_id(), + )); + let view = ReadView::new(state.metadata.clone(), published.clone()).await?; + let previous_view = state.read_view.swap(Arc::new(view)); + if let Ok(previous_view) = Arc::try_unwrap(previous_view) { + previous_view.pin.release().await; + } + state.published_view.store(published); + state + .metrics + .metadata_publications + .fetch_add(1, Ordering::Relaxed); + Ok(()) +} + +fn build_records(state: &WorkerState) -> Result<(StreamChunkRecord, Option)> { + let mut completed = state.snapshot.record().clone(); + let mut lengths = completed.mappings.lengths().map_err(codec_error)?; + for extent in &state.extents { + if extent.chunk_id != completed.chunk_id { + return Err(StreamError::Corruption( + "unpublished mappings span multiple chunks".into(), + )); + } + lengths.push( + u32::try_from(extent.logical_end - extent.logical_start) + .map_err(|_| StreamError::Corruption("append length exceeds u32".into()))?, + ); + } + completed.mappings = StreamMappings::new(&lengths).map_err(codec_error)?; + completed.logical_end = Some(state.progress.logical_tail); + completed.writer_epoch = state.writer_epoch; + completed.owner_session = state.owner_session; + completed.generation = state + .snapshot + .record() + .generation + .checked_add(1) + .ok_or_else(|| StreamError::Corruption("Stream publication generation overflow".into()))?; + completed.trim_offset = state.progress.trim_offset; + completed.closed = state.progress.closed; + if let Some(active) = &state.progress.active { + if active.chunk_id != completed.chunk_id { + completed.state = StreamChunkState::Sealed; + let tail = StreamChunkRecord { + stream_id: state.stream_name, + purpose: state.progress.purpose, + chunk_id: active.chunk_id, + state: StreamChunkState::Active, + logical_start: Some(state.progress.logical_tail), + logical_end: Some(state.progress.logical_tail), + sequence: completed + .sequence + .checked_add(1) + .ok_or_else(|| StreamError::Corruption("Stream chunk sequence overflow".into()))?, + writer_epoch: state.writer_epoch, + generation: completed.generation, + owner_session: state.owner_session, + trim_offset: state.progress.trim_offset, + closed: false, + mappings: StreamMappings::default(), + }; + return Ok((tail, Some(completed))); + } + completed.state = StreamChunkState::Active; + } else { + completed.state = StreamChunkState::Sealed; + } + Ok((completed, None)) +} + +fn codec_error(error: impl std::fmt::Display) -> StreamError { + StreamError::Corruption(error.to_string()) +} diff --git a/lib/crowdb-chunk-stream/src/stream/read.rs b/lib/crowdb-chunk-stream/src/stream/read.rs new file mode 100644 index 000000000..ffd21c926 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/read.rs @@ -0,0 +1,401 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Snapshot reads and bounded sequential prefetch. + +use super::{ChunkStream, PhysicalRead, ReadHint, ReadSegment, ReadView}; +use crate::metadata::resolve_group; + +use crate::perf::{perf, PhaseTimer}; +use crate::{ChunkId, Result, StreamError}; +use bytes::{Bytes, BytesMut}; +use crowdb_protocol::frame::{parse_frame, FrameLayout, FrameMagic, FrameRole, FRAME_GROUP_HEADER_BYTES}; +use std::sync::atomic::Ordering; +use std::time::Instant; + +impl ChunkStream { + /// Reads an exact retained durable logical range. + /// + /// # Errors + /// + /// Returns an error for an invalid range, corrupt metadata, or unavailable data. + pub async fn read_at(&self, offset: u64, length: usize) -> Result { + let segments = self.read_at_with_provenance(offset, length).await?; + let mut output = BytesMut::with_capacity(length); + for segment in segments { + output.extend_from_slice(&segment.data); + } + Ok(output.freeze()) + } + + /// Reads an exact range while retaining the physical chunk identity for + /// every returned logical segment. + /// + /// # Errors + /// + /// Returns an error for an invalid range, corrupt metadata, or unavailable data. + pub async fn read_at_with_provenance(&self, offset: u64, length: usize) -> Result> { + let _timer = PhaseTimer::start(&perf().read); + let plan_timer = PhaseTimer::start(&perf().read_plan); + let end = offset + .checked_add(length as u64) + .ok_or_else(|| StreamError::InvalidRequest("read range overflows".into()))?; + let view = self.read_view.load_full(); + view.pin.check()?; + let state = &view.state; + if offset < state.trim_offset || end > view.end() { + return Err(StreamError::InvalidRequest( + "read is outside retained durable range".into(), + )); + } + let mut cursor = offset; + let mut reads = Vec::new(); + while cursor < end { + let location = if cursor >= state.published_tail { + view.resolve(cursor)? + } else { + let (record, cached) = self.metadata.locate_for_read(state.stream_name, cursor).await?; + let counter = if cached { + &self.metrics.record_cache_hits + } else { + &self.metrics.record_cache_misses + }; + counter.fetch_add(1, Ordering::Relaxed); + if record.record().purpose != state.purpose { + return Err(StreamError::Corruption( + "Stream chunk record purpose differs".into(), + )); + } + let mapping = record + .index() + .locate(&record.record().mappings, cursor) + .map_err(|error| StreamError::Corruption(error.to_string()))?; + resolve_group( + record.record().chunk_id, + mapping.logical_start, + mapping.logical_end, + mapping.physical_start, + cursor, + )? + }; + let read_len = usize::try_from(location.available.min(end - cursor)) + .map_err(|_| StreamError::InvalidRequest("read length exceeds addressable range".into()))?; + let read_len = read_len.min(self.config.read_request_bytes); + reads.push(PhysicalRead { + chunk_id: location.chunk_id, + logical_start: cursor, + frame_start: location.frame_offset, + frame_length: usize::try_from(location.frame_length).map_err(|_| { + StreamError::Corruption("stream frame length exceeds address space".into()) + })?, + payload_start: usize::try_from(location.payload_offset).map_err(|_| { + StreamError::Corruption("stream frame payload offset exceeds address space".into()) + })?, + length: read_len, + group_start: location.group_offset, + group_length: location.group_length, + ordinal: location.ordinal, + }); + cursor += read_len as u64; + } + drop(plan_timer); + let io_timer = PhaseTimer::start(&perf().read_io); + let output = self.read_physical_ranges(&reads, &view).await?; + view.pin.check()?; + drop(io_timer); + let output_len = output + .iter() + .fold(0_usize, |total, segment| total.saturating_add(segment.data.len())); + if output_len != length { + return Err(StreamError::Corruption( + "chunk reads returned an unexpected length".into(), + )); + } + self.metrics + .read_bytes + .fetch_add(length as u64, Ordering::Relaxed); + Ok(output) + } + + async fn read_physical_ranges( + &self, + reads: &[PhysicalRead], + view: &std::sync::Arc, + ) -> Result> { + self.metrics + .physical_read_requests + .fetch_add(reads.len() as u64, Ordering::Relaxed); + if let [read] = reads { + return Ok(vec![self.read_physical(*read, view).await?]); + } + let mut output = vec![None; reads.len()]; + let mut tasks = tokio::task::JoinSet::new(); + let mut next = 0; + while next < reads.len() || !tasks.is_empty() { + while next < reads.len() && tasks.len() < self.config.read_concurrency { + let index = next; + let read = reads[index]; + let stream = self.clone(); + let view = std::sync::Arc::clone(view); + tasks.spawn(async move { + stream + .read_physical(read, &view) + .await + .map(|value| (index, value)) + }); + next += 1; + } + let (index, value) = tasks + .join_next() + .await + .ok_or_else(|| StreamError::Internal("physical read task set ended early".into()))? + .map_err(|error| StreamError::Internal(format!("physical read task failed: {error}")))??; + output[index] = Some(value); + } + output + .into_iter() + .map(|value| value.ok_or_else(|| StreamError::Internal("physical read result is missing".into()))) + .collect() + } + + async fn read_physical(&self, read: PhysicalRead, view: &ReadView) -> Result { + view.pin.check()?; + let group_bytes = FrameLayout::StreamGroupV2 + .physical_length(read.group_length) + .map_err(|error| StreamError::Corruption(error.to_string()))?; + let synchronized_end = read + .group_start + .checked_add(group_bytes) + .ok_or_else(|| StreamError::Corruption("captured group boundary overflows".into()))?; + let frame = self + .read_chunk_with_watchdog( + read.chunk_id, + read.frame_start, + read.frame_length, + read.logical_start, + synchronized_end, + &view.pin, + ) + .await?; + decode_segment(read, &frame) + } + + /// Creates a bounded sequential reader ending at the current durable tail. + /// + /// # Errors + /// + /// Returns an error if `offset` is outside the retained durable range. + pub fn read_from(&self, offset: u64) -> Result { + self.reader(offset, ReadHint::ToEnd) + } + + /// Creates a seekable reader with a finite or captured-tail hint. + /// + /// # Errors + /// + /// Returns an error if `offset` or its hint is outside the retained range. + pub fn reader(&self, offset: u64, hint: ReadHint) -> Result { + let view = self.read_view.load(); + let state = &view.state; + let tail = view.end(); + if offset < state.trim_offset || offset > tail { + return Err(StreamError::InvalidRequest( + "read cursor is outside retained range".into(), + )); + } + let end = match hint { + ReadHint::ToEnd => tail, + ReadHint::Bytes(length) => offset + .checked_add(length) + .ok_or_else(|| StreamError::InvalidRequest("reader hint overflows".into()))? + .min(tail), + }; + let mut captured = self.clone(); + captured.state = std::sync::Arc::new(arc_swap::ArcSwap::from(std::sync::Arc::clone(&view.state))); + captured.read_view = std::sync::Arc::new(arc_swap::ArcSwap::from(std::sync::Arc::clone(&view))); + Ok(StreamReader { + stream: captured, + offset, + end, + pending: None, + }) + } + + async fn read_chunk_with_watchdog( + &self, + chunk_id: ChunkId, + physical_offset: u64, + length: usize, + logical_offset: u64, + synchronized_end: u64, + pin: &crate::StreamReadPin, + ) -> Result { + let read = self + .chunks + .read_verified_frame(chunk_id, physical_offset, length, synchronized_end); + tokio::pin!(read); + let started = Instant::now(); + loop { + tokio::select! { + result = &mut read => return result, + () = tokio::time::sleep(pin.remaining()) => pin.check()?, + () = tokio::time::sleep(self.config.watchdog_interval) => { + pin.check()?; + self.metrics.watchdog_observations.fetch_add(1, Ordering::Relaxed); + tracing::warn!( + stream_high = self.stream_name.high, + stream_low = self.stream_name.low, + writer_epoch = self.writer_epoch, + logical_offset, + read_bytes = length, + operation_age_ms = started.elapsed().as_millis(), + stage = "chunk_read", + "chunk-stream read window remains in flight" + ); + } + } + } + } +} + +fn decode_segment(read: PhysicalRead, frame: &Bytes) -> Result { + let parsed = parse_frame(frame, read.chunk_id) + .map_err(|error| StreamError::Corruption(format!("invalid stream frame: {error}")))?; + let expected_role = if read.ordinal == 0 { + FrameRole::Head { + logical_length: read.group_length, + frame_count: u32::try_from( + read.group_length + .div_ceil(FrameLayout::StreamGroupV2.payload_capacity()), + ) + .map_err(|_| StreamError::Corruption("stream group count exceeds u32".into()))?, + } + } else { + FrameRole::Follower { + head_offset: read.group_start, + ordinal: read.ordinal, + } + }; + if parsed.header.magic != FrameMagic::StreamV2 + || parsed.role != expected_role + || parsed.physical_length != read.frame_length + { + return Err(StreamError::Corruption( + "stream extent has the wrong frame kind".into(), + )); + } + let payload_end = read + .payload_start + .checked_add(read.length) + .ok_or_else(|| StreamError::Corruption("stream frame payload range overflows".into()))?; + if read.payload_start < FRAME_GROUP_HEADER_BYTES + || payload_end > FRAME_GROUP_HEADER_BYTES + parsed.payload.len() + { + return Err(StreamError::Corruption( + "stream extent exceeds frame payload".into(), + )); + } + let data = frame.slice(read.payload_start..payload_end); + Ok(ReadSegment { + chunk_id: read.chunk_id, + logical_start: read.logical_start, + data, + }) +} + +type PrefetchResult = tokio::task::JoinHandle)>>; + +pub struct StreamReader { + pub(super) stream: ChunkStream, + pub(super) offset: u64, + pub(super) end: u64, + pub(super) pending: Option, +} + +impl StreamReader { + /// Moves the reader within its captured retained range. + /// + /// # Errors + /// + /// Returns an error when `offset` is outside the captured range. + pub fn seek(&mut self, offset: u64) -> Result<()> { + let trim = self.stream.state.load().trim_offset; + if offset < trim || offset > self.end { + return Err(StreamError::InvalidRequest( + "seek is outside the reader range".into(), + )); + } + if let Some(pending) = self.pending.take() { + pending.abort(); + } + self.offset = offset; + Ok(()) + } + + /// Returns the next bounded logical window, or `None` at the captured tail. + /// + /// # Errors + /// + /// Returns a typed metadata or chunk-read error. + pub async fn next(&mut self) -> Result> { + let Some(segments) = self.next_with_provenance().await? else { + return Ok(None); + }; + let length = segments.iter().map(|segment| segment.data.len()).sum(); + let mut bytes = BytesMut::with_capacity(length); + for segment in segments { + bytes.extend_from_slice(&segment.data); + } + Ok(Some(bytes.freeze())) + } + + /// Returns the next bounded logical window split by physical chunk. + /// + /// # Errors + /// + /// Returns a typed metadata, task, or chunk-read error. + pub async fn next_with_provenance(&mut self) -> Result>> { + if self.offset == self.end { + return Ok(None); + } + self.start_prefetch()?; + let pending = self + .pending + .take() + .ok_or_else(|| StreamError::Internal("reader prefetch was not installed".into()))?; + let (offset, segments) = pending + .await + .map_err(|error| StreamError::Internal(format!("reader prefetch task failed: {error}")))??; + if offset != self.offset { + return Err(StreamError::Internal( + "reader prefetch completed for a stale offset".into(), + )); + } + let length = segments.iter().try_fold(0_u64, |length, segment| { + length.checked_add(segment.data.len() as u64) + }); + self.offset = self + .offset + .checked_add(length.ok_or_else(|| StreamError::Corruption("reader length overflows".into()))?) + .ok_or_else(|| StreamError::Corruption("reader offset overflows".into()))?; + self.start_prefetch()?; + Ok(Some(segments)) + } + + fn start_prefetch(&mut self) -> Result<()> { + if self.pending.is_some() || self.offset == self.end { + return Ok(()); + } + let offset = self.offset; + let length = usize::try_from((self.end - offset).min(self.stream.config.read_window_bytes as u64)) + .map_err(|_| StreamError::InvalidRequest("read window exceeds addressable range".into()))?; + let stream = self.stream.clone(); + self.pending = Some(tokio::spawn(async move { + stream + .read_at_with_provenance(offset, length) + .await + .map(|segments| (offset, segments)) + })); + Ok(()) + } +} diff --git a/lib/crowdb-chunk-stream/src/stream/recovery.rs b/lib/crowdb-chunk-stream/src/stream/recovery.rs deleted file mode 100644 index d5350cde4..000000000 --- a/lib/crowdb-chunk-stream/src/stream/recovery.rs +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Rebuild mappings for durable frames beyond the last published directory. - -use super::{Extent, StreamChunkStore}; -use crate::{ActiveChunkDescriptor, Result, StreamError}; -use crowdb_protocol::frame::{ - frame_length, parse_frame, parse_header, FrameMagic, FRAME_HEADER_PREFIX_BYTES, -}; - -pub(super) async fn recover_active_frames( - chunks: &dyn StreamChunkStore, - active: &ActiveChunkDescriptor, - durable_offset: u64, - extents: &mut Vec, -) -> Result { - let mut physical = active.acknowledged_cursor; - let mut logical = active.logical_start; - while physical < durable_offset { - if durable_offset - physical < FRAME_HEADER_PREFIX_BYTES as u64 { - return Err(StreamError::Corruption( - "durable stream tail ends inside a frame header".into(), - )); - } - let prefix = chunks - .read(active.chunk_id, physical, FRAME_HEADER_PREFIX_BYTES) - .await?; - let header = parse_header(&prefix).map_err(|error| frame_error(&error))?; - let length = frame_length(header).map_err(|error| frame_error(&error))?; - if length as u64 > durable_offset - physical { - return Err(StreamError::Corruption( - "durable stream tail ends inside a frame".into(), - )); - } - let frame = chunks - .read_verified_frame(active.chunk_id, physical, length) - .await?; - let parsed = parse_frame(&frame, active.chunk_id).map_err(|error| frame_error(&error))?; - if parsed.header.magic != FrameMagic::StreamV1 { - return Err(StreamError::Corruption( - "durable stream tail has the wrong frame kind".into(), - )); - } - if parsed.payload.is_empty() { - return Err(StreamError::Corruption( - "recovered stream frame has an empty payload".into(), - )); - } - let end = logical - .checked_add(parsed.payload.len() as u64) - .ok_or_else(|| StreamError::Corruption("recovered logical stream tail overflows".into()))?; - extents.push(Extent { - chunk_id: active.chunk_id, - logical_start: logical, - logical_end: end, - frame_start: physical, - frame_length: u32::try_from(length) - .map_err(|_| frame_error(&crowdb_protocol::frame::FrameError::FrameTooLarge))?, - }); - physical += length as u64; - logical = end; - } - Ok(logical) -} - -fn frame_error(error: &crowdb_protocol::frame::FrameError) -> StreamError { - StreamError::Corruption(format!("invalid recovered stream frame: {error}")) -} diff --git a/lib/crowdb-chunk-stream/src/stream/state.rs b/lib/crowdb-chunk-stream/src/stream/state.rs new file mode 100644 index 000000000..15a3e3d57 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/state.rs @@ -0,0 +1,85 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Mutable owner progress and a bounded captured view of the tail chunk record. + +use super::StreamChunkRecordSnapshot; +use crate::{Result, StreamChunkStore, StreamError}; +use crowdb_protocol::chunk_stream::{ + ActiveChunkDescriptor, StreamChunkRecord, StreamChunkState, StreamName, StreamPurpose, +}; + +pub(super) struct StreamViewState { + pub(super) purpose: StreamPurpose, + pub(super) stream_name: StreamName, + pub(super) metadata_group_id: u64, + pub(super) writer_epoch: u64, + pub(super) generation: u64, + pub(super) trim_offset: u64, + pub(super) published_tail: u64, + pub(super) closed: bool, + pub(super) snapshot: StreamChunkRecordSnapshot, +} + +impl StreamViewState { + pub(super) fn new(snapshot: StreamChunkRecordSnapshot, group: u64) -> Self { + let record = snapshot.record(); + Self { + purpose: record.purpose, + stream_name: record.stream_id, + metadata_group_id: group, + writer_epoch: record.writer_epoch, + generation: record.generation, + trim_offset: record.trim_offset, + published_tail: record.logical_end.expect("validated attached record"), + closed: record.closed, + snapshot, + } + } +} + +pub(super) struct WriterProgress { + pub(super) purpose: StreamPurpose, + pub(super) generation: u64, + pub(super) trim_offset: u64, + pub(super) logical_tail: u64, + pub(super) active: Option, + pub(super) closed: bool, +} + +impl WriterProgress { + pub(super) fn new(record: &StreamChunkRecord, active: Option) -> Self { + Self { + purpose: record.purpose, + generation: record.generation, + trim_offset: record.trim_offset, + logical_tail: record.logical_end.expect("validated attached record"), + active, + closed: record.closed, + } + } +} + +pub(super) async fn load_active( + snapshot: &StreamChunkRecordSnapshot, + chunks: &dyn StreamChunkStore, +) -> Result> { + let record = snapshot.record(); + if record.state != StreamChunkState::Active { + return Ok(None); + } + let capacity = chunks.current_capacity(record.chunk_id).await?; + let physical = snapshot.index().physical_end(); + if physical > capacity { + return Err(StreamError::Corruption( + "Stream checkpoint exceeds chunk capacity".into(), + )); + } + Ok(Some(ActiveChunkDescriptor { + chunk_id: record.chunk_id, + logical_start: record.logical_end.expect("validated"), + physical_start: physical, + acknowledged_cursor: physical, + capacity, + })) +} diff --git a/lib/crowdb-chunk-stream/src/stream/successor.rs b/lib/crowdb-chunk-stream/src/stream/successor.rs new file mode 100644 index 000000000..4436bbac7 --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/successor.rs @@ -0,0 +1,316 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One unwritten successor overlaps appends to the current attached chunk. + +use std::sync::Arc; +use std::time::Duration; + +use tokio::sync::oneshot; +use tokio::task::JoinHandle; + +use super::WorkerState; +use crate::perf::{perf, PhaseTimer}; +use crate::{ActiveChunkDescriptor, Result, StreamChunkStore, StreamError, StreamWriterSession}; + +pub(super) struct SuccessorPreparation { + handoff: oneshot::Sender, + task: JoinHandle>, +} + +pub(super) fn start(state: &mut WorkerState) { + let Some(authority) = state.authority.as_ref() else { + return; + }; + if state.successor.is_some() || state.progress.active.is_none() || authority.check().is_err() { + return; + } + let request = SuccessorRequest { + chunks: Arc::clone(&state.chunks), + metadata: state.metadata.clone(), + sequence: state.snapshot.record().sequence + 1, + owner_session: state.owner_session, + authority: Arc::clone(authority), + name: state.stream_name, + epoch: state.writer_epoch, + purpose: state.progress.purpose, + interval: state.config.liveness_interval, + required: state + .capacity_plan + .target( + 0, + 0, + state.config.chunk_capacity_bytes, + state.config.batch_bytes as u64, + ) + .expect("nonzero chunk capacity"), + }; + let (handoff, receive) = oneshot::channel(); + state.successor = Some(SuccessorPreparation { + handoff, + task: tokio::spawn(request.run(receive)), + }); +} + +struct SuccessorRequest { + chunks: Arc, + metadata: crate::StreamChunkRecordStore, + sequence: u64, + owner_session: [u8; 16], + authority: Arc, + name: crate::StreamName, + epoch: u64, + purpose: crate::StreamPurpose, + interval: Duration, + required: u64, +} + +impl SuccessorRequest { + async fn register(&self, active: &ActiveChunkDescriptor) -> Result<()> { + use crowdb_protocol::chunk_stream::{StreamChunkRecord, StreamChunkState, StreamMappings}; + let record = StreamChunkRecord { + stream_id: self.name, + purpose: self.purpose, + chunk_id: active.chunk_id, + state: StreamChunkState::Candidate, + logical_start: None, + logical_end: None, + sequence: self.sequence, + writer_epoch: self.epoch, + generation: 0, + owner_session: self.owner_session, + trim_offset: 0, + closed: false, + mappings: StreamMappings::default(), + }; + loop { + self.authority.check()?; + match self.metadata.register_candidate(&record).await { + Ok(_) => return Ok(()), + Err( + error @ (StreamError::StaleWriter + | StreamError::Corruption(_) + | StreamError::InvalidRequest(_)), + ) => return Err(error), + Err(error) => { + tracing::warn!(%error,"candidate registration unavailable; retrying"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + } + } + + async fn run(self, mut receive: oneshot::Receiver) -> Result { + let mut use_requested = false; + loop { + self.authority.check()?; + let result = { + let _timer = PhaseTimer::start(&perf().successor_prepare); + allocate( + self.chunks.as_ref(), + self.name, + self.epoch, + self.purpose, + self.required, + ) + .await + }; + let active = match result { + Ok(active) => active, + Err( + error @ (StreamError::ReadUnavailable(_) + | StreamError::Internal(_) + | StreamError::Backpressure), + ) => { + tracing::warn!(%error, "successor allocation unavailable; retrying in background"); + if !use_requested { + match receive.try_recv() { + Ok(true) => use_requested = true, + Ok(false) | Err(oneshot::error::TryRecvError::Closed) => return Err(error), + Err(oneshot::error::TryRecvError::Empty) => {} + } + } + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + Err(error) => return Err(error), + }; + if active.physical_start != 0 || active.acknowledged_cursor != 0 || active.capacity == 0 { + return Err(StreamError::Corruption( + "prepared successor is not an initialized empty chunk".into(), + )); + } + self.register(&active).await?; + if use_requested + || await_handoff( + self.chunks.as_ref(), + &self.authority, + self.epoch, + &active, + self.interval, + &mut receive, + ) + .await? + { + self.authority.check()?; + return Ok(active); + } + self.metadata + .remove_candidate(self.name, self.sequence, active.chunk_id) + .await?; + if let Err(error) = reclaim(self.chunks.as_ref(), self.epoch, &active).await { + tracing::warn!(?active.chunk_id, %error, "failed candidate cleanup deferred to chunk expiry"); + } + } + } +} + +async fn allocate( + chunks: &dyn StreamChunkStore, + name: crate::StreamName, + epoch: u64, + purpose: crate::StreamPurpose, + required: u64, +) -> Result { + let mut active = chunks.allocate_mirrored(name, epoch, purpose).await?; + if active.capacity < required { + match chunks.grow_mirrored(name, epoch, active.chunk_id, required).await { + Ok(Some(grown)) if grown.chunk_id == active.chunk_id && grown.capacity >= required => { + active = grown; + } + Ok(None) => {} + result => { + let error = result.err().unwrap_or_else(|| { + StreamError::Corruption("successor growth changed its identity".into()) + }); + // No user data was admitted. Initialization has drained, so the + // failed candidate can be sealed empty and reclaimed safely. + reclaim(chunks, epoch, &active).await?; + return Err(match error { + StreamError::StaleWriter | StreamError::Corruption(_) => error, + other => StreamError::ReadUnavailable(format!( + "unused successor initialization failed: {other}" + )), + }); + } + } + } + Ok(active) +} + +pub(super) async fn take(state: &mut WorkerState) -> Result { + let Some(preparation) = state.successor.take() else { + start(state); + let preparation = state.successor.take().ok_or(StreamError::WriteStalled)?; + return join(preparation, true).await; + }; + let _timer = PhaseTimer::start(&perf().successor_wait); + join(preparation, true).await +} + +async fn await_handoff( + chunks: &dyn StreamChunkStore, + authority: &StreamWriterSession, + epoch: u64, + active: &ActiveChunkDescriptor, + interval: Duration, + receive: &mut oneshot::Receiver, +) -> Result { + let mut ticks = tokio::time::interval(interval); + ticks.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + ticks.tick().await; + loop { + tokio::select! { + biased; + _ = &mut *receive => { + authority.check()?; + return Ok(true); + } + _ = ticks.tick() => { + authority.check()?; + if let Err(error) = chunks.renew_liveness(active.chunk_id, epoch).await { + tracing::warn!(?active.chunk_id, %error, "successor liveness renewal failed; replacing candidate"); + return Ok(false); + } + } + } + } +} + +pub(super) async fn discard(state: &mut WorkerState) -> Result<()> { + let Some(preparation) = state.successor.take() else { + return Ok(()); + }; + // Drain initialization before sealing or releasing owner authority. Never + // abort submitted IO: a late initializer must not overwrite a later writer. + let active = match join(preparation, false).await { + Ok(active) => active, + Err(error) => { + // A failed unused allocation cannot invalidate the attached chunk. + // No descriptor was returned; chunk-layer expiry owns its cleanup. + tracing::warn!(%error, "unused successor preparation failed"); + return Ok(()); + } + }; + state + .metadata + .remove_candidate( + state.stream_name, + state.snapshot.record().sequence + 1, + active.chunk_id, + ) + .await?; + if let Err(error) = reclaim(state.chunks.as_ref(), state.writer_epoch, &active).await { + if matches!( + error, + StreamError::StaleWriter | StreamError::Corruption(_) | StreamError::InvalidRequest(_) + ) { + return Err(error); + } + // The task has drained initialization and this detached candidate never + // admitted data. Keep its allocation for chunk expiry; cleanup cannot + // invalidate the attached stream's acknowledged prefix or release proof. + tracing::warn!(?active.chunk_id, %error, "unused candidate cleanup deferred to chunk expiry"); + } + Ok(()) +} + +async fn reclaim(chunks: &dyn StreamChunkStore, epoch: u64, active: &ActiveChunkDescriptor) -> Result<()> { + chunks.seal(active.chunk_id, epoch, 0).await?; + chunks.release_trimmed(active.chunk_id, 0, 0).await?; + Ok(()) +} + +pub(super) async fn reclaim_excluded( + metadata: &crate::StreamChunkRecordStore, + chunks: &dyn StreamChunkStore, + name: crate::StreamName, + epoch: u64, +) -> Result<()> { + // Reservations from the excluded owner contain no acknowledged data. + for candidate in metadata.candidates(name).await? { + let record = candidate.record(); + if record.writer_epoch <= epoch { + let cursor = chunks.durable_cursor(record.chunk_id, epoch).await?; + if cursor.offset != 0 { + return Err(StreamError::Corruption( + "unattached candidate contains committed data".into(), + )); + } + chunks.seal(record.chunk_id, epoch, 0).await?; + chunks.release_trimmed(record.chunk_id, 0, 0).await?; + metadata + .remove_candidate(name, record.sequence, record.chunk_id) + .await?; + } + } + Ok(()) +} + +async fn join(preparation: SuccessorPreparation, use_candidate: bool) -> Result { + let _ = preparation.handoff.send(use_candidate); + preparation + .task + .await + .map_err(|error| StreamError::Internal(format!("successor preparation task failed: {error}")))? +} diff --git a/lib/crowdb-chunk-stream/src/stream/view.rs b/lib/crowdb-chunk-stream/src/stream/view.rs new file mode 100644 index 000000000..396611bef --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/view.rs @@ -0,0 +1,127 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Published chunk record plus append-only local mappings since that checkpoint. + +use std::ops::Bound::Included; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; + +use arc_swap::ArcSwap; +use crossbeam_skiplist::SkipMap; +use tokio::sync::oneshot; + +use super::state::StreamViewState; +use super::{ChunkStream, Command, Extent}; +use crate::metadata::resolve_group; +use crate::{ExtentLocation, Result, StreamError}; +use crate::{StreamChunkRecordStore, StreamReadPin}; + +/// The source of a handle's finite or live read boundary. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StreamViewSource { + LiveOwner, + OwnerCheckpoint, + PersistedCheckpoint, +} + +pub(super) struct ReadView { + pub(super) state: Arc, + extents: SkipMap, + end: AtomicU64, + pub(super) pin: StreamReadPin, +} + +impl ReadView { + pub(super) async fn new(metadata: StreamChunkRecordStore, state: Arc) -> Result { + let pin = metadata.pin_read(&state.snapshot).await?; + Ok(Self { + pin, + end: AtomicU64::new(state.published_tail), + state, + extents: SkipMap::new(), + }) + } + + pub(super) fn end(&self) -> u64 { + self.end.load(Ordering::Acquire) + } + + pub(super) fn append(&self, extents: &[Extent], end: u64) { + for extent in extents { + self.extents.insert(extent.logical_start, extent.clone()); + } + // Readers acquire the end only after every immutable mapping is visible. + self.end.store(end, Ordering::Release); + } + + pub(super) fn resolve(&self, offset: u64) -> Result { + let entry = self + .extents + .upper_bound(Included(&offset)) + .ok_or_else(|| StreamError::Corruption("local append mapping is absent".into()))?; + let extent = entry.value(); + resolve_group( + extent.chunk_id, + extent.logical_start, + extent.logical_end, + extent.frame_start, + offset, + ) + } +} + +impl ChunkStream { + #[must_use] + pub fn view_source(&self) -> StreamViewSource { + self.view_source + } + + /// Orders a chunk record checkpoint after earlier queued appends and captures + /// that finite view. Later appends do not extend the returned handle. + /// + /// # Errors + /// Returns a publication, stale-writer or stalled-worker error. + pub async fn capture_read_only(&self) -> Result { + if self.view_source != StreamViewSource::LiveOwner { + return Ok(self.clone()); + } + if self.closed.load(Ordering::Acquire) { + let view = self.read_view.load_full(); + return if view.state.closed + || self + .authority + .as_ref() + .is_some_and(|authority| authority.is_released()) + { + Ok(self.captured(view)) + } else { + Err(StreamError::WriteStalled) + }; + } + let (completion, result) = oneshot::channel(); + self.sender + .send(Command::Capture(completion)) + .await + .map_err(|_| StreamError::WriteStalled)?; + let view = result.await.map_err(|_| StreamError::WriteStalled)??; + Ok(self.captured(view)) + } + + fn captured(&self, view: Arc) -> Self { + let mut reader = self.clone(); + reader.authority = None; + reader.state = Arc::new(ArcSwap::from(Arc::clone(&view.state))); + reader.read_view = Arc::new(ArcSwap::from(view)); + reader.closed = Arc::new(AtomicBool::new(true)); + reader.view_source = StreamViewSource::OwnerCheckpoint; + reader + } + + /// Persists the finite chunk record prefix captured by an ordered owner barrier. + /// # Errors + /// Returns the same errors as `capture_read_only`. + pub async fn checkpoint(&self) -> Result { + Ok(self.capture_read_only().await?.tail()) + } +} diff --git a/lib/crowdb-chunk-stream/src/stream/worker.rs b/lib/crowdb-chunk-stream/src/stream/worker.rs new file mode 100644 index 000000000..bfdeecfed --- /dev/null +++ b/lib/crowdb-chunk-stream/src/stream/worker.rs @@ -0,0 +1,334 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Queue dispatch, capacity preparation and lifecycle commands. + +use super::append::write_with_recovery; +use super::capacity::{fits_active, prepare_active_for_append}; +use super::publication::publish_state; +use super::{ + finish_failed, liveness, preparation, successor, AppendRequest, BatchFailure, Command, ReadView, + WorkerState, +}; +use crate::perf::{observe, perf, PhaseTimer}; +use crate::{Result, StreamError}; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc; + +pub(super) async fn run_worker(mut state: WorkerState, mut receiver: mpsc::Receiver) { + let _renewal = liveness::start(&state); + let mut pending = None; + let mut liveness = tokio::time::interval(state.config.liveness_interval); + liveness.tick().await; + loop { + successor::start(&mut state); + let command = if let Some(command) = pending.take() { + command + } else { + tokio::select! { + command = receiver.recv() => match command { + Some(command) => command, + None => break, + }, + () = state.liveness_changed.notified() => { + liveness::maintain(&mut state).await; + continue; + } + _ = liveness.tick() => { + liveness::maintain(&mut state).await; + continue; + } + } + }; + match command { + Command::Append(first) => { + process_append_batch(&mut state, first, &mut receiver, &mut pending).await; + } + Command::Trim { offset, completion } => { + let _ = completion.send(process_trim(&mut state, offset).await); + } + Command::Capture(completion) => { + let _ = completion.send(process_capture(&mut state).await); + } + Command::Release(completion) => { + let _ = completion.send(process_release(&mut state).await); + } + Command::Reclaim { + retained_generation, + max_records, + completion, + } => { + let _ = completion.send( + super::collection::reclaim_metadata(&mut state, retained_generation, max_records).await, + ); + } + Command::Close(completion) => { + let result = process_close(&mut state).await; + let _ = completion.send(result); + break; + } + } + } + // Never detach initialization IO that could outlive its owner or a seal. + if let Err(error) = preparation::drain(&mut state).await { + state.log_unavailable("drain_capacity", &error); + } + if let Err(error) = successor::discard(&mut state).await { + state.log_unavailable("discard_successor", &error); + } + if let Some(authority) = &state.authority { + authority.fence(); + } + state.closed_view.store(true, Ordering::Release); +} + +async fn process_capture(state: &mut WorkerState) -> Result> { + let _timer = PhaseTimer::start(&perf().checkpoint); + if state.stalled { + return Err(StreamError::WriteStalled); + } + state + .authority + .as_ref() + .ok_or(StreamError::WriteStalled)? + .check()?; + if state.progress.logical_tail != state.published_view.load().published_tail { + publish_state(state).await?; + } else if state + .metadata + .load_tail(state.stream_name) + .await? + .as_ref() + .map(crate::StreamChunkRecordSnapshot::record) + != Some(state.snapshot.record()) + { + state.stalled = true; + return Err(StreamError::StaleWriter); + } + Ok(Arc::new( + ReadView::new(state.metadata.clone(), state.published_view.load_full()).await?, + )) +} + +async fn process_append_batch( + state: &mut WorkerState, + first: AppendRequest, + receiver: &mut mpsc::Receiver, + pending: &mut Option, +) { + liveness::maintain(state).await; + if state.stalled || state.progress.closed { + finish_failed(state, vec![first], &StreamError::WriteStalled); + return; + } + if let Err(error) = first.authority.check_io() { + state.stalled = true; + finish_failed(state, vec![first], &error); + return; + } + observe(&perf().queue, first.enqueued_at); + let first_len = match first.physical_len() { + Ok(length) => length, + Err(error) => { + finish_failed(state, vec![first], &error); + return; + } + }; + let capacity_timer = PhaseTimer::start(&perf().capacity); + let preparation = prepare_active_for_append(state, first_len).await; + drop(capacity_timer); + if let Err(error) = preparation { + state.stalled = true; + finish_failed(state, vec![first], &error); + return; + } + + let mut requests = vec![first]; + let mut bytes = first_len; + while requests.len() < state.config.batch_requests && bytes < state.config.batch_bytes { + match receiver.try_recv() { + Ok(Command::Append(request)) + if bytes + .checked_add(request.physical_len().unwrap_or(usize::MAX)) + .is_some_and(|total| total <= state.config.batch_bytes && fits_active(state, total)) => + { + observe(&perf().queue, request.enqueued_at); + bytes += request.physical_len().expect("checked above"); + requests.push(request); + } + Ok(command) => { + *pending = Some(command); + break; + } + Err(_) => break, + } + } + preparation::start(state, bytes); + successor::start(state); + let result = write_with_recovery(state, &requests, bytes).await; + match result { + Ok(ranges) => { + for (request, range) in requests.into_iter().zip(ranges) { + state + .queued_bytes + .fetch_sub(request.logical_len() as u64, Ordering::AcqRel); + state.queued_requests.fetch_sub(1, Ordering::AcqRel); + let result = request.authority.check_ack().map(|()| range); + if result.is_ok() { + state.metrics.completed.fetch_add(1, Ordering::Relaxed); + state + .metrics + .logical_append_bytes + .fetch_add(request.logical_len() as u64, Ordering::Relaxed); + observe(&perf().append, request.enqueued_at); + } + let _ = request.completion.send(result); + } + } + Err(BatchFailure::MirrorWrite(error) | BatchFailure::Other(error)) => { + state.stalled = true; + finish_failed(state, requests, &error); + } + } +} + +impl WorkerState { + pub(super) fn log_unavailable(&mut self, stage: &'static str, error: &StreamError) { + let now = Instant::now(); + let should_log = self + .last_unavailable_log + .map_or(true, |last| now.duration_since(last) >= Duration::from_secs(1)); + if should_log { + self.last_unavailable_log = Some(now); + tracing::warn!( + stream_high = self.stream_name.high, + stream_low = self.stream_name.low, + writer_epoch = self.writer_epoch, + stage, + %error, + "chunk-stream storage remains unavailable; retrying" + ); + } + } +} + +pub(super) async fn rotate_externally_sealed_active(state: &mut WorkerState) -> Result { + preparation::drain(state).await?; + let Some(active) = state.progress.active.clone() else { + return Ok(false); + }; + let durable = state + .chunks + .durable_cursor(active.chunk_id, state.writer_epoch) + .await?; + if !durable.sealed { + return Ok(false); + } + if durable.offset != active.acknowledged_cursor { + return Err(StreamError::Corruption( + "externally sealed active chunk has an unexpected durable cursor".into(), + )); + } + + let tail = state.tail_view.load(Ordering::Acquire); + state.checkpoint = None; + state.progress.active = None; + state.progress.logical_tail = tail; + let mut successor = successor::take(state).await?; + successor.logical_start = tail; + state.progress.active = Some(successor); + publish_state(state).await?; + successor::start(state); + state.stalled = false; + state.metrics.rollovers.fetch_add(1, Ordering::Relaxed); + Ok(true) +} + +async fn process_trim(state: &mut WorkerState, offset: u64) -> Result { + if state + .authority + .as_ref() + .is_some_and(|authority| authority.is_released()) + { + return if offset == state.progress.trim_offset { + Ok(0) + } else { + Err(StreamError::StaleWriter) + }; + } + if state.stalled { + return Err(StreamError::WriteStalled); + } + state + .authority + .as_ref() + .ok_or(StreamError::WriteStalled)? + .check()?; + let tail = state.tail_view.load(Ordering::Acquire); + if offset < state.progress.trim_offset || offset > tail { + return Err(StreamError::InvalidRequest( + "trim offset regresses or exceeds durable tail".into(), + )); + } + if offset != state.progress.trim_offset { + state.progress.trim_offset = offset; + publish_state(state).await?; + } + let reclaimed = super::collection::reclaim_chunks(state).await?; + state + .metrics + .reclaimed_bytes + .fetch_add(reclaimed, Ordering::Relaxed); + Ok(reclaimed) +} + +async fn process_close(state: &mut WorkerState) -> Result<()> { + if state.stalled { + return Err(StreamError::WriteStalled); + } + state + .authority + .as_ref() + .ok_or(StreamError::WriteStalled)? + .check()?; + if state.progress.closed { + return Ok(()); + } + preparation::drain(state).await?; + successor::discard(state).await?; + if let Some(active) = state.progress.active.take() { + state + .chunks + .seal(active.chunk_id, state.writer_epoch, active.acknowledged_cursor) + .await?; + let tail = state.tail_view.load(Ordering::Acquire); + state.progress.logical_tail = tail; + } + state.checkpoint = None; + state.progress.closed = true; + publish_state(state).await?; + if let Some(authority) = &state.authority { + authority.release().await?; + } + state.closed_view.store(true, Ordering::Release); + Ok(()) +} + +async fn process_release(state: &mut WorkerState) -> Result<()> { + if state.stalled { + return Err(StreamError::WriteStalled); + } + preparation::drain(state).await?; + successor::discard(state).await?; + if state.progress.logical_tail != state.published_view.load().published_tail { + publish_state(state).await?; + } + state + .authority + .as_ref() + .ok_or(StreamError::WriteStalled)? + .release() + .await +} diff --git a/lib/crowdb-chunk-stream/tests/benchmark_target_test.rs b/lib/crowdb-chunk-stream/tests/benchmark_target_test.rs new file mode 100644 index 000000000..c569513bb --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/benchmark_target_test.rs @@ -0,0 +1,18 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "../benches/production_bounds/config.rs"] +mod config; + +#[test] +fn production_measurement_requires_an_explicit_existing_target() { + let error = config::BenchConfig::parse(std::iter::empty()).unwrap_err(); + assert!(error.contains("--management-seed is required"), "{error}"); + let configured = config::BenchConfig::parse( + ["--management-seed", "http://127.0.0.1:19347"] + .into_iter() + .map(str::to_owned), + ) + .unwrap(); + assert_eq!(configured.management_seed, "http://127.0.0.1:19347"); +} diff --git a/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs b/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs new file mode 100644 index 000000000..deb0d6dfd --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/capacity_plan_test.rs @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::time::Duration; + +use crowdb_chunk_stream::capacity_target_for_tests; + +const MIB: u64 = 1024 * 1024; + +#[test] +fn bandwidth_and_preparation_latency_increase_the_reserved_capacity() { + let plan = |bytes, millis| { + capacity_target_for_tests( + &[(bytes, Duration::from_millis(100))], + Duration::from_millis(millis), + 0, + 8 * MIB, + 256 * MIB, + 4096, + ) + }; + assert_eq!(plan(MIB, 50), None, "a slow writer has ample ready capacity"); + let fast = plan(20 * MIB, 50).unwrap(); + let slower_preparation = plan(20 * MIB, 100).unwrap(); + assert_eq!(fast, 40 * MIB + 4096); + assert_eq!(slower_preparation, 80 * MIB + 4096); +} + +#[test] +fn refill_starts_with_two_preparation_intervals_still_ready() { + let request = |capacity| { + capacity_target_for_tests( + &[(20 * MIB, Duration::from_millis(100))], + Duration::from_millis(50), + MIB, + capacity, + 256 * MIB, + 4096, + ) + }; + assert!(request(21 * MIB + 4096 + 1).is_none()); + assert!(request(21 * MIB + 4096).is_some()); +} + +#[test] +fn cold_start_extreme_rates_and_chunk_end_remain_bounded() { + assert_eq!( + capacity_target_for_tests(&[], Duration::ZERO, 0, MIB, 256 * MIB, 4096), + Some(4 * MIB + 4096) + ); + let writes = [(u64::MAX, Duration::ZERO)]; + assert_eq!( + capacity_target_for_tests(&writes, Duration::MAX, MIB, 2 * MIB, 256 * MIB, 4096), + Some(256 * MIB) + ); + assert_eq!( + capacity_target_for_tests(&writes, Duration::MAX, MIB, 256 * MIB, 256 * MIB, 4096), + None + ); +} + +#[test] +fn one_slow_write_does_not_remove_a_fast_writers_margin() { + let target = capacity_target_for_tests( + &[ + (20 * MIB, Duration::from_millis(100)), + (4096, Duration::from_secs(1)), + ], + Duration::from_millis(50), + 0, + 8 * MIB, + 256 * MIB, + 4096, + ) + .unwrap(); + assert!( + target >= 35 * MIB, + "a slow sample must not immediately shrink the reserve" + ); +} diff --git a/lib/crowdb-chunk-stream/tests/common/grouped_write.rs b/lib/crowdb-chunk-stream/tests/common/grouped_write.rs new file mode 100644 index 000000000..e9b3a2f30 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/common/grouped_write.rs @@ -0,0 +1,55 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_chunk_stream::{ + ChunkId, MirrorStripImage, ProductionStreamChunkStore, StreamChunkStore, StreamName, +}; +use crowdb_protocol::frame::encode_append_group_views; + +#[derive(Default)] +pub struct TestGroupedStrip { + prefix: Vec, +} + +impl TestGroupedStrip { + pub async fn append( + &mut self, + store: &ProductionStreamChunkStore, + name: StreamName, + epoch: u64, + chunk: ChunkId, + payload: &'static [u8], + ) -> u64 { + let offset = self.prefix.len() as u64; + let views = encode_append_group_views(chunk, offset, &Bytes::from_static(payload), 1).unwrap(); + let bytes = Bytes::from( + views + .iter() + .flat_map(|view| view.iter().copied()) + .collect::>(), + ); + self.prefix.extend_from_slice(&bytes); + let complete_end = self.prefix.len() as u64; + let full_image = Bytes::copy_from_slice(&self.prefix); + store + .write_mirrors( + name, + epoch, + chunk, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: offset, + complete_end, + images: &[MirrorStripImage { + block_offset: offset, + data: bytes, + full_image, + }], + }, + ) + .await + .unwrap(); + self.prefix.len() as u64 + } +} diff --git a/lib/crowdb-chunk-stream/tests/common/metadata_server.rs b/lib/crowdb-chunk-stream/tests/common/metadata_server.rs new file mode 100644 index 000000000..875dce289 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/common/metadata_server.rs @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_kv::cluster::group::PxGroup; +use crowdb_kv::cluster::kv_server::KvServer; +use crowdb_kv::cluster::local_replica::{PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv::cluster::px_kv_store::PxKvStore; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; + +pub struct TestMetadataServer { + pub server: Arc, + pub client: Arc, +} + +impl TestMetadataServer { + pub async fn start() -> Self { + let server = Arc::new(PxKvStore::new(1, "127.0.0.1:0".parse().unwrap())); + server.add_group(PxGroup::new( + 1, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + server.start().await.unwrap(); + // This fixture explicitly seeds its one real RPC endpoint; discovery is + // outside the metadata adapter contract exercised here. + let client = Arc::new(CrowdbKvClient::new(ClientConfig::new(Vec::new()))); + client.seed_leader(1, 1, server.listen_addr().unwrap().to_string()); + Self { server, client } + } +} + +impl Drop for TestMetadataServer { + fn drop(&mut self) { + self.server.stop(); + } +} diff --git a/lib/crowdb-chunk-stream/tests/durable_cursor_recovery_test.rs b/lib/crowdb-chunk-stream/tests/durable_cursor_recovery_test.rs new file mode 100644 index 000000000..d2e9721f5 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/durable_cursor_recovery_test.rs @@ -0,0 +1,154 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use bytes::{Bytes, BytesMut}; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, CursorAdvance, StreamChunkStore, StreamConfig, StreamError, + StreamName, StreamPurpose, StreamRecord, StreamRecordState, +}; +use crowdb_protocol::frame::encode_append_group_views; + +#[tokio::test] +async fn durable_cursor_recovers_frames_without_a_new_record_publication() { + check_tail(true, false).await; +} + +#[tokio::test] +async fn an_old_checkpoint_cannot_authorize_discarding_synchronized_bytes() { + check_tail(false, false).await; +} + +#[tokio::test] +async fn a_durable_cursor_does_not_authorize_corrupt_unpublished_frames() { + check_tail(true, true).await; +} + +async fn initial_stream() -> (Arc, ChunkStream, StreamName) { + let store = Arc::new(MemoryStreamStore::new(1024)); + let name = StreamName { high: 51, low: 61 }; + let config = StreamConfig::default(); + let writer = ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Stream, + stream_name: name, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 9, + config.clone(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + writer.append(&[Bytes::from_static(b"old")]).await.unwrap(); + writer.checkpoint().await.unwrap(); + (store, writer, name) +} + +async fn check_tail(advance: bool, corrupt: bool) { + let (store, writer, name) = initial_stream().await; + let published = store + .load_tail_for_tests(name) + .await + .unwrap() + .unwrap() + .record() + .clone(); + let active = &store.active_checkpoint_for_tests(name).await.unwrap(); + let publication_count = store.metadata_publish_count(); + let frames = encode_append_group_views( + active.chunk_id, + active.acknowledged_cursor, + &Bytes::from_static(b"new"), + 1, + ) + .unwrap(); + let mut staging = BytesMut::new(); + for frame in frames { + staging.extend_from_slice(&frame); + } + let data = staging.freeze(); + let end = active.acknowledged_cursor + u64::try_from(data.len()).unwrap(); + store + .write_groups_for_tests(name, 9, active.chunk_id, active.acknowledged_cursor, data.clone()) + .await + .unwrap(); + if advance { + assert_eq!( + store + .advance_cursor( + name, + 9, + active.chunk_id, + active.acknowledged_cursor, + end, + crc32fast::hash(&data) + ) + .await + .unwrap(), + CursorAdvance::Committed + ); + } + if corrupt { + store + .flip_durable_byte( + active.chunk_id, + usize::try_from(active.acknowledged_cursor).unwrap() + + crowdb_protocol::frame::FRAME_HEADER_PREFIX_BYTES, + ) + .await; + } + assert_eq!(store.metadata_publish_count(), publication_count); + assert_eq!( + store + .load_tail_for_tests(name) + .await + .unwrap() + .unwrap() + .record() + .clone(), + published + ); + let observed = store.durable_cursor(active.chunk_id, 9).await.unwrap(); + assert_eq!( + observed.offset, + if advance { end } else { active.acknowledged_cursor } + ); + assert_eq!(writer.tail(), 3); + + let store_for_retention_check = store.clone(); + let retained_bytes = store.synchronized_bytes_for_tests(active.chunk_id).await; + let reopened = ChunkStream::open( + name, + 10, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await; + if corrupt { + assert!(matches!(reopened, Err(StreamError::Corruption(_)))); + return; + } + let reopened = reopened.unwrap(); + let expected = &b"oldnew"[..]; + assert!(!store_for_retention_check.is_released(active.chunk_id).await); + assert_eq!( + store_for_retention_check + .synchronized_bytes_for_tests(active.chunk_id) + .await, + retained_bytes + ); + assert_eq!(reopened.tail(), u64::try_from(expected.len()).unwrap()); + assert_eq!(reopened.read_at(0, expected.len()).await.unwrap(), expected); + assert!(writer.append(&[Bytes::from_static(b"stale")]).await.is_err()); +} diff --git a/lib/crowdb-chunk-stream/tests/grouped_append_test.rs b/lib/crowdb-chunk-stream/tests/grouped_append_test.rs new file mode 100644 index 000000000..6dc161b2c --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/grouped_append_test.rs @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, +}; +use crowdb_protocol::frame::{FrameLayout, MAX_FRAME_GROUP_PAYLOAD_BYTES}; + +#[tokio::test] +async fn every_append_has_one_location_and_subranges_read_only_intersecting_frames() { + let store = Arc::new(MemoryStreamStore::new(8 * 1024 * 1024)); + let name = StreamName::generate(); + let stream = ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Stream, + stream_name: name, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + let sizes = [ + 1, + MAX_FRAME_GROUP_PAYLOAD_BYTES, + MAX_FRAME_GROUP_PAYLOAD_BYTES + 1, + 1024 * 1024, + 4 * 1024 * 1024, + ]; + let mut physical = 0_u64; + for size in sizes { + let payload = Bytes::from( + (0..size) + .map(|index| u8::try_from(index % 251).unwrap()) + .collect::>(), + ); + let range = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + stream.checkpoint().await.unwrap(); + let snapshot = store.load_tail_for_tests(name).await.unwrap().unwrap(); + physical += FrameLayout::StreamGroupV2.physical_length(size as u64).unwrap(); + assert_eq!(snapshot.index().physical_end(), physical); + assert_eq!(stream.metrics().physical_append_bytes, physical); + for offset in [0, size / 2, size - 1] { + let count = (size - offset).min(101); + let before = stream.metrics().physical_read_requests; + assert_eq!( + stream.read_at(range.begin + offset as u64, count).await.unwrap(), + payload.slice(offset..offset + count) + ); + let frames = (offset + count - 1) / MAX_FRAME_GROUP_PAYLOAD_BYTES + - offset / MAX_FRAME_GROUP_PAYLOAD_BYTES + + 1; + assert_eq!(stream.metrics().physical_read_requests - before, frames as u64); + } + if size > MAX_FRAME_GROUP_PAYLOAD_BYTES { + let offset = MAX_FRAME_GROUP_PAYLOAD_BYTES - 3; + let count = (size - offset).min(13); + assert_eq!( + stream.read_at(range.begin + offset as u64, count).await.unwrap(), + payload.slice(offset..offset + count) + ); + } + } + let snapshot = store.load_tail_for_tests(name).await.unwrap().unwrap(); + let locations = snapshot.record().mappings.count() as usize; + assert_eq!( + locations, + sizes.len(), + "frame count cannot multiply append locations" + ); +} diff --git a/lib/crowdb-chunk-stream/tests/idle_authority_test.rs b/lib/crowdb-chunk-stream/tests/idle_authority_test.rs index bc2ba6d78..0a5d14d61 100644 --- a/lib/crowdb-chunk-stream/tests/idle_authority_test.rs +++ b/lib/crowdb-chunk-stream/tests/idle_authority_test.rs @@ -4,8 +4,8 @@ use bytes::Bytes; use crowdb_chunk_stream::memory::MemoryStreamStore; use crowdb_chunk_stream::{ - ChunkStream, StreamBinding, StreamBindingState, StreamChunkStore, StreamConfig, StreamError, - StreamMetadataStore, StreamName, StreamPurpose, + ChunkStream, StreamChunkStore, StreamConfig, StreamError, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, }; use std::{sync::Arc, time::Duration}; @@ -27,13 +27,15 @@ async fn assert_retired_idle_writer(new_epoch: u64) { ..StreamConfig::default() }; let old = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: StreamPurpose::Stream, stream_name: name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("idle-authority".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("idle-authority".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 9, config.clone(), @@ -58,7 +60,11 @@ async fn assert_retired_idle_writer(new_epoch: u64) { ) .await .unwrap(); - let head = store.load_current(name).await.unwrap(); + let head = store + .load_tail_for_tests(name) + .await + .unwrap() + .map(|snapshot| snapshot.record().clone()); let publications = store.metadata_publish_count(); for _ in 0..3 { tokio::time::advance(Duration::from_secs(10)).await; @@ -70,10 +76,17 @@ async fn assert_retired_idle_writer(new_epoch: u64) { "old writer must stop before renewing its chunk" ); assert_eq!(store.metadata_publish_count(), publications); - assert_eq!(store.load_current(name).await.unwrap(), head); + assert_eq!( + store + .load_tail_for_tests(name) + .await + .unwrap() + .map(|snapshot| snapshot.record().clone()), + head + ); assert_eq!( old.append(&[Bytes::from_static(b"stale")]).await, - Err(StreamError::WriteStalled) + Err(StreamError::StaleWriter) ); new.append(&[Bytes::from_static(b"new")]).await.unwrap(); assert_eq!(new.read_at(0, 6).await.unwrap(), Bytes::from_static(b"oldnew")); @@ -88,13 +101,15 @@ async fn current_idle_writer_can_rotate_its_own_sealed_chunk() { ..StreamConfig::default() }; let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: StreamPurpose::Stream, stream_name: name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: None, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: None, + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 9, config, @@ -105,8 +120,8 @@ async fn current_idle_writer_can_rotate_its_own_sealed_chunk() { .await .unwrap(); stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); - let head = store.load_current(name).await.unwrap().unwrap(); - let active = head.active.unwrap(); + stream.checkpoint().await.unwrap(); + let active = store.active_checkpoint_for_tests(name).await.unwrap(); store .seal(active.chunk_id, 9, active.acknowledged_cursor) .await diff --git a/lib/crowdb-chunk-stream/tests/live_view_test.rs b/lib/crowdb-chunk-stream/tests/live_view_test.rs new file mode 100644 index 000000000..1c923db4f --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/live_view_test.rs @@ -0,0 +1,160 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use bytes::{Bytes, BytesMut}; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, StreamViewSource, +}; + +async fn create() -> (Arc, ChunkStream) { + let store = Arc::new(MemoryStreamStore::new(1024 * 1024)); + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(StreamName::generate(), None, StreamPurpose::Stream) + }, + 1, + StreamConfig { + ..StreamConfig::default() + }, + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + (store, stream) +} + +#[tokio::test] +async fn append_reads_and_recovery_do_not_require_per_batch_record_publication() { + let (store, stream) = create().await; + let mut expected = BytesMut::new(); + for value in 0_u64..2048 { + let payload = Bytes::copy_from_slice(&value.to_le_bytes()); + let range = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + assert_eq!(range.begin, expected.len() as u64); + expected.extend_from_slice(&payload); + } + assert_eq!(store.metadata_publish_count(), 1, "initial attached record only"); + assert_eq!(stream.metrics().metadata_publications, 1); + assert_eq!(stream.tail(), expected.len() as u64); + assert_eq!(stream.read_at(0, expected.len()).await.unwrap(), expected); + assert_eq!( + stream.metrics().record_cache_misses, + 0, + "local reads use live mappings" + ); + let persisted = store + .load_tail_for_tests(stream.stream_name()) + .await + .unwrap() + .unwrap(); + assert_eq!(persisted.record().logical_end.unwrap(), 0); + assert_eq!(persisted.record().mappings.count(), 0); + + let recovered = ChunkStream::open( + stream.stream_name(), + 2, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await + .unwrap(); + assert_eq!(recovered.tail(), expected.len() as u64); + assert_eq!(recovered.read_at(0, expected.len()).await.unwrap(), expected); + assert!(stream.append(&[Bytes::from_static(b"stale")]).await.is_err()); +} + +#[tokio::test] +async fn owner_checkpoint_is_finite_and_fallback_reports_its_older_boundary() { + let (store, stream) = create().await; + stream.append(&[Bytes::from_static(b"first")]).await.unwrap(); + let captured = stream.capture_read_only().await.unwrap(); + assert_eq!(captured.view_source(), StreamViewSource::OwnerCheckpoint); + assert_eq!(captured.tail(), 5); + let publications = store.metadata_publish_count(); + assert_eq!(stream.checkpoint().await.unwrap(), 5); + assert_eq!( + store.metadata_publish_count(), + publications, + "unchanged checkpoint is free" + ); + + stream.append(&[Bytes::from_static(b"second")]).await.unwrap(); + let fallback = ChunkStream::open_read_only( + stream.stream_name(), + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + assert_eq!(fallback.view_source(), StreamViewSource::PersistedCheckpoint); + assert_eq!(fallback.tail(), 5); + assert_eq!(captured.tail(), 5); + assert_eq!( + captured.read_at(0, 5).await.unwrap(), + Bytes::from_static(b"first") + ); + assert!(fallback.read_at(0, 11).await.is_err()); + assert_eq!( + stream.read_at(0, 11).await.unwrap(), + Bytes::from_static(b"firstsecond") + ); + assert_eq!(store.metadata_publish_count(), publications); + assert_eq!(stream.checkpoint().await.unwrap(), 11); + assert_eq!(captured.tail(), 5); + assert_eq!(fallback.tail(), 5); +} + +#[tokio::test] +async fn a_capture_waits_for_the_queued_synchronized_append() { + let (store, stream) = create().await; + store.pause_writes(); + let writer = stream.clone(); + let append = tokio::spawn(async move { writer.append(&[Bytes::from_static(b"pending")]).await }); + store.wait_for_write().await; + let owner = stream.clone(); + let capture = tokio::spawn(async move { owner.capture_read_only().await }); + tokio::task::yield_now().await; + assert!(!capture.is_finished()); + assert_eq!(stream.tail(), 0); + store.resume_writes(); + append.await.unwrap().unwrap(); + let reader = capture.await.unwrap().unwrap(); + assert_eq!(reader.tail(), 7); + assert_eq!( + reader.read_at(0, 7).await.unwrap(), + Bytes::from_static(b"pending") + ); +} + +#[tokio::test] +async fn an_unchanged_checkpoint_cannot_claim_current_owner_after_takeover() { + let (store, writer) = create().await; + writer.append(&[Bytes::from_static(b"first")]).await.unwrap(); + writer.checkpoint().await.unwrap(); + let new_owner = ChunkStream::open( + writer.stream_name(), + 2, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await + .unwrap(); + assert!(matches!( + writer.capture_read_only().await, + Err(crowdb_chunk_stream::StreamError::StaleWriter) + )); + assert_eq!(new_owner.tail(), 5); +} diff --git a/lib/crowdb-chunk-stream/tests/memory_grouped_progress_test.rs b/lib/crowdb-chunk-stream/tests/memory_grouped_progress_test.rs new file mode 100644 index 000000000..9e0781811 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/memory_grouped_progress_test.rs @@ -0,0 +1,35 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_chunk_stream::{memory::MemoryStreamStore, StreamChunkStore, StreamName, StreamPurpose}; +use crowdb_protocol::frame::encode_append_group_views; + +#[tokio::test] +async fn grouped_memory_progress_remains_local_until_an_explicit_seal() { + let store = MemoryStreamStore::new(1024 * 1024); + let name = StreamName { high: 7, low: 9 }; + let chunk = store + .allocate_mirrored(name, 9, StreamPurpose::Stream) + .await + .unwrap(); + let mut offset = 0; + for payload in [b"first".as_slice(), b"second".as_slice()] { + let views = + encode_append_group_views(chunk.chunk_id, offset, &Bytes::copy_from_slice(payload), 1).unwrap(); + let data = Bytes::from(views.concat()); + let length = data.len() as u64; + store + .write_groups_for_tests(name, 9, chunk.chunk_id, offset, data) + .await + .unwrap(); + offset += length; + assert_eq!(store.durable_cursor(chunk.chunk_id, 9).await.unwrap().offset, 0); + } + assert_eq!(store.cursor_advance_count(), 0); + store.seal(chunk.chunk_id, 9, offset).await.unwrap(); + assert_eq!( + store.durable_cursor(chunk.chunk_id, 9).await.unwrap().offset, + offset + ); +} diff --git a/lib/crowdb-chunk-stream/tests/metadata_test.rs b/lib/crowdb-chunk-stream/tests/metadata_test.rs deleted file mode 100644 index b23c08637..000000000 --- a/lib/crowdb-chunk-stream/tests/metadata_test.rs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use crowdb_chunk_stream::{ - resolve_extent, validate_manifest, ActiveChunkDescriptor, StreamExtentPage, StreamExtentPageFence, - StreamManifest, StreamName, -}; -use crowdb_protocol::common::ChunkId; - -fn name() -> StreamName { - StreamName { high: 1, low: 2 } -} - -fn extent_page() -> StreamExtentPage { - StreamExtentPage { - stream_name: name(), - writer_epoch: 7, - generation: 3, - page_index: 0, - chunk_ids: vec![ - ChunkId { - high: 4_u64 << 56, - low: 10, - }, - ChunkId { - high: 4_u64 << 56, - low: 11, - }, - ], - logical_offsets: vec![0, 10, 25], - physical_offsets: vec![4, 8], - frame_lengths: vec![44, 49], - } -} - -#[test] -fn resolves_first_interior_and_boundary_offsets() { - let page = extent_page(); - let first = resolve_extent(&page, 0).unwrap(); - assert_eq!(first.chunk_id.low, 10); - assert_eq!(first.frame_offset, 4); - assert_eq!(first.payload_offset, 14); - assert_eq!(first.available, 10); - - let interior = resolve_extent(&page, 6).unwrap(); - assert_eq!(interior.frame_offset, 4); - assert_eq!(interior.payload_offset, 20); - assert_eq!(interior.available, 4); - - let boundary = resolve_extent(&page, 10).unwrap(); - assert_eq!(boundary.chunk_id.low, 11); - assert_eq!(boundary.frame_offset, 8); - assert_eq!(boundary.payload_offset, 14); - assert_eq!(boundary.available, 15); -} - -#[test] -fn rejects_malformed_extent_geometry() { - let mut page = extent_page(); - page.logical_offsets = vec![0, 10]; - assert!(resolve_extent(&page, 0).is_err()); - - let mut page = extent_page(); - page.logical_offsets = vec![0, 10, 10]; - assert!(resolve_extent(&page, 10).is_err()); - - let mut page = extent_page(); - page.physical_offsets[1] = u64::MAX; - assert!(resolve_extent(&page, 10).is_err()); -} - -#[test] -fn manifest_tail_includes_only_acknowledged_active_bytes() { - let page = extent_page(); - let manifest = StreamManifest { - purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, - stream_name: name(), - metadata_group_id: 7, - writer_epoch: 7, - generation: 3, - trim_offset: 5, - sealed_tail: 25, - active: Some(ActiveChunkDescriptor { - chunk_id: ChunkId { - high: 4_u64 << 56, - low: 12, - }, - physical_start: 10, - logical_start: 25, - acknowledged_cursor: 18, - capacity: 100, - }), - extent_pages: vec![StreamExtentPageFence { - page_index: 0, - first_logical: 0, - end_logical: 25, - }], - previous_generation: Some(2), - closed: false, - }; - assert_eq!(validate_manifest(&manifest, &[page]).unwrap(), 33); -} diff --git a/lib/crowdb-chunk-stream/tests/mirror_completion_test.rs b/lib/crowdb-chunk-stream/tests/mirror_completion_test.rs new file mode 100644 index 000000000..b4a7a8e75 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/mirror_completion_test.rs @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamChunkStore, StreamConfig, StreamError, StreamName, + StreamPurpose, StreamRecord, StreamRecordState, +}; +use std::sync::Arc; + +async fn setup() -> (Arc, ChunkStream) { + let store = Arc::new(MemoryStreamStore::new(8 * 1024 * 1024)); + let name = StreamName { high: 743, low: 9 }; + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + (store, stream) +} + +#[tokio::test] +async fn acknowledged_multiframe_writes_survive_reopen_without_any_cursor_commit() { + let (store, stream) = setup().await; + let payload = Bytes::from(vec![0x6b; 2 * 1024 * 1024]); + let first = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + stream.append(&[Bytes::from_static(b"last")]).await.unwrap(); + assert_eq!(store.cursor_advance_count(), 0); + assert_eq!( + store.metadata_publish_count(), + 1, + "initial attached chunk record only" + ); + assert_eq!( + store + .durable_cursor(first.chunk_id.unwrap(), 1) + .await + .unwrap() + .offset, + 0 + ); + assert_eq!(stream.read_at(0, payload.len()).await.unwrap(), payload); + let recovered = ChunkStream::open( + stream.stream_name(), + 2, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + assert_eq!(recovered.tail(), payload.len() as u64 + 4); + assert_eq!(recovered.read_at(0, payload.len()).await.unwrap(), payload); + assert_eq!( + recovered.read_at(payload.len() as u64, 4).await.unwrap(), + b"last"[..] + ); + assert_eq!( + store.cursor_advance_count(), + 0, + "recovery seals the observed boundary directly" + ); + assert!(matches!( + stream.append(&[Bytes::from_static(b"stale")]).await, + Err(StreamError::StaleWriter) + )); +} + +#[tokio::test] +async fn checkpoint_commits_many_appends_once_and_recovery_discards_an_incomplete_group() { + let (store, stream) = setup().await; + for _ in 0..100 { + stream.append(&[Bytes::from_static(b"good")]).await.unwrap(); + } + assert_eq!(store.cursor_advance_count(), 0); + assert_eq!(stream.checkpoint().await.unwrap(), 400); + assert_eq!(store.cursor_advance_count(), 1); + let active = store + .active_checkpoint_for_tests(stream.stream_name()) + .await + .unwrap(); + let views = crowdb_protocol::frame::encode_append_group_views( + active.chunk_id, + active.acknowledged_cursor, + &Bytes::from(vec![7; 100_000]), + 1, + ) + .unwrap(); + let mut torn: Vec<_> = views.iter().flat_map(|view| view.iter().copied()).collect(); + // The first frame is complete, but its grouped append still needs followers. + torn.truncate(crowdb_protocol::frame::MAX_FRAME_BYTES); + store + .stage_incomplete_tail_for_tests(active.chunk_id, Bytes::from(torn)) + .await; + let recovered = ChunkStream::open( + stream.stream_name(), + 2, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + assert_eq!(recovered.tail(), 400); + assert_eq!(recovered.read_at(396, 4).await.unwrap(), b"good"[..]); + assert_eq!( + recovered + .append(&[Bytes::from_static(b"next")]) + .await + .unwrap() + .begin, + 400 + ); +} + +#[tokio::test] +async fn revoked_owner_cannot_acknowledge_a_previously_admitted_write() { + let (store, stream) = setup().await; + store.pause_writes(); + let old = stream.clone(); + let write = tokio::spawn(async move { old.append(&[Bytes::from_static(b"pending")]).await }); + store.wait_for_write().await; + store.fence_writer_for_tests(stream.stream_name()).await; + store.resume_writes(); + assert!(matches!(write.await.unwrap(), Err(StreamError::StaleWriter))); + assert_eq!(stream.tail(), 0); + assert_eq!(store.cursor_advance_count(), 0); + assert!( + stream.release_writer().await.is_err(), + "a stalled write cannot produce a clean release proof" + ); +} + +#[tokio::test(start_paused = true)] +async fn chunk_liveness_renews_while_a_data_write_is_blocked() { + let store = Arc::new(MemoryStreamStore::new(1024)); + let name = StreamName { high: 743, low: 10 }; + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }, + 1, + StreamConfig { + liveness_interval: std::time::Duration::from_secs(1), + ..StreamConfig::default() + }, + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + stream.append(&[Bytes::from_static(b"first")]).await.unwrap(); + store.pause_writes(); + let old = stream.clone(); + let write = tokio::spawn(async move { old.append(&[Bytes::from_static(b"pending")]).await }); + store.wait_for_write().await; + tokio::time::advance(std::time::Duration::from_secs(1)).await; + tokio::task::yield_now().await; + assert_eq!( + store.liveness_renew_count(), + 2, + "both the attached chunk and its ready successor renew while data IO is blocked" + ); + assert!(!write.is_finished()); + assert_eq!(store.cursor_advance_count(), 0); + store.resume_writes(); + assert_eq!(write.await.unwrap().unwrap().end, 12); +} diff --git a/lib/crowdb-chunk-stream/tests/observation_test.rs b/lib/crowdb-chunk-stream/tests/observation_test.rs index 75556ace7..d06e8a265 100644 --- a/lib/crowdb-chunk-stream/tests/observation_test.rs +++ b/lib/crowdb-chunk-stream/tests/observation_test.rs @@ -5,24 +5,25 @@ use std::sync::Arc; use bytes::Bytes; use crowdb_chunk_stream::{ - memory::MemoryStreamStore, ChunkStream, StreamBinding, StreamBindingState, StreamConfig, StreamName, + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamRecord, StreamRecordState, }; #[tokio::test] -async fn published_extent_index_is_bounded_fenced_and_reads_no_data() { - let store = Arc::new(MemoryStreamStore::new(4096)); +async fn published_chunk_record_summary_is_bounded_fenced_and_reads_no_data() { + let store = Arc::new(MemoryStreamStore::new(65536)); let stream = ChunkStream::create( - StreamBinding { + StreamRecord { stream_name: StreamName { high: 1, low: 2 }, purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: None, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: None, + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, u64::MAX, StreamConfig { - extent_page_entries: 1, ..StreamConfig::default() }, store.clone(), @@ -34,16 +35,15 @@ async fn published_extent_index_is_bounded_fenced_and_reads_no_data() { for _ in 0..105 { stream.append(&[Bytes::from_static(b"x")]).await.unwrap(); } + stream.checkpoint().await.unwrap(); let before = stream.metrics(); let publications = store.metadata_publish_count(); let first = stream.observe_metadata(None, 0).unwrap(); assert_eq!(first.writer_epoch, u64::MAX); - assert_eq!(first.extent_pages.len(), 100); - assert_eq!(first.next_offset, Some(100)); - let second = stream.observe_metadata(Some(first.generation), 100).unwrap(); - assert_eq!(second.extent_pages.len(), 5); - assert_eq!(second.extent_pages[0].first_logical, 100); - assert_eq!(second.next_offset, None); + assert_eq!(first.chunk.mapping_count, 105); + assert_eq!(first.chunk.logical_start, 0); + assert_eq!(first.chunk.logical_end, 105); + assert_eq!(first.next_offset, None); assert!(stream.observe_metadata(None, 100).is_err()); assert!(stream .observe_metadata(Some(first.generation), usize::MAX) @@ -51,13 +51,24 @@ async fn published_extent_index_is_bounded_fenced_and_reads_no_data() { assert_eq!(stream.metrics(), before); assert_eq!(store.metadata_publish_count(), publications); stream.append(&[Bytes::from_static(b"y")]).await.unwrap(); + assert_eq!( + stream.observe_metadata(None, 0).unwrap().generation, + first.generation + ); + stream.checkpoint().await.unwrap(); assert!(stream.observe_metadata(Some(first.generation), 100).is_err()); let current = stream.observe_metadata(None, 0).unwrap(); assert!(current.generation > first.generation); - assert!(current.active.is_some()); + assert_eq!( + current.chunk.state, + crowdb_protocol::chunk_stream::StreamChunkState::Active + ); stream.close().await.unwrap(); let closed = stream.observe_metadata(None, 0).unwrap(); assert!(closed.closed); - assert!(closed.active.is_none()); - assert_eq!(closed.sealed_tail, 106); + assert_eq!( + closed.chunk.state, + crowdb_protocol::chunk_stream::StreamChunkState::Sealed + ); + assert_eq!(closed.published_tail, 106); } diff --git a/lib/crowdb-chunk-stream/tests/ordered_kv_record_test.rs b/lib/crowdb-chunk-stream/tests/ordered_kv_record_test.rs new file mode 100644 index 000000000..b9f8c75e9 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/ordered_kv_record_test.rs @@ -0,0 +1,283 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/metadata_server.rs"] +mod metadata_server; +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, KvStreamMetadataStore, StreamConfig, StreamMetadataStore, + StreamName, StreamPurpose, StreamRecord, StreamRecordState, +}; +use crowdb_kv_client::ReadMode; +use crowdb_protocol::{ + chunk_stream::{decode_stream_chunk_record, StreamChunkState}, + key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}, +}; +use std::sync::Arc; + +#[tokio::test] +async fn actual_kv_reverse_seek_uses_ordered_keys_and_never_updates_historical_records() { + let server = metadata_server::TestMetadataServer::start().await; + let chunks = Arc::new(MemoryStreamStore::new(128)); + let metadata = Arc::new(KvStreamMetadataStore::new(server.client.clone(), 1, 1).unwrap()); + let name = StreamName::generate(); + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, Some("test".into()), StreamPurpose::Stream) + }, + 1, + StreamConfig::default(), + chunks.clone(), + metadata.clone(), + chunks.clone(), + ) + .await + .unwrap(); + for byte in 1..=16 { + stream.append(&[Bytes::from(vec![byte; 100])]).await.unwrap(); + } + stream.checkpoint().await.unwrap(); + let records = server + .client + .scan( + 1, + 1, + &StreamChunkRecordKey::attached_prefix(name), + &[], + &[], + 32, + ReadMode::Linearizable, + None, + false, + None, + ) + .await + .unwrap(); + assert_eq!(records.items.len(), 16); + let first = records.items[0].clone(); + let store = metadata.record_store(1).unwrap(); + for index in 0..16 { + let snapshot = store.locate(name, index * 100).await.unwrap(); + assert_eq!(snapshot.record().logical_start, Some(index * 100)); + assert_eq!(snapshot.record().logical_end, Some((index + 1) * 100)); + assert_eq!( + stream.read_at(index * 100, 1).await.unwrap(), + Bytes::from(vec![u8::try_from(index + 1).unwrap()]) + ); + } + let tail = store.load_tail(name).await.unwrap().unwrap(); + assert_eq!(tail.record().state, StreamChunkState::Active); + assert_eq!(tail.record().logical_start, Some(1500)); + for byte in 17..=24 { + stream.append(&[Bytes::from(vec![byte; 100])]).await.unwrap(); + } + let first_again = server + .client + .get(1, 1, &first.0, ReadMode::Linearizable, None) + .await + .unwrap(); + let crowdb_kv_client::GetOutcome::Found { value, .. } = first_again else { + panic!("sealed record missing") + }; + assert_eq!(value, first.1); + let key = StreamChunkRecordKey::from_bytes(&first.0).unwrap(); + assert_eq!(key.order, StreamChunkRecordOrder::Attached(0)); + assert_eq!(decode_stream_chunk_record(&value).unwrap().logical_end, Some(100)); + let captured = stream.capture_read_only().await.unwrap(); + assert_eq!(captured.tail(), 2400); + stream.close().await.unwrap(); + let reopened = ChunkStream::open(name, 2, StreamConfig::default(), chunks.clone(), metadata, chunks) + .await + .unwrap(); + assert_eq!(reopened.tail(), 2400); + assert_eq!( + reopened.read_at(2300, 100).await.unwrap(), + Bytes::from(vec![24; 100]) + ); +} + +#[tokio::test] +async fn actual_kv_collection_uses_bounded_keys_and_releases_only_trimmed_chunks() { + let server = metadata_server::TestMetadataServer::start().await; + let chunks = Arc::new(MemoryStreamStore::new(128)); + let metadata = Arc::new(KvStreamMetadataStore::new(server.client.clone(), 1, 1).unwrap()); + let name = StreamName::generate(); + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }, + 1, + StreamConfig::default(), + chunks.clone(), + metadata.clone(), + chunks.clone(), + ) + .await + .unwrap(); + let first = stream.append(&[Bytes::from(vec![1; 100])]).await.unwrap(); + stream.append(&[Bytes::from(vec![2; 100])]).await.unwrap(); + stream.append(&[Bytes::from(vec![3; 100])]).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + match stream.trim_prefix(100).await { + Ok(_) | Err(crowdb_chunk_stream::StreamError::Backpressure) => {} + Err(error) => panic!("actual KV collection failed: {error}"), + } + if chunks.is_released(first.chunk_id.unwrap()).await { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(stream.read_at(100, 100).await.unwrap(), Bytes::from(vec![2; 100])); + assert!(metadata.record_store(1).unwrap().locate(name, 0).await.is_err()); + stream.close().await.unwrap(); +} + +#[tokio::test] +async fn checkpoint_publication_waits_for_an_inflight_tail_guard_update() { + use crowdb_kv_client::GetOutcome; + use std::time::Duration; + + let server = metadata_server::TestMetadataServer::start().await; + let chunks = Arc::new(MemoryStreamStore::new(1024)); + let metadata = Arc::new(KvStreamMetadataStore::new(server.client.clone(), 1, 1).unwrap()); + let name = StreamName::generate(); + let _stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }, + 1, + StreamConfig::default(), + chunks.clone(), + metadata.clone(), + chunks, + ) + .await + .unwrap(); + let store = metadata.record_store(1).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while store.candidates(name).await.unwrap().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let tail = store.load_tail(name).await.unwrap().unwrap(); + let key = StreamChunkRecordKey { + stream_id: name, + order: StreamChunkRecordOrder::Attached(0), + } + .to_bytes(); + let GetOutcome::Found { value, revision } = server + .client + .get(1, 1, &key, ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!("tail guard is absent"); + }; + let group = server.server.get_group(1).unwrap(); + let permits = group.try_acquire_all_inflight_permits(); + let client = Arc::clone(&server.client); + let guard = key.clone(); + let updating = tokio::spawn(async move { client.put_cas(1, 1, &guard, &value, revision).await }); + tokio::time::timeout(Duration::from_secs(1), async { + while !group.conditional_write_pending_for_tests(&key) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let mut checkpoint = tail.record().clone(); + checkpoint.generation += 1; + let publishing = store.publish(Some(&tail), checkpoint.clone(), None, None); + let release = async { + tokio::time::sleep(Duration::from_millis(20)).await; + drop(permits); + }; + let (result, ()) = tokio::join!(publishing, release); + updating.await.unwrap().unwrap(); + let published = result.expect("a short tail-guard CAS must not fail checkpoint publication"); + assert_eq!(published.record(), &checkpoint); + assert_eq!( + store.load_tail(name).await.unwrap().unwrap().record(), + &checkpoint + ); +} + +#[tokio::test] +async fn read_pin_waits_for_an_inflight_tail_guard_update() { + use crowdb_kv_client::GetOutcome; + use std::time::Duration; + + let server = metadata_server::TestMetadataServer::start().await; + let chunks = Arc::new(MemoryStreamStore::new(1024)); + let metadata = Arc::new(KvStreamMetadataStore::new(server.client.clone(), 1, 1).unwrap()); + let name = StreamName::generate(); + let _stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }, + 1, + StreamConfig::default(), + chunks.clone(), + metadata.clone(), + chunks, + ) + .await + .unwrap(); + let store = metadata.record_store(1).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while store.candidates(name).await.unwrap().is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let tail = store.load_tail(name).await.unwrap().unwrap(); + let key = StreamChunkRecordKey { + stream_id: name, + order: StreamChunkRecordOrder::Attached(0), + } + .to_bytes(); + let GetOutcome::Found { value, revision } = server + .client + .get(1, 1, &key, ReadMode::Linearizable, None) + .await + .unwrap() + else { + panic!("tail guard is absent"); + }; + let group = server.server.get_group(1).unwrap(); + let permits = group.try_acquire_all_inflight_permits(); + let client = Arc::clone(&server.client); + let guard = key.clone(); + let updating = tokio::spawn(async move { client.put_cas(1, 1, &guard, &value, revision).await }); + tokio::time::timeout(Duration::from_secs(1), async { + while !group.conditional_write_pending_for_tests(&key) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let pinning = store.pin_read(&tail); + let release = async { + tokio::time::sleep(Duration::from_millis(100)).await; + drop(permits); + }; + let (result, ()) = tokio::join!(pinning, release); + updating.await.unwrap().unwrap(); + let pin = result.expect("a short tail-guard CAS must not exhaust the read-pin budget"); + pin.check().unwrap(); + assert_eq!( + store.load_tail(name).await.unwrap().unwrap().record(), + tail.record() + ); +} diff --git a/lib/crowdb-chunk-stream/tests/ordered_record_fault_test.rs b/lib/crowdb-chunk-stream/tests/ordered_record_fault_test.rs new file mode 100644 index 000000000..c2689f7bf --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/ordered_record_fault_test.rs @@ -0,0 +1,297 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamError, StreamMetadataStore, StreamName, + StreamPurpose, StreamRecord, StreamRecordState, StreamRegistry, +}; +use crowdb_protocol::chunk_stream::{encode_stream_chunk_record, StreamMappings}; +use std::{sync::Arc, time::Duration}; + +#[path = "ordered_record_fault_test/initialization.rs"] +mod initialization; + +async fn create(capacity: u64) -> (Arc, ChunkStream) { + create_with_config(capacity, StreamConfig::default()).await +} + +async fn create_with_config(capacity: u64, config: StreamConfig) -> (Arc, ChunkStream) { + let store = Arc::new(MemoryStreamStore::new(capacity)); + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + consumer_id: Some("ordered-record-test".into()), + ..StreamRecord::creating(StreamName::generate(), Some("test".into()), StreamPurpose::Stream) + }, + 1, + config, + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + (store, stream) +} + +#[tokio::test] +async fn lost_publication_response_is_reconciled_without_duplicate_records() { + let (store, stream) = create(1024).await; + stream + .append(&[Bytes::from_static(b"acknowledged")]) + .await + .unwrap(); + store.lose_publication_response_for_tests(); + assert_eq!(stream.checkpoint().await.unwrap(), 12); + let records = store.chunk_records_for_tests(stream.stream_name()).await.unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].logical_end, Some(12)); + assert_eq!(records[0].mappings.count(), 1); + assert_eq!(store.metadata_publish_count(), 2); + assert_eq!( + stream.read_at(0, 12).await.unwrap(), + Bytes::from_static(b"acknowledged") + ); +} + +#[tokio::test] +async fn checkpoint_cannot_rewrite_previously_published_append_boundaries() { + let (store, stream) = create(1024).await; + stream.append(&[Bytes::from_static(b"a")]).await.unwrap(); + stream.append(&[Bytes::from_static(b"bcd")]).await.unwrap(); + stream.checkpoint().await.unwrap(); + let metadata = store.record_store(1).unwrap(); + let previous = metadata.load_tail(stream.stream_name()).await.unwrap().unwrap(); + let mut changed = previous.record().clone(); + changed.generation += 1; + changed.mappings = StreamMappings::new(&[2, 2]).unwrap(); + assert!(matches!( + metadata.publish(Some(&previous), changed, None, None).await, + Err(StreamError::Corruption(_)) + )); + assert_eq!( + metadata + .load_tail(stream.stream_name()) + .await + .unwrap() + .unwrap() + .record(), + previous.record() + ); +} + +#[tokio::test] +async fn finite_reader_retains_trimmed_chunks_until_it_is_released() { + let (store, stream) = create(128).await; + let first = stream.append(&[Bytes::from(vec![1; 100])]).await.unwrap(); + stream.append(&[Bytes::from(vec![2; 100])]).await.unwrap(); + let mut reader = stream + .reader(0, crowdb_chunk_stream::ReadHint::Bytes(100)) + .unwrap(); + assert_eq!(stream.trim_prefix(100).await.unwrap(), 0); + assert!(!store.is_released(first.chunk_id.unwrap()).await); + assert_eq!(reader.next().await.unwrap().unwrap(), Bytes::from(vec![1; 100])); + drop(reader); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + match stream.trim_prefix(100).await { + Ok(_) | Err(StreamError::Backpressure) => {} + Err(error) => panic!("trim after reader release: {error}"), + } + if store.is_released(first.chunk_id.unwrap()).await { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); +} + +#[tokio::test] +async fn rollover_preserves_historical_records_and_static_stream_identity() { + let (store, stream) = create(128).await; + let identity = store.load(stream.stream_name()).await.unwrap().unwrap(); + stream.append(&[Bytes::from(vec![1; 100])]).await.unwrap(); + stream.append(&[Bytes::from(vec![2; 100])]).await.unwrap(); + let first = store + .chunk_records_for_tests(stream.stream_name()) + .await + .unwrap() + .remove(0); + let encoded = encode_stream_chunk_record(&first).unwrap(); + for _ in 0..254 { + stream.append(&[Bytes::from(vec![3; 100])]).await.unwrap(); + } + let records = store.chunk_records_for_tests(stream.stream_name()).await.unwrap(); + assert_eq!(records.len(), 256); + assert_eq!(encode_stream_chunk_record(&records[0]).unwrap(), encoded); + assert_eq!(store.load(stream.stream_name()).await.unwrap().unwrap(), identity); + assert_eq!(stream.read_at(100, 1).await.unwrap(), Bytes::from_static(&[2])); + assert_eq!(stream.metrics().record_cache_misses, 1); +} + +#[tokio::test] +async fn paused_publication_does_not_expose_uncommitted_checkpoint_bounds() { + let (store, stream) = create(1024).await; + stream + .append(&[Bytes::from_static(b"synchronized")]) + .await + .unwrap(); + store.pause_publication_for_tests(); + let writer = stream.clone(); + let checkpoint = tokio::spawn(async move { writer.checkpoint().await }); + store.wait_for_publication_for_tests().await; + let before = store + .load_tail_for_tests(stream.stream_name()) + .await + .unwrap() + .unwrap(); + assert_eq!(before.record().logical_end, Some(0)); + assert_eq!( + stream.read_at(0, 12).await.unwrap(), + Bytes::from_static(b"synchronized") + ); + store.resume_publication_for_tests(); + assert_eq!(checkpoint.await.unwrap().unwrap(), 12); + assert_eq!( + store + .load_tail_for_tests(stream.stream_name()) + .await + .unwrap() + .unwrap() + .record() + .logical_end, + Some(12) + ); +} + +#[tokio::test] +async fn takeover_after_seal_fences_a_paused_rollover_before_successor_data() { + let (store, stream) = create(128).await; + stream.append(&[Bytes::from(vec![1; 100])]).await.unwrap(); + store.pause_publication_for_tests(); + let old = stream.clone(); + let pending = tokio::spawn(async move { old.append(&[Bytes::from(vec![2; 100])]).await }); + store.wait_for_publication_for_tests().await; + let new = ChunkStream::open( + stream.stream_name(), + 2, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + assert_eq!(new.tail(), 100); + store.resume_publication_for_tests(); + assert!(matches!(pending.await.unwrap(), Err(StreamError::StaleWriter))); + assert_eq!(new.read_at(0, 100).await.unwrap(), Bytes::from(vec![1; 100])); + assert_eq!( + new.append(&[Bytes::from_static(b"new")]).await.unwrap().begin, + 100 + ); + let records = store.chunk_records_for_tests(stream.stream_name()).await.unwrap(); + assert_eq!(records.len(), 2); + assert_eq!(records[0].logical_end, records[1].logical_start); +} + +#[tokio::test] +async fn duplicate_identity_creation_cannot_overwrite_consumer_or_creation_time() { + let (store, stream) = create(1024).await; + let original = store.load(stream.stream_name()).await.unwrap().unwrap(); + let mut conflicting = original.clone(); + conflicting.consumer_id = Some("another-consumer".into()); + conflicting.created_at_ms += 1; + assert!(store.create(conflicting).await.is_err()); + assert_eq!(store.load(stream.stream_name()).await.unwrap().unwrap(), original); +} + +#[tokio::test(start_paused = true)] +async fn expired_read_lease_stops_pending_io_without_waiting_for_the_watchdog() { + let (store, stream) = create_with_config( + 1024, + StreamConfig { + watchdog_interval: Duration::from_secs(3600), + ..StreamConfig::default() + }, + ) + .await; + stream.append(&[Bytes::from_static(b"valid")]).await.unwrap(); + let captured = stream.capture_read_only().await.unwrap(); + store.pause_reads(); + let reader = captured.clone(); + let reading = tokio::spawn(async move { reader.read_at(0, 5).await }); + store.wait_for_concurrent_reads(1).await; + captured.expire_read_lease_for_tests(); + tokio::time::advance(Duration::from_secs(60)).await; + let result = tokio::time::timeout(Duration::from_millis(100), reading) + .await + .unwrap() + .unwrap(); + assert!(matches!(result, Err(StreamError::ReadUnavailable(_)))); + store.resume_reads(); +} + +#[tokio::test(start_paused = true)] +async fn a_pin_renewal_conflict_retries_before_the_next_regular_lease_interval() { + let (store, _stream) = create(1024).await; + tokio::task::yield_now().await; + store.fail_pin_renewals_for_tests(1); + tokio::time::advance(Duration::from_secs(15)).await; + tokio::task::yield_now().await; + assert_eq!(store.pin_renewal_count_for_tests(), 0); + tokio::time::advance(Duration::from_millis(100)).await; + tokio::task::yield_now().await; + assert_eq!(store.pin_renewal_count_for_tests(), 1); + assert_eq!(store.metadata_publish_count(), 1); +} + +#[tokio::test] +async fn interrupted_collection_keeps_a_retryable_record_across_owner_recovery() { + for fail_delete in [false, true] { + let (store, stream) = create(128).await; + let first = stream.append(&[Bytes::from(vec![1; 100])]).await.unwrap(); + stream.append(&[Bytes::from(vec![2; 100])]).await.unwrap(); + stream.append(&[Bytes::from(vec![3; 100])]).await.unwrap(); + if fail_delete { + store.fail_record_delete_for_tests(); + } else { + store.fail_release_for_tests(); + } + assert!(matches!( + stream.trim_prefix(100).await, + Err(StreamError::ReadUnavailable(_)) + )); + let records = store.chunk_records_for_tests(stream.stream_name()).await.unwrap(); + assert_eq!( + records[0].state, + crowdb_protocol::chunk_stream::StreamChunkState::Deleting + ); + assert_eq!(store.is_released(first.chunk_id.unwrap()).await, fail_delete); + let recovered = ChunkStream::open( + stream.stream_name(), + 2, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + recovered.trim_prefix(100).await.unwrap(); + assert!(store.is_released(first.chunk_id.unwrap()).await); + assert!(store + .chunk_records_for_tests(stream.stream_name()) + .await + .unwrap() + .iter() + .all(|record| record.logical_start != Some(0))); + assert_eq!( + recovered.read_at(100, 100).await.unwrap(), + Bytes::from(vec![2; 100]) + ); + } +} diff --git a/lib/crowdb-chunk-stream/tests/ordered_record_fault_test/initialization.rs b/lib/crowdb-chunk-stream/tests/ordered_record_fault_test/initialization.rs new file mode 100644 index 000000000..4a3fb546b --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/ordered_record_fault_test/initialization.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn excluded_initializer_cannot_recreate_a_collected_first_record() { + replace_paused_initializer(2).await; +} + +#[tokio::test] +async fn same_epoch_process_recovery_replaces_the_initial_reservation_without_reviving_the_old_writer() { + replace_paused_initializer(1).await; +} + +async fn replace_paused_initializer(writer_epoch: u64) { + let store = Arc::new(MemoryStreamStore::new(128)); + let identity = StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(StreamName::generate(), Some("test".into()), StreamPurpose::Stream) + }; + let name = identity.stream_name; + store.pause_publication_for_tests(); + let old_store = store.clone(); + let initializing = tokio::spawn(async move { + ChunkStream::create( + identity, + 1, + StreamConfig::default(), + old_store.clone(), + old_store.clone(), + old_store, + ) + .await + }); + store.wait_for_publication_for_tests().await; + let current = ChunkStream::create_registered( + name, + writer_epoch, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + current.append(&[Bytes::from(vec![1; 100])]).await.unwrap(); + current.append(&[Bytes::from(vec![2; 100])]).await.unwrap(); + current.trim_prefix(100).await.unwrap(); + store.resume_publication_for_tests(); + assert!(matches!( + initializing.await.unwrap(), + Err(StreamError::StaleWriter) + )); + assert!(store + .chunk_records_for_tests(name) + .await + .unwrap() + .iter() + .all(|record| record.logical_start != Some(0))); + assert_eq!(current.tail(), 200); + assert_eq!( + current.read_at(100, 100).await.unwrap(), + Bytes::from(vec![2; 100]) + ); +} diff --git a/lib/crowdb-chunk-stream/tests/ordered_record_test.rs b/lib/crowdb-chunk-stream/tests/ordered_record_test.rs new file mode 100644 index 000000000..3894172d3 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/ordered_record_test.rs @@ -0,0 +1,122 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamMetadataStore, StreamName, StreamPurpose, + StreamRecord, StreamRecordState, +}; +use crowdb_protocol::chunk_stream::StreamChunkState; +use std::sync::Arc; + +async fn create(capacity: u64) -> (Arc, ChunkStream) { + let store = Arc::new(MemoryStreamStore::new(capacity)); + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(StreamName::generate(), Some("test".into()), StreamPurpose::Stream) + }, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + (store, stream) +} + +#[tokio::test] +async fn ordered_records_cover_rollover_and_only_tail_recovery() { + let (store, stream) = create(128).await; + let first = stream.append(&[Bytes::from(vec![1; 100])]).await.unwrap(); + let second = stream.append(&[Bytes::from(vec![2; 100])]).await.unwrap(); + assert_ne!(first.chunk_id, second.chunk_id); + let metadata = store.record_store(1).unwrap(); + let old = metadata.locate(stream.stream_name(), 0).await.unwrap(); + assert_eq!(old.record().state, StreamChunkState::Sealed); + assert_eq!(old.record().logical_start, Some(0)); + assert_eq!(old.record().logical_end, Some(100)); + assert_eq!( + stream.read_at(0, 200).await.unwrap(), + Bytes::from([vec![1; 100], vec![2; 100]].concat()) + ); + let reopened = ChunkStream::open( + stream.stream_name(), + 2, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + assert_eq!(reopened.tail(), 200); + assert_eq!( + reopened.read_at(100, 100).await.unwrap(), + Bytes::from(vec![2; 100]) + ); +} + +#[tokio::test] +async fn checkpoint_capture_is_finite_and_close_survives_reopen() { + let (store, stream) = create(1024).await; + stream.append(&[Bytes::from_static(b"first")]).await.unwrap(); + let captured = stream.capture_read_only().await.unwrap(); + stream.append(&[Bytes::from_static(b"second")]).await.unwrap(); + assert_eq!(captured.tail(), 5); + assert_eq!( + captured.read_at(0, 5).await.unwrap(), + Bytes::from_static(b"first") + ); + assert!(captured.read_at(5, 1).await.is_err()); + stream.close().await.unwrap(); + let tail = store + .load_tail_for_tests(stream.stream_name()) + .await + .unwrap() + .unwrap(); + assert!(tail.record().closed); + assert_eq!(tail.record().logical_end, Some(11)); + let reopened = ChunkStream::open( + stream.stream_name(), + 2, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await + .unwrap(); + assert_eq!(reopened.tail(), 11); + assert!(reopened.append(&[Bytes::from_static(b"x")]).await.is_err()); +} + +#[tokio::test] +async fn registered_candidate_is_distinct_from_active_tail() { + let (store, stream) = create(1024).await; + let metadata = store.record_store(1).unwrap(); + let candidates = tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + let candidates = metadata.candidates(stream.stream_name()).await.unwrap(); + if !candidates.is_empty() { + break candidates; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let tail = metadata.load_tail(stream.stream_name()).await.unwrap().unwrap(); + assert_eq!(tail.record().state, StreamChunkState::Active); + assert_eq!(candidates[0].record().state, StreamChunkState::Candidate); + assert!(candidates[0].record().logical_start.is_none()); + assert_ne!(tail.record().chunk_id, candidates[0].record().chunk_id); + stream.close().await.unwrap(); + assert!(metadata + .candidates(stream.stream_name()) + .await + .unwrap() + .is_empty()); +} diff --git a/lib/crowdb-chunk-stream/tests/perf_metrics_test.rs b/lib/crowdb-chunk-stream/tests/perf_metrics_test.rs new file mode 100644 index 000000000..f63edb046 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/perf_metrics_test.rs @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, register_stream_perf_metrics, stream_perf_metrics, ChunkStream, StreamConfig, + StreamName, StreamPurpose, StreamRecord, StreamRecordState, +}; +use crowdb_common::metrics::{MetricPoint, MetricsRegistry}; + +#[tokio::test] +async fn stream_stages_appear_in_perf_output_and_survive_window_flushes() { + let mut registry = MetricsRegistry::new(); + register_stream_perf_metrics(&mut registry); + register_stream_perf_metrics(&mut registry); + let store = Arc::new(MemoryStreamStore::new(1024)); + let name = StreamName::generate(); + let stream = ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }, + 1, + StreamConfig::default(), + store.clone(), + store.clone(), + store, + ) + .await + .unwrap(); + assert_eq!(stream.metrics().metadata_publications, 1); + assert!(matches!( + registry.snapshot_named("stream.metadata.publish.lh", 1.0), + Some(MetricPoint::Histogram { count: 1, .. }) + )); + stream.append(&[Bytes::from_static(b"sample")]).await.unwrap(); + assert_eq!(stream.read_at(0, 6).await.unwrap(), Bytes::from_static(b"sample")); + assert!(matches!( + registry.snapshot_named("stream.checkpoint.cursor.lh", 1.0), + Some(MetricPoint::Histogram { count: 0, .. }) + )); + stream.checkpoint().await.unwrap(); + for name in [ + "stream.append.e2e.lh", + "stream.append.queue.lh", + "stream.append.encode.lh", + "stream.append.mirror.lh", + "stream.checkpoint.cursor.lh", + "stream.append.local_view.lh", + "stream.read.e2e.lh", + "stream.read.plan.lh", + "stream.read.io.lh", + ] { + assert!( + matches!( + registry.snapshot_named(name, 1.0), + Some(MetricPoint::Histogram { count: 1, .. }) + ), + "missing phase: {name}" + ); + } + assert!( + matches!( + registry.snapshot_named("stream.capacity.wait.lh", 1.0), + Some(MetricPoint::Histogram { count: 0, .. }) + ), + "prepared initial capacity needs no foreground wait" + ); + stream.close().await.unwrap(); + assert!(matches!( + registry.snapshot_named("stream.successor.prepare.lh", 1.0), + Some(MetricPoint::Histogram { count: 1.., .. }) + )); + assert!(matches!( + registry.snapshot_named("stream.successor.wait.lh", 1.0), + Some(MetricPoint::Histogram { .. }) + )); + let cumulative = stream_perf_metrics(); + let mut output = Vec::new(); + registry.flush(&mut output, 1.0, "test"); + let text = String::from_utf8(output).unwrap(); + assert_eq!(text.matches("stream.checkpoint.cursor.lh").count(), 1); + assert!(text.contains("stream.metadata.publish.lh")); + assert!(matches!( + registry.snapshot_named("stream.append.e2e.lh", 1.0), + Some(MetricPoint::Histogram { + count: 0, + total: 1, + .. + }) + )); + for ((name, before), (after_name, after)) in cumulative.into_iter().zip(stream_perf_metrics()) { + assert_eq!(name, after_name); + assert_eq!(before.count, after.count); + assert_eq!(before.p99, after.p99); + } +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs index 0cc4552c7..2b290fc45 100644 --- a/lib/crowdb-chunk-stream/tests/production_chunk_test.rs +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test.rs @@ -10,8 +10,8 @@ use bytes::Bytes; use crowdb_chunk_client::{ChunkAllocator, ChunkIoClient, ChunkReadPolicy, DiskWriter, IoError}; use crowdb_chunk_stream::{ memory::MemoryStreamStore, ChunkStream, CursorAdvance, MirrorStripImage, ProductionStreamChunkStore, - ProductionStreamRuntime, StreamBinding, StreamBindingState, StreamChunkStore, StreamConfig, - StreamMetadataStore, StreamName, StreamRegistry, + ProductionStreamRuntime, StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, StreamRecord, + StreamRecordState, StreamRegistry, }; use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; use crowdb_protocol::chunkdb::rpc::{ @@ -20,29 +20,92 @@ use crowdb_protocol::chunkdb::rpc::{ AppendChunkResponse, Chunk, ChunkState, ChunkStrip, DeleteChunkRequest, DeleteChunkResponse, DiscardReplacementSegmentRequest, DiscardReplacementSegmentResponse, MirrorStrip, QueryChunkRequest, QueryChunkResponse, ReplaceChunkStripRangeRequest, ReplaceChunkStripRangeResponse, SealChunkRequest, - SealChunkResponse, Strip, StripType, UpdateChunkStripRequest, UpdateChunkStripResponse, + SealChunkResponse, Strip, StripType, }; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::diskdb::rpc::Segment; +#[path = "production_chunk_test/recovery.rs"] +mod recovery; + +#[path = "production_chunk_test/local_progress.rs"] +mod local_progress; + +#[path = "production_chunk_test/growth.rs"] +mod growth; + +#[path = "production_chunk_test/preparation.rs"] +mod preparation; + +#[path = "production_chunk_test/scope.rs"] +mod scope; + +#[path = "production_chunk_test/unwritten.rs"] +mod unwritten; + +#[path = "production_chunk_test/duplicate.rs"] +mod duplicate; + +#[path = "production_chunk_test/owner_capture.rs"] +mod owner_capture; + +#[path = "common/grouped_write.rs"] +mod grouped_write; +use grouped_write::TestGroupedStrip; + +#[path = "production_chunk_test/allocated.rs"] +mod allocated; + +#[path = "production_chunk_test/sealing.rs"] +mod sealing; + +#[path = "production_chunk_test/colocated.rs"] +mod colocated; + struct Allocator { chunk: Mutex>, + successors: Mutex>, + next_chunk: AtomicU64, + stale_query_once: Mutex>, fail_advance_after_commit: AtomicBool, fail_replacement_response_once: AtomicBool, fail_query_once: AtomicBool, replacement_calls: AtomicUsize, replacements: AtomicUsize, + allocation_copy_limit: AtomicUsize, + colocated_offset: AtomicU64, + pause_growth: AtomicBool, + growth_started: tokio::sync::Notify, + growth_resume: tokio::sync::Notify, + conflict_advance_once: AtomicBool, + advance_calls: AtomicUsize, + fail_growth_once: AtomicBool, + advance_during_growth: AtomicU64, + conflict_replacement_once: AtomicBool, } impl Allocator { fn new() -> Self { Self { chunk: Mutex::new(None), + successors: Mutex::new(HashMap::new()), + next_chunk: AtomicU64::new(8), + stale_query_once: Mutex::new(None), fail_advance_after_commit: AtomicBool::new(false), fail_replacement_response_once: AtomicBool::new(false), fail_query_once: AtomicBool::new(false), replacement_calls: AtomicUsize::new(0), replacements: AtomicUsize::new(0), + allocation_copy_limit: AtomicUsize::new(0), + colocated_offset: AtomicU64::new(u64::MAX), + pause_growth: AtomicBool::new(false), + growth_started: tokio::sync::Notify::new(), + growth_resume: tokio::sync::Notify::new(), + conflict_advance_once: AtomicBool::new(false), + advance_calls: AtomicUsize::new(0), + fail_growth_once: AtomicBool::new(false), + advance_during_growth: AtomicU64::new(0), + conflict_replacement_once: AtomicBool::new(false), } } } @@ -90,16 +153,32 @@ impl ChunkAllocator for Allocator { ) -> crowdb_chunk_client::Result { let chunk_id = ChunkId { high: u64::try_from(request.chunk_type).unwrap() << 56, - low: 8, + low: self.next_chunk.fetch_add(1, Ordering::AcqRel), + }; + let limit = self.allocation_copy_limit.load(Ordering::Acquire); + let copies = if limit == 0 { + request.copy_count + } else { + request.copy_count.min(u32::try_from(limit).unwrap()) }; - let segments = (1..=request.copy_count) + let colocated_offset = self.colocated_offset.load(Ordering::Acquire); + let segments = (1..=copies) .map(|disk| Segment { disk_id: Some(DiskId { - high: u64::from(disk), + high: (chunk_id.low - 8) * 1000 + + if colocated_offset == u64::MAX { + u64::from(disk) + } else { + 1 + }, low: 0, }), owner_chunk: Some(chunk_id), - unit_offset: 0, + unit_offset: if colocated_offset == u64::MAX { + 0 + } else { + u64::from(disk - 1) * colocated_offset + }, zone_index: 0, unit_count: request.write_granularity / 4, allocation_ts: 1, @@ -126,7 +205,12 @@ impl ChunkAllocator for Allocator { owner_key: request.owner_key, ..Chunk::default() }; - *self.chunk.lock().unwrap() = Some(chunk.clone()); + let mut primary = self.chunk.lock().unwrap(); + if primary.is_none() { + *primary = Some(chunk.clone()); + } else { + self.successors.lock().unwrap().insert(chunk_id, chunk.clone()); + } Ok(AllocateChunkResponse { chunk: Some(chunk) }) } @@ -134,8 +218,19 @@ impl ChunkAllocator for Allocator { &self, request: AppendChunkRequest, ) -> crowdb_chunk_client::Result { - let mut guard = self.chunk.lock().unwrap(); - let chunk = guard.as_mut().unwrap(); + if request.chunk_id.is_some_and(|id| id.low == 8) && self.pause_growth.swap(false, Ordering::AcqRel) { + self.growth_started.notify_one(); + self.growth_resume.notified().await; + } + if request.chunk_id.is_some_and(|id| id.low == 8) + && self.fail_growth_once.swap(false, Ordering::AcqRel) + { + return Err(IoError::AllocationFailed( + "injected allocation unavailability".into(), + )); + } + let mut guard = self.chunk_for(request.chunk_id); + let chunk = &mut *guard; if chunk.id != request.chunk_id || chunk.modify_ts != request.modify_ts { return Ok(AppendChunkResponse { modify_ts: chunk.modify_ts, @@ -143,40 +238,53 @@ impl ChunkAllocator for Allocator { chunk: Some(chunk.clone()), }); } - let first = chunk.strips.first().unwrap(); - let capacity = request.strip_size * first.unit_kb; - let offset = chunk.capacity; - let sequence = chunk.next_strip_sequence; - let chunk_id = chunk.id.unwrap(); - let segments = (1..=request.copy_count) - .map(|disk| Segment { - disk_id: Some(DiskId { - high: u64::from(sequence) * 10 + u64::from(disk), - low: 0, - }), - owner_chunk: Some(chunk_id), - unit_offset: 0, - zone_index: 0, - unit_count: request.strip_size, - allocation_ts: 1, - }) - .collect(); - let strip = ChunkStrip { - chunk_offset: offset, - strip_sequence: sequence, - unit_kb: first.unit_kb, - capacity, - strip_type: StripType::Mirror as i32, - strip: Some(Strip::MirrorStrip(MirrorStrip { segments })), - ..ChunkStrip::default() + let unit_kb = chunk.strips.first().unwrap().unit_kb; + let concurrent_cursor = if chunk.id.is_some_and(|id| id.low == 8) { + self.advance_during_growth.swap(0, Ordering::AcqRel) + } else { + 0 }; - chunk.strips.push(strip.clone()); - chunk.capacity += capacity; - chunk.next_strip_sequence += 1; + if concurrent_cursor != 0 { + chunk.acknowledged_cursor = concurrent_cursor; + chunk.modify_ts += 1; + } + let capacity = request.strip_size * unit_kb; + let chunk_id = chunk.id.unwrap(); + let mut added = Vec::new(); + for _ in 0..request.strip_count { + let offset = chunk.capacity; + let sequence = chunk.next_strip_sequence; + let segments = (1..=request.copy_count) + .map(|disk| Segment { + disk_id: Some(DiskId { + high: (chunk_id.low - 8) * 1000 + u64::from(sequence) * 10 + u64::from(disk), + low: 0, + }), + owner_chunk: Some(chunk_id), + unit_offset: 0, + zone_index: 0, + unit_count: request.strip_size, + allocation_ts: 1, + }) + .collect(); + let strip = ChunkStrip { + chunk_offset: offset, + strip_sequence: sequence, + unit_kb, + capacity, + strip_type: StripType::Mirror as i32, + strip: Some(Strip::MirrorStrip(MirrorStrip { segments })), + ..ChunkStrip::default() + }; + chunk.strips.push(strip.clone()); + added.push(strip); + chunk.capacity += capacity; + chunk.next_strip_sequence += 1; + } chunk.modify_ts += 1; Ok(AppendChunkResponse { modify_ts: chunk.modify_ts, - strips: vec![strip], + strips: added, chunk: None, }) } @@ -185,8 +293,12 @@ impl ChunkAllocator for Allocator { &self, request: AdvanceChunkWriteRequest, ) -> crowdb_chunk_client::Result { - let mut guard = self.chunk.lock().unwrap(); - let chunk = guard.as_mut().unwrap(); + self.advance_calls.fetch_add(1, Ordering::AcqRel); + let mut guard = self.chunk_for(request.chunk_id); + let chunk = &mut *guard; + if self.conflict_advance_once.swap(false, Ordering::AcqRel) { + chunk.modify_ts += 1; + } if chunk.id != request.chunk_id || chunk.writer_epoch != request.writer_epoch || chunk.modify_ts != request.expected_modify_ts @@ -204,11 +316,13 @@ impl ChunkAllocator for Allocator { } async fn seal_chunk(&self, request: SealChunkRequest) -> crowdb_chunk_client::Result { - let mut guard = self.chunk.lock().unwrap(); - let chunk = guard.as_mut().unwrap(); + let mut guard = self.chunk_for(request.chunk_id); + let chunk = &mut *guard; assert_eq!(chunk.id, request.chunk_id); chunk.state = ChunkState::Sealed as i32; - chunk.sealed_length = request.seal_length; + chunk.sealed_length = u32::try_from(request.seal_bytes.div_ceil(1024)).unwrap(); + chunk.acknowledged_cursor = request.seal_bytes; + chunk.modify_ts += 1; Ok(SealChunkResponse { chunk: Some(chunk.clone()), }) @@ -218,8 +332,8 @@ impl ChunkAllocator for Allocator { &self, request: DeleteChunkRequest, ) -> crowdb_chunk_client::Result { - let mut guard = self.chunk.lock().unwrap(); - let chunk = guard.as_mut().unwrap(); + let mut guard = self.chunk_for(request.chunk_id); + let chunk = &mut *guard; assert_eq!(chunk.id, request.chunk_id); chunk.state = ChunkState::Deleted as i32; Ok(DeleteChunkResponse { @@ -227,13 +341,6 @@ impl ChunkAllocator for Allocator { }) } - async fn update_chunk_strip( - &self, - _request: UpdateChunkStripRequest, - ) -> crowdb_chunk_client::Result { - unreachable!() - } - async fn query_chunk( &self, request: QueryChunkRequest, @@ -241,7 +348,18 @@ impl ChunkAllocator for Allocator { if self.fail_query_once.swap(false, Ordering::AcqRel) { return Err(IoError::AllocationFailed("injected query outage".into())); } - let chunk = self.chunk.lock().unwrap().clone(); + let chunk = self.stale_query_once.lock().unwrap().take().or_else(|| { + let primary = self.chunk.lock().unwrap(); + if primary.as_ref().and_then(|chunk| chunk.id) == request.chunk_id { + primary.clone() + } else { + self.successors + .lock() + .unwrap() + .get(&request.chunk_id.unwrap()) + .cloned() + } + }); if chunk.as_ref().and_then(|chunk| chunk.id) != request.chunk_id { return Err(IoError::ChunkNotFound("missing test chunk".into())); } @@ -274,9 +392,12 @@ impl ChunkAllocator for Allocator { request: ReplaceChunkStripRangeRequest, ) -> crowdb_chunk_client::Result { self.replacement_calls.fetch_add(1, Ordering::AcqRel); - let mut guard = self.chunk.lock().unwrap(); - let chunk = guard.as_mut().unwrap(); + let mut guard = self.chunk_for(request.chunk_id); + let chunk = &mut *guard; let index = request.start_index as usize; + if self.conflict_replacement_once.swap(false, Ordering::AcqRel) { + chunk.modify_ts += 1; + } if chunk.modify_ts != request.expected_modify_ts || chunk.strips.get(index) != request.old_strips.first() { @@ -306,9 +427,17 @@ impl ChunkAllocator for Allocator { #[derive(Default)] struct Disks { bytes: Mutex>>, + writes: Mutex>, fsyncs: AtomicUsize, fail_disk_once: AtomicU64, fail_fsync_once: AtomicU64, + pause_initialization: AtomicBool, + initialization_started: tokio::sync::Notify, + initialization_resume: tokio::sync::Notify, + drain_delay_ms: AtomicU64, + active_drains: AtomicUsize, + max_active_drains: AtomicUsize, + completed_drains: AtomicUsize, } #[test] @@ -335,6 +464,43 @@ fn production_runtime_shares_connected_chunk_io_parts() { #[async_trait] impl DiskWriter for Disks { + async fn drain_segment( + &self, + segment: &Segment, + _unit_bytes: u64, + _deadline: std::time::Instant, + ) -> crowdb_chunk_client::Result<()> { + let active = self.active_drains.fetch_add(1, Ordering::AcqRel) + 1; + self.max_active_drains.fetch_max(active, Ordering::AcqRel); + let delay = self.drain_delay_ms.load(Ordering::Acquire); + if delay != 0 { + tokio::time::sleep(std::time::Duration::from_millis(delay)).await; + } + let result = self.fsync(segment).await; + self.active_drains.fetch_sub(1, Ordering::AcqRel); + self.completed_drains.fetch_add(1, Ordering::AcqRel); + result + } + async fn initialize_synchronized_at_byte_offset( + &self, + segment: &Segment, + unit_bytes: u64, + offset: u64, + data: Bytes, + ) -> crowdb_chunk_client::Result<()> { + if segment + .disk_id + .is_some_and(|disk| disk.high > 2 && disk.high < 1000) + && self.pause_initialization.swap(false, Ordering::AcqRel) + { + self.initialization_started.notify_one(); + self.initialization_resume.notified().await; + } + self.write_at_byte_offset(segment, unit_bytes, offset, data) + .await?; + self.fsync(segment).await + } + async fn write( &self, _segment: &Segment, @@ -347,7 +513,7 @@ impl DiskWriter for Disks { async fn write_at_byte_offset( &self, segment: &Segment, - _unit_bytes: u64, + unit_bytes: u64, offset: u64, data: Bytes, ) -> crowdb_chunk_client::Result<()> { @@ -359,6 +525,10 @@ impl DiskWriter for Disks { { return Err(IoError::WriteFailed("injected mirror write failure".into())); } + self.writes + .lock() + .unwrap() + .push((*segment, unit_bytes, offset, data.clone())); let mut disks = self.bytes.lock().unwrap(); let bytes = disks.entry(disk).or_default(); let start = usize::try_from(offset).unwrap(); @@ -412,42 +582,45 @@ async fn production_store_writes_reads_advances_and_releases_one_mirror_chunk() .allocate_mirrored(name, 9, crowdb_protocol::chunk_stream::StreamPurpose::Stream) .await .unwrap(); - let data = Bytes::from_static(b"stream"); - store - .write_mirrors_with_images( - name, - 9, - active.chunk_id, - 0, - data.clone(), - &[MirrorStripImage { - block_offset: 0, - data: data.clone(), - full_image: data, - }], - ) - .await - .unwrap(); - assert_eq!(disks.fsyncs.load(Ordering::Relaxed), 2); + let mut strip = TestGroupedStrip::default(); + let end = strip.append(&store, name, 9, active.chunk_id, b"stream").await; + assert_eq!(disks.fsyncs.load(Ordering::Relaxed), 4); assert_eq!( store - .advance_cursor(name, 9, active.chunk_id, 0, 6, 17) + .advance_cursor(name, 9, active.chunk_id, 0, end, 17) .await .unwrap(), CursorAdvance::Committed ); assert_eq!( - store.read(active.chunk_id, 0, 6).await.unwrap(), + store + .read( + active.chunk_id, + crowdb_protocol::frame::FRAME_GROUP_HEADER_BYTES as u64, + 6 + ) + .await + .unwrap(), Bytes::from_static(b"stream") ); - store.seal(active.chunk_id, 9, 6).await.unwrap(); + assert!(store.release_trimmed(active.chunk_id, 6, end).await.is_err()); + store.seal(active.chunk_id, 9, end).await.unwrap(); + assert!(store.release_trimmed(active.chunk_id, 6, end - 1).await.is_err()); assert_eq!( store - .release_trimmed(active.chunk_id, 6) + .release_trimmed(active.chunk_id, 6, end) .await .unwrap() .reclaimed_bytes, - 6 + end + ); + assert_eq!( + store + .release_trimmed(active.chunk_id, 6, end) + .await + .unwrap() + .reclaimed_bytes, + 0 ); } @@ -468,52 +641,51 @@ async fn production_store_grows_and_writes_across_mirror_strips() { .await .unwrap() .unwrap(); - assert_eq!(grown.capacity, 2 * 1024 * 1024); - let offset = active.capacity - 2; - assert_eq!( - store - .advance_cursor(name, 9, active.chunk_id, 0, offset, 0) - .await - .unwrap(), - CursorAdvance::Committed + assert_eq!(grown.capacity, 8 * 1024 * 1024); + let payload = Bytes::from(vec![0x5a; 1024 * 1024 + 5]); + let data = Bytes::from( + crowdb_protocol::frame::encode_append_group_views(active.chunk_id, 0, &payload, 1) + .unwrap() + .concat(), ); - let data = Bytes::from_static(b"split"); - let first = data.slice(..2); - let second = data.slice(2..); - let mut prefix = vec![0; usize::try_from(offset).unwrap()]; - prefix.extend_from_slice(&first); + let complete_end = data.len() as u64; + let boundary = usize::try_from(active.capacity).unwrap(); + let images: Vec<_> = (0..data.len()) + .step_by(boundary) + .map(|start| { + let part = data.slice(start..(start + boundary).min(data.len())); + MirrorStripImage { + block_offset: 0, + data: part.clone(), + full_image: part, + } + }) + .collect(); store - .write_mirrors_with_images( + .write_mirrors( name, 9, active.chunk_id, - offset, - data, - &[ - MirrorStripImage { - block_offset: offset, - data: first, - full_image: Bytes::from(prefix), - }, - MirrorStripImage { - block_offset: 0, - data: second.clone(), - full_image: second, - }, - ], + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: 0, + complete_end, + images: &images, + }, ) .await .unwrap(); + let end = complete_end; assert_eq!( store - .advance_cursor(name, 9, active.chunk_id, offset, offset + 5, 0) + .advance_cursor(name, 9, active.chunk_id, 0, end, 0) .await .unwrap(), CursorAdvance::Committed ); assert_eq!( - store.read(active.chunk_id, offset, 5).await.unwrap(), - Bytes::from_static(b"split") + store.read(active.chunk_id, active.capacity - 2, 5).await.unwrap(), + data.slice(boundary - 2..boundary + 3) ); } @@ -555,22 +727,25 @@ async fn configured_stream_chunk_capacity_limits_growth_without_changing_strip_s #[tokio::test] async fn chunk_stream_runs_end_to_end_over_the_production_chunk_adapter() { let allocator: Arc = Arc::new(Allocator::new()); - let disks: Arc = Arc::new(Disks::default()); + let disks = Arc::new(Disks::default()); let chunks: Arc = Arc::new( - ProductionStreamChunkStore::new(allocator, disks, 30_000, ChunkReadPolicy::default()).unwrap(), + ProductionStreamChunkStore::new(allocator, disks.clone(), 30_000, ChunkReadPolicy::default()) + .unwrap(), ); let metadata = Arc::new(MemoryStreamStore::new(1)); let registry: Arc = metadata.clone(); let metadata_store: Arc = metadata; let stream_name = StreamName { high: 40, low: 41 }; let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, stream_name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("test".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 9, StreamConfig::default(), @@ -587,6 +762,12 @@ async fn chunk_stream_runs_end_to_end_over_the_production_chunk_adapter() { assert_eq!((range.begin, range.end), (0, 5)); assert_eq!(stream.read_at(0, 5).await.unwrap(), Bytes::from_static(b"frame")); assert_eq!(range.chunk_id.unwrap().low, 8); + assert_eq!( + stream.metrics().physical_append_bytes, + 2 * (5 + + (crowdb_protocol::frame::FRAME_GROUP_HEADER_BYTES + crowdb_protocol::frame::FRAME_FOOTER_BYTES) + as u64) + ); } #[tokio::test] @@ -604,13 +785,15 @@ async fn stream_replaces_failed_mirrors_from_retained_strip_image() { ); let metadata = Arc::new(MemoryStreamStore::new(1)); let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, stream_name: StreamName { high: 50, low: 51 }, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("test".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 9, StreamConfig::default(), @@ -667,13 +850,15 @@ async fn stream_repair_preserves_a_batch_crossing_mirror_strips() { ); let metadata = Arc::new(MemoryStreamStore::new(1)); let stream = ChunkStream::create( - StreamBinding { + StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, stream_name: StreamName { high: 52, low: 53 }, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("test".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), }, 9, StreamConfig::default(), @@ -683,7 +868,10 @@ async fn stream_repair_preserves_a_batch_crossing_mirror_strips() { ) .await .unwrap(); - let prefix = Bytes::from(vec![b'a'; 1024 * 1024 - 60]); + let prefix = Bytes::from(vec![ + b'a'; + 16 * crowdb_protocol::frame::MAX_FRAME_GROUP_PAYLOAD_BYTES - 7 + ]); stream.append(std::slice::from_ref(&prefix)).await.unwrap(); disks.fail_disk_once.store(11, Ordering::Release); let suffix = Bytes::from_static(b"crossing-the-boundary"); @@ -694,7 +882,7 @@ async fn stream_repair_preserves_a_batch_crossing_mirror_strips() { assert_eq!(&read[prefix.len()..], suffix.as_ref()); assert_eq!(stream.metrics().rollovers, 0); let chunk = allocator.chunk.lock().unwrap().clone().unwrap(); - assert_eq!(chunk.strips.len(), 2); + assert_eq!(chunk.strips.len(), 8); let Some(Strip::MirrorStrip(mirror)) = &chunk.strips[1].strip else { panic!("expected mirror strip"); }; @@ -718,29 +906,18 @@ async fn production_store_reconciles_a_post_commit_cursor_timeout() { .allocate_mirrored(name, 9, crowdb_protocol::chunk_stream::StreamPurpose::Stream) .await .unwrap(); - let data = Bytes::from_static(b"once"); - store - .write_mirrors_with_images( - name, - 9, - active.chunk_id, - 0, - data.clone(), - &[MirrorStripImage { - block_offset: 0, - data: data.clone(), - full_image: data, - }], - ) - .await - .unwrap(); + let mut strip = TestGroupedStrip::default(); + let end = strip.append(&store, name, 9, active.chunk_id, b"once").await; allocator.fail_advance_after_commit.store(true, Ordering::Release); assert_eq!( store - .advance_cursor(name, 9, active.chunk_id, 0, 4, 19) + .advance_cursor(name, 9, active.chunk_id, 0, end, 19) .await .unwrap(), CursorAdvance::Committed ); - assert_eq!(store.durable_cursor(active.chunk_id, 9).await.unwrap().offset, 4); + assert_eq!( + store.durable_cursor(active.chunk_id, 9).await.unwrap().offset, + end + ); } diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/allocated.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/allocated.rs new file mode 100644 index 000000000..882914737 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/allocated.rs @@ -0,0 +1,48 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Preserve the primary fixture chunk while permitting independent successors. + +use std::{ + ops::{Deref, DerefMut}, + sync::MutexGuard, +}; + +use super::{Allocator, Chunk, ChunkId, HashMap}; + +pub(super) struct TestChunkGuard<'a> { + primary: MutexGuard<'a, Option>, + others: MutexGuard<'a, HashMap>, + id: ChunkId, +} + +impl Allocator { + pub(super) fn chunk_for(&self, id: Option) -> TestChunkGuard<'_> { + TestChunkGuard { + primary: self.chunk.lock().unwrap(), + others: self.successors.lock().unwrap(), + id: id.unwrap(), + } + } +} + +impl Deref for TestChunkGuard<'_> { + type Target = Chunk; + fn deref(&self) -> &Self::Target { + if self.primary.as_ref().and_then(|chunk| chunk.id) == Some(self.id) { + self.primary.as_ref().unwrap() + } else { + self.others.get(&self.id).unwrap() + } + } +} + +impl DerefMut for TestChunkGuard<'_> { + fn deref_mut(&mut self) -> &mut Self::Target { + if self.primary.as_ref().and_then(|chunk| chunk.id) == Some(self.id) { + self.primary.as_mut().unwrap() + } else { + self.others.get_mut(&self.id).unwrap() + } + } +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/colocated.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/colocated.rs new file mode 100644 index 000000000..15a32faa7 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/colocated.rs @@ -0,0 +1,32 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn fresh_colocated_mirrors_require_disjoint_allocations_before_initialization() { + for offset in [0, 255, 256] { + let allocator = Arc::new(Allocator::new()); + allocator.colocated_offset.store(offset, Ordering::Release); + let disks = Arc::new(Disks::default()); + let store = + ProductionStreamChunkStore::new(allocator, disks.clone(), 30_000, ChunkReadPolicy::default()) + .unwrap(); + let result = store + .allocate_mirrored( + StreamName { high: 191, low: 193 }, + 9, + crowdb_chunk_stream::StreamPurpose::Stream, + ) + .await; + assert_eq!( + result.is_ok(), + offset == 256, + "replica unit offset {offset}: {result:?}" + ); + assert_eq!( + disks.fsyncs.load(Ordering::Relaxed), + if offset == 256 { 2 } else { 0 } + ); + } +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/duplicate.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/duplicate.rs new file mode 100644 index 000000000..c69d2901e --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/duplicate.rs @@ -0,0 +1,56 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_protocol::frame::{parse_header, FrameMagic}; + +#[tokio::test] +async fn delayed_duplicate_append_preserves_later_acknowledged_groups() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let chunks = Arc::new( + ProductionStreamChunkStore::new(allocator, disks.clone(), 30_000, ChunkReadPolicy::default()) + .unwrap(), + ); + let metadata = Arc::new(MemoryStreamStore::new(1)); + let stream = ChunkStream::create( + StreamRecord { + purpose: crowdb_chunk_stream::StreamPurpose::Stream, + stream_name: StreamName { high: 151, low: 157 }, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 9, + StreamConfig::default(), + metadata.clone(), + metadata, + chunks, + ) + .await + .unwrap(); + stream.append(&[Bytes::from_static(b"first")]).await.unwrap(); + let delayed: Vec<_> = disks + .writes + .lock() + .unwrap() + .iter() + .filter(|(_, _, _, data)| parse_header(data).is_ok_and(|header| header.magic == FrameMagic::StreamV2)) + .cloned() + .collect(); + assert_eq!(delayed.len(), 2); + stream.append(&[Bytes::from_static(b"second")]).await.unwrap(); + for (segment, unit_bytes, offset, data) in delayed { + disks + .write_at_byte_offset(&segment, unit_bytes, offset, data) + .await + .unwrap(); + } + assert_eq!( + stream.read_at(0, 11).await.unwrap(), + Bytes::from_static(b"firstsecond") + ); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs new file mode 100644 index 000000000..a5d5c683d --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/growth.rs @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn a_small_waiting_append_uses_a_smaller_growth_window() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new(allocator, disks.clone(), 30_000, ChunkReadPolicy::default()) + .unwrap(); + let name = StreamName { high: 181, low: 191 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let payload = Bytes::from(vec![ + 0x71; + 15 * crowdb_protocol::frame::MAX_FRAME_GROUP_PAYLOAD_BYTES + ]); + let data = Bytes::from( + crowdb_protocol::frame::encode_append_group_views(active.chunk_id, 0, &payload, 1) + .unwrap() + .concat(), + ); + let end = data.len() as u64; + store + .write_mirrors( + name, + 9, + active.chunk_id, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: 0, + complete_end: end, + images: &[MirrorStripImage { + block_offset: 0, + data: data.clone(), + full_image: data, + }], + }, + ) + .await + .unwrap(); + assert_eq!(end, 15 * 65536); + let grown = store + .grow_mirrored(name, 9, active.chunk_id, end + 128 * 1024) + .await + .unwrap() + .unwrap(); + assert_eq!(grown.capacity, 2 * 1024 * 1024); + assert_eq!(grown.acknowledged_cursor, end); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 6); + let grown = store + .grow_mirrored(name, 9, active.chunk_id, end + 3 * 1024 * 1024) + .await + .unwrap() + .unwrap(); + assert_eq!(grown.capacity, 8 * 1024 * 1024); + assert_eq!(grown.acknowledged_cursor, end); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 18); +} + +#[tokio::test] +async fn growth_amortizes_allocation_and_initializes_every_replica_within_the_hard_capacity() { + const MIB: u64 = 1024 * 1024; + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new_with_mirror_copies_and_capacity( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + 2, + 10 * MIB, + ) + .unwrap(); + let name = StreamName { high: 173, low: 179 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + assert_eq!(active.capacity, MIB); + let grown = store + .grow_mirrored(name, 9, active.chunk_id, MIB + 1) + .await + .unwrap() + .unwrap(); + assert_eq!(grown.capacity, 8 * MIB); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 16); + let revision = allocator.chunk.lock().unwrap().as_ref().unwrap().modify_ts; + for required in [2 * MIB, 4 * MIB, 7 * MIB, 8 * MIB] { + assert_eq!( + store + .grow_mirrored(name, 9, active.chunk_id, required) + .await + .unwrap() + .unwrap() + .capacity, + 8 * MIB + ); + } + assert_eq!( + allocator.chunk.lock().unwrap().as_ref().unwrap().modify_ts, + revision + ); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 16); + assert_eq!( + store + .grow_mirrored(name, 9, active.chunk_id, 9 * MIB) + .await + .unwrap() + .unwrap() + .capacity, + 10 * MIB + ); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 20); + assert!(store + .grow_mirrored(name, 9, active.chunk_id, 10 * MIB + 1) + .await + .unwrap() + .is_none()); + let media = disks.bytes.lock().unwrap(); + assert_eq!(media.len(), 20); + assert!(media + .values() + .all(|bytes| bytes.len() == usize::try_from(MIB).unwrap() && bytes.iter().all(|byte| *byte == 0))); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs new file mode 100644 index 000000000..2205e14f1 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/local_progress.rs @@ -0,0 +1,250 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn configured_replica_count_cannot_silently_accept_a_two_copy_layout() { + for copies in [3, 5] { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new_with_mirror_copies( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + copies, + ) + .unwrap(); + let name = StreamName { high: 41, low: 43 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let original = allocator.chunk.lock().unwrap().clone().unwrap(); + { + let mut guard = allocator.chunk.lock().unwrap(); + let chunk = guard.as_mut().unwrap(); + let Some(Strip::MirrorStrip(mirror)) = &mut chunk.strips[0].strip else { + panic!("mirror missing") + }; + mirror.segments.truncate(2); + chunk.modify_ts += 1; + } + store.durable_cursor(active.chunk_id, 9).await.unwrap(); + let data = Bytes::from( + crowdb_protocol::frame::encode_append_group_views( + active.chunk_id, + 0, + &Bytes::from_static(b"record"), + 1, + ) + .unwrap() + .concat(), + ); + let complete_end = data.len() as u64; + let result = store + .write_mirrors( + name, + 9, + active.chunk_id, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: 0, + complete_end, + images: &[MirrorStripImage { + block_offset: 0, + data: data.clone(), + full_image: data, + }], + }, + ) + .await; + assert!( + matches!(&result, Err(crowdb_chunk_stream::StreamError::ReadUnavailable(_))), + "configured {copies} copies: result={result:?}, synchronized copies={}", + disks.fsyncs.load(Ordering::Relaxed) + ); + assert_eq!(disks.fsyncs.load(Ordering::Relaxed), copies as usize); + assert_eq!(store.durable_cursor(active.chunk_id, 9).await.unwrap().offset, 0); + { + let mut guard = allocator.chunk.lock().unwrap(); + let current = guard.as_mut().unwrap(); + current.strips = original.strips; + current.modify_ts += 1; + } + store.durable_cursor(active.chunk_id, 9).await.unwrap(); + let mut retry = TestGroupedStrip::default(); + retry.append(&store, name, 9, active.chunk_id, b"record").await; + assert_eq!(disks.fsyncs.load(Ordering::Relaxed), 2 * copies as usize); + } +} + +#[tokio::test] +async fn grouped_progress_survives_checkpoint_query_renew_growth_and_mirror_repair() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let name = StreamName { high: 7, low: 9 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let mut strip = TestGroupedStrip::default(); + let first = strip.append(&store, name, 9, active.chunk_id, b"first").await; + assert_eq!(store.durable_cursor(active.chunk_id, 9).await.unwrap().offset, 0); + store.renew_liveness(active.chunk_id, 9).await.unwrap(); + let old_layout = allocator.chunk.lock().unwrap().clone(); + let grown = store + .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .await + .unwrap() + .unwrap(); + assert_eq!(grown.acknowledged_cursor, first); + assert_eq!(grown.capacity, 4 * 1024 * 1024); + *allocator.stale_query_once.lock().unwrap() = old_layout; + store.durable_cursor(active.chunk_id, 9).await.unwrap(); + assert_eq!( + store.current_capacity(active.chunk_id).await.unwrap(), + 4 * 1024 * 1024 + ); + disks.fail_disk_once.store(1, Ordering::Release); + let second = strip.append(&store, name, 9, active.chunk_id, b"second").await; + assert!(second > first); + assert_eq!(allocator.replacements.load(Ordering::Acquire), 1); + assert_eq!(store.durable_cursor(active.chunk_id, 9).await.unwrap().offset, 0); + let end = strip.append(&store, name, 9, active.chunk_id, b"third").await; + assert_eq!( + allocator + .chunk + .lock() + .unwrap() + .as_ref() + .unwrap() + .acknowledged_cursor, + 0 + ); + store.seal(active.chunk_id, 9, end).await.unwrap(); + let durable = store.durable_cursor(active.chunk_id, 9).await.unwrap(); + assert_eq!(durable.offset, end); + assert!(durable.sealed); +} + +#[tokio::test] +async fn a_grouped_chunk_rejects_removed_magic_after_synchronized_progress() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let name = StreamName { high: 71, low: 73 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let mut strip = TestGroupedStrip::default(); + let offset = strip.append(&store, name, 9, active.chunk_id, b"first").await; + let mut removed = crowdb_protocol::frame::encode_append_group_views( + active.chunk_id, + offset, + &Bytes::from_static(b"unsupported"), + 1, + ) + .unwrap() + .concat(); + removed[..2].copy_from_slice(&0x0301_u16.to_le_bytes()); + let legacy = Bytes::from(removed); + assert!(matches!( + store + .write_mirrors( + name, + 9, + active.chunk_id, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: offset, + complete_end: offset + legacy.len() as u64, + images: &[MirrorStripImage { + block_offset: offset, + data: legacy.clone(), + full_image: legacy + }] + } + ) + .await, + Err(crowdb_chunk_stream::StreamError::InvalidRequest(_)) + )); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 4); + assert_eq!(store.durable_cursor(active.chunk_id, 9).await.unwrap().offset, 0); +} + +#[tokio::test] +async fn a_cold_adapter_cannot_overwrite_the_previous_owners_uncheckpointed_group() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let name = StreamName { high: 17, low: 19 }; + let original = make(); + let active = original + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let mut strip = TestGroupedStrip::default(); + strip + .append(&original, name, 9, active.chunk_id, b"retained") + .await; + assert_eq!( + original.durable_cursor(active.chunk_id, 9).await.unwrap().offset, + 0 + ); + let retained = disks.bytes.lock().unwrap().clone(); + let cold = make(); + let views = crowdb_protocol::frame::encode_append_group_views( + active.chunk_id, + 0, + &Bytes::from_static(b"replacement"), + 1, + ) + .unwrap(); + let bytes = Bytes::from(views.concat()); + let complete_end = bytes.len() as u64; + assert!(matches!( + cold.write_mirrors( + name, + 9, + active.chunk_id, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: 0, + complete_end, + images: &[MirrorStripImage { + block_offset: 0, + data: bytes.clone(), + full_image: bytes + }] + } + ) + .await, + Err(crowdb_chunk_stream::StreamError::InvalidRequest(_)) + )); + assert_eq!(*disks.bytes.lock().unwrap(), retained); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 4); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/owner_capture.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/owner_capture.rs new file mode 100644 index 000000000..193291dd7 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/owner_capture.rs @@ -0,0 +1,324 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_chunk_stream::{StreamPurpose, StreamViewSource}; +use crowdb_kv::cluster::{ + group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; + +struct TestRuntimeKv(Arc); + +impl Drop for TestRuntimeKv { + fn drop(&mut self) { + self.0.stop(); + } +} + +async fn runtime_kv() -> (TestRuntimeKv, Arc) { + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + for group in 0..=1 { + store.add_group(PxGroup::new( + group, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + } + store.start().await.unwrap(); + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + for group in 0..=1 { + kv.seed_leader(0, group, store.listen_addr().unwrap().to_string()); + } + (TestRuntimeKv(store), kv) +} + +#[tokio::test] +async fn runtime_captures_local_owner_and_identifies_older_remote_checkpoint() { + let (_server, kv) = runtime_kv().await; + let chunk_io = ChunkIoClient::from_parts( + Arc::new(Allocator::new()), + Arc::new(Disks::default()), + crowdb_protocol::chunkdb::rpc::ChunkType::Stream, + ); + let make_runtime = || { + ProductionStreamRuntime::new( + kv.clone(), + &chunk_io, + 30_000, + ChunkReadPolicy::default(), + StreamConfig::default(), + ) + .unwrap() + }; + let owner = make_runtime(); + let remote = make_runtime(); + let name = StreamName::generate(); + owner + .registry() + .create(StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }) + .await + .unwrap(); + let writer = owner.create_registered(name, 0, 1).await.unwrap(); + writer.append(&[Bytes::from_static(b"first")]).await.unwrap(); + let fallback = remote.open_read_only_current(name, 0).await.unwrap(); + assert_eq!(fallback.view_source(), StreamViewSource::PersistedCheckpoint); + assert_eq!(fallback.tail(), 0); + let captured = owner.open_read_only(name, 0, 1).await.unwrap(); + assert_eq!(captured.view_source(), StreamViewSource::OwnerCheckpoint); + assert_eq!( + captured.read_at(0, 5).await.unwrap(), + Bytes::from_static(b"first") + ); + writer.append(&[Bytes::from_static(b"second")]).await.unwrap(); + let fallback = remote.open_read_only_current(name, 0).await.unwrap(); + assert_eq!(fallback.tail(), 5); + assert_eq!( + fallback.read_at(0, 5).await.unwrap(), + Bytes::from_static(b"first") + ); + assert_eq!(captured.tail(), 5); + assert_eq!(owner.open_read_only_current(name, 0).await.unwrap().tail(), 11); + writer.close().await.unwrap(); + assert_eq!(owner.open_read_only(name, 0, 1).await.unwrap().tail(), 11); + let successor = owner.registry().claim_writer(name, 2).await.unwrap(); + writer.release_writer().await.unwrap(); + successor.capture().unwrap().check_io().unwrap(); + successor.release().await.unwrap(); +} + +#[tokio::test] +async fn owner_claim_is_exclusive_and_an_old_release_cannot_release_its_successor() { + let (_server, kv) = runtime_kv().await; + let registry = crowdb_chunk_stream::KvStreamRegistry::new(kv); + let name = StreamName::generate(); + registry + .create(StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }) + .await + .unwrap(); + let first = registry.claim_writer(name, 1).await.unwrap(); + assert!(matches!( + registry.claim_writer(name, 1).await, + Err(crowdb_chunk_stream::StreamError::StaleWriter) + )); + first.release().await.unwrap(); + let (a, b) = tokio::join!(registry.claim_writer(name, 1), registry.claim_writer(name, 1)); + assert_ne!(a.is_ok(), b.is_ok(), "only one concurrent claim can win"); + let successor = a.or(b).unwrap(); + assert!( + first.release().await.is_err(), + "an old assignment cannot release its successor even at the same epoch" + ); + successor.capture().unwrap().check_io().unwrap(); + assert!(registry.claim_writer(name, 2).await.is_err()); + successor.release().await.unwrap(); + registry + .claim_writer(name, 2) + .await + .unwrap() + .release() + .await + .unwrap(); +} + +#[tokio::test] +async fn drained_release_allows_another_runtime_to_take_over_without_waiting_for_process_expiry() { + let (_server, kv) = runtime_kv().await; + let first = crowdb_chunk_stream::KvStreamRegistry::new(kv.clone()); + let second = crowdb_chunk_stream::KvStreamRegistry::new(kv); + let name = StreamName::generate(); + first + .create(StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }) + .await + .unwrap(); + let old = first.claim_writer(name, 1).await.unwrap(); + old.release().await.unwrap(); + let new = tokio::time::timeout(std::time::Duration::from_secs(1), second.claim_writer(name, 2)) + .await + .unwrap() + .unwrap(); + assert!(old.capture().is_err()); + new.release().await.unwrap(); +} + +#[tokio::test] +async fn failed_initial_allocation_releases_the_unpublished_writer_assignment() { + let (_server, kv) = runtime_kv().await; + let registry = Arc::new(crowdb_chunk_stream::KvStreamRegistry::new(kv)); + let memory = Arc::new(MemoryStreamStore::new(1024)); + let name = StreamName::generate(); + registry + .create(StreamRecord { + metadata_group_id: 1, + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }) + .await + .unwrap(); + memory.fail_allocation_for_tests(1); + let first = ChunkStream::create_registered( + name, + 1, + StreamConfig::default(), + registry.clone(), + memory.clone(), + memory.clone(), + ) + .await; + assert!(matches!( + first, + Err(crowdb_chunk_stream::StreamError::ReadUnavailable(_)) + )); + let writer = + ChunkStream::create_registered(name, 1, StreamConfig::default(), registry, memory.clone(), memory) + .await + .unwrap(); + writer.append(&[Bytes::from_static(b"ready")]).await.unwrap(); + assert_eq!(writer.read_at(0, 5).await.unwrap(), Bytes::from_static(b"ready")); + writer.release_writer().await.unwrap(); +} + +#[tokio::test] +async fn failed_reopen_releases_its_assignment_before_retrying_in_the_same_process() { + let (_server, kv) = runtime_kv().await; + let registry = Arc::new(crowdb_chunk_stream::KvStreamRegistry::new(kv)); + let memory = Arc::new(MemoryStreamStore::new(1024)); + let name = StreamName::generate(); + registry + .create(StreamRecord { + metadata_group_id: 1, + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }) + .await + .unwrap(); + let writer = ChunkStream::create_registered( + name, + 1, + StreamConfig::default(), + registry.clone(), + memory.clone(), + memory.clone(), + ) + .await + .unwrap(); + writer.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + writer.release_writer().await.unwrap(); + let next = memory + .chunks_for_tests() + .await + .into_iter() + .map(|(id, ..)| id.low) + .max() + .unwrap() + + 1; + memory.fail_allocation_for_tests(next); + let failed = ChunkStream::open( + name, + 2, + StreamConfig::default(), + registry.clone(), + memory.clone(), + memory.clone(), + ) + .await; + assert!(matches!( + failed, + Err(crowdb_chunk_stream::StreamError::ReadUnavailable(_)) + )); + let recovered = ChunkStream::open(name, 2, StreamConfig::default(), registry, memory.clone(), memory) + .await + .expect("a failed recovery cannot retain an unpublished writer assignment"); + assert_eq!( + recovered.read_at(0, 8).await.unwrap(), + Bytes::from_static(b"retained") + ); + recovered.append(&[Bytes::from_static(b"-after")]).await.unwrap(); + assert_eq!( + recovered.read_at(0, 14).await.unwrap(), + Bytes::from_static(b"retained-after") + ); + recovered.release_writer().await.unwrap(); +} + +#[tokio::test] +async fn release_during_a_pending_claim_wakes_the_successor_without_process_expiry() { + let (_server, kv) = runtime_kv().await; + let first = crowdb_chunk_stream::KvStreamRegistry::new(kv.clone()); + let second = crowdb_chunk_stream::KvStreamRegistry::new(kv); + let name = StreamName::generate(); + first + .create(StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }) + .await + .unwrap(); + let owner = first.claim_writer(name, 1).await.unwrap(); + let pending = second.claim_writer(name, 2); + tokio::pin!(pending); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), pending.as_mut()) + .await + .is_err(), + "a live writer cannot be preempted" + ); + owner.release().await.unwrap(); + let successor = tokio::time::timeout(std::time::Duration::from_secs(1), pending) + .await + .expect("durable release must wake an already waiting claim") + .unwrap(); + assert!(owner.capture().is_err()); + successor.release().await.unwrap(); +} + +#[tokio::test] +async fn recovery_waiting_for_release_loads_the_final_published_tail() { + let (_server, kv) = runtime_kv().await; + let first = Arc::new(crowdb_chunk_stream::KvStreamRegistry::new(kv.clone())); + let second = Arc::new(crowdb_chunk_stream::KvStreamRegistry::new(kv)); + let memory = Arc::new(MemoryStreamStore::new(1024)); + let name = StreamName::generate(); + let writer = ChunkStream::create( + StreamRecord { + metadata_group_id: 1, + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, StreamPurpose::Stream) + }, + 1, + StreamConfig::default(), + first, + memory.clone(), + memory.clone(), + ) + .await + .unwrap(); + writer.append(&[Bytes::from_static(b"before")]).await.unwrap(); + let pending = ChunkStream::open(name, 2, StreamConfig::default(), second, memory.clone(), memory); + tokio::pin!(pending); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), pending.as_mut()) + .await + .is_err(), + "recovery must wait for the live writer" + ); + writer.append(&[Bytes::from_static(b"-after")]).await.unwrap(); + writer.release_writer().await.unwrap(); + let successor = tokio::time::timeout(std::time::Duration::from_secs(1), pending) + .await + .unwrap() + .expect("recovery must use the checkpoint published by the release it awaited"); + assert_eq!( + successor.read_at(0, 12).await.unwrap(), + Bytes::from_static(b"before-after") + ); + successor.append(&[Bytes::from_static(b"-next")]).await.unwrap(); + successor.release_writer().await.unwrap(); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs new file mode 100644 index 000000000..152b8b7ae --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/preparation.rs @@ -0,0 +1,310 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use tokio::time::{timeout, Duration}; + +#[tokio::test] +async fn preparation_revision_does_not_abort_or_reinitialize_mirror_repair() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let name = StreamName { high: 241, low: 251 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + allocator.conflict_replacement_once.store(true, Ordering::Release); + disks.fail_disk_once.store(1, Ordering::Release); + let end = TestGroupedStrip::default() + .append(&store, name, 9, active.chunk_id, b"retained") + .await; + assert!(end > 0); + assert_eq!(allocator.replacement_calls.load(Ordering::Acquire), 2); + assert_eq!(allocator.replacements.load(Ordering::Acquire), 1); + let writes = disks.writes.lock().unwrap(); + assert_eq!( + writes + .iter() + .filter(|(segment, _, _, _)| segment.disk_id.is_some_and(|id| id.high == 101)) + .count(), + 1 + ); +} + +#[tokio::test] +async fn growth_publication_preserves_cursor_progress_committed_during_allocation() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let name = StreamName { high: 233, low: 239 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let end = TestGroupedStrip::default() + .append(&store, name, 9, active.chunk_id, b"retained") + .await; + allocator.advance_during_growth.store(end, Ordering::Release); + let prepared = store + .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .await + .unwrap() + .unwrap(); + assert_eq!(prepared.acknowledged_cursor, end); + let writes = disks.writes.lock().unwrap().len(); + assert_eq!( + store + .advance_cursor(name, 9, active.chunk_id, end, end, 17) + .await + .unwrap(), + CursorAdvance::Committed + ); + assert_eq!(disks.writes.lock().unwrap().len(), writes); +} + +#[tokio::test] +async fn temporary_allocation_failure_does_not_poison_later_capacity() { + let allocator = Arc::new(Allocator::new()); + allocator.pause_growth.store(true, Ordering::Release); + allocator.fail_growth_once.store(true, Ordering::Release); + let stream = test_stream(allocator.clone(), Arc::new(Disks::default())).await; + stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + timeout(Duration::from_secs(1), allocator.growth_started.notified()) + .await + .unwrap(); + allocator.growth_resume.notify_one(); + let data = Bytes::from(vec![0x61; 1024 * 1024]); + let range = timeout(Duration::from_secs(1), stream.append(std::slice::from_ref(&data))) + .await + .unwrap() + .unwrap(); + assert_eq!(range.begin, 8); + assert_eq!(stream.read_at(range.begin, data.len()).await.unwrap(), data); + stream.close().await.unwrap(); +} + +#[tokio::test] +async fn uncertain_unused_candidate_is_reclaimed_and_replaced_without_reinitialization() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + // The primary chunk uses disks 1/2, 11/12, ...; its first candidate + // has distinct disks 1001/1002, 1011/1012, ... . Fail candidate growth. + disks.fail_fsync_once.store(1011, Ordering::Release); + let stream = test_stream(allocator.clone(), disks.clone()).await; + stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + timeout(Duration::from_secs(1), async { + while allocator.successors.lock().unwrap().len() < 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + stream.close().await.unwrap(); + { + let candidates = allocator.successors.lock().unwrap(); + assert_eq!(candidates.len(), 2); + assert!(candidates + .values() + .all(|chunk| chunk.state == ChunkState::Deleted as i32 && chunk.acknowledged_cursor == 0)); + } + assert_eq!( + disks + .writes + .lock() + .unwrap() + .iter() + .filter(|(segment, ..)| segment.disk_id.unwrap().high == 1011) + .count(), + 1, + "uncertain initialization must never repeat on that candidate" + ); + assert_eq!(stream.metrics().failed, 0); + assert_eq!(stream.metrics().rollovers, 0); + assert_eq!( + stream.read_at(0, 8).await.unwrap(), + Bytes::from_static(b"retained") + ); +} + +async fn test_stream(allocator: Arc, disks: Arc) -> ChunkStream { + let metadata = Arc::new(MemoryStreamStore::new(256 * 1024 * 1024)); + let chunks = Arc::new( + ProductionStreamChunkStore::new(allocator, disks, 30_000, ChunkReadPolicy::default()).unwrap(), + ); + ChunkStream::create( + StreamRecord { + purpose: crowdb_chunk_stream::StreamPurpose::Stream, + stream_name: StreamName { high: 211, low: 223 }, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 9, + StreamConfig::default(), + metadata.clone(), + metadata, + chunks, + ) + .await + .unwrap() +} + +#[tokio::test] +async fn allocation_in_flight_does_not_block_writes_in_ready_capacity() { + let allocator = Arc::new(Allocator::new()); + allocator.pause_growth.store(true, Ordering::Release); + let stream = test_stream(allocator.clone(), Arc::new(Disks::default())).await; + let payload = Bytes::from(vec![0x51; 4096]); + timeout( + Duration::from_secs(1), + stream.append(std::slice::from_ref(&payload)), + ) + .await + .unwrap() + .unwrap(); + timeout(Duration::from_secs(1), allocator.growth_started.notified()) + .await + .unwrap(); + let range = timeout( + Duration::from_secs(1), + stream.append(std::slice::from_ref(&payload)), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(range.begin, 4096); + allocator.growth_resume.notify_one(); + stream.close().await.unwrap(); + assert_eq!(stream.read_at(range.begin, payload.len()).await.unwrap(), payload); +} + +#[tokio::test] +async fn only_a_write_reaching_uninitialized_capacity_waits() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + disks.pause_initialization.store(true, Ordering::Release); + let stream = test_stream(allocator.clone(), disks.clone()).await; + stream.append(&[Bytes::from(vec![0x31; 4096])]).await.unwrap(); + timeout(Duration::from_secs(1), disks.initialization_started.notified()) + .await + .unwrap(); + timeout( + Duration::from_secs(1), + stream.append(&[Bytes::from(vec![0x32; 4096])]), + ) + .await + .unwrap() + .unwrap(); + let large = Bytes::from(vec![0x33; 1024 * 1024]); + let writer = stream.clone(); + let data = large.clone(); + let mut pending = tokio::spawn(async move { writer.append(&[data]).await }); + assert!(timeout(Duration::from_millis(20), &mut pending).await.is_err()); + assert_eq!(stream.tail(), 8192); + disks.initialization_resume.notify_one(); + let range = timeout(Duration::from_secs(1), pending) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(range.begin, 8192); + assert_eq!(stream.read_at(range.begin, large.len()).await.unwrap(), large); + stream.close().await.unwrap(); +} + +#[tokio::test] +async fn close_drains_preparation_before_sealing() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + disks.pause_initialization.store(true, Ordering::Release); + let stream = test_stream(allocator.clone(), disks.clone()).await; + stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + timeout(Duration::from_secs(1), disks.initialization_started.notified()) + .await + .unwrap(); + let mut closing = tokio::spawn(async move { stream.close().await }); + assert!(timeout(Duration::from_millis(20), &mut closing).await.is_err()); + assert_eq!( + allocator.chunk.lock().unwrap().as_ref().unwrap().state, + ChunkState::Active as i32 + ); + disks.initialization_resume.notify_one(); + timeout(Duration::from_secs(1), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + allocator.chunk.lock().unwrap().as_ref().unwrap().state, + ChunkState::Sealed as i32 + ); +} + +#[tokio::test] +async fn failed_preparation_never_exposes_or_reinitializes_the_new_suffix() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + disks.fail_fsync_once.store(11, Ordering::Release); + let stream = test_stream(allocator, disks.clone()).await; + stream.append(&[Bytes::from_static(b"retained")]).await.unwrap(); + assert!(stream + .append(&[Bytes::from(vec![0x41; 1024 * 1024])]) + .await + .is_err()); + assert_eq!(stream.tail(), 8); + assert_eq!( + stream.read_at(0, 8).await.unwrap(), + Bytes::from_static(b"retained") + ); + let attempts = disks.writes.lock().unwrap().len(); + assert!(stream.append(&[Bytes::from_static(b"later")]).await.is_err()); + assert_eq!(disks.writes.lock().unwrap().len(), attempts); +} + +#[tokio::test] +async fn definite_revision_conflict_rebases_cursor_without_replaying_data() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let name = StreamName { high: 227, low: 229 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let end = TestGroupedStrip::default() + .append(&store, name, 9, active.chunk_id, b"retained") + .await; + let writes = disks.writes.lock().unwrap().len(); + allocator.conflict_advance_once.store(true, Ordering::Release); + assert_eq!( + store + .advance_cursor(name, 9, active.chunk_id, 0, end, 17) + .await + .unwrap(), + CursorAdvance::Committed + ); + assert_eq!(allocator.advance_calls.load(Ordering::Acquire), 2); + assert_eq!(disks.writes.lock().unwrap().len(), writes); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs new file mode 100644 index 000000000..d3bb7085e --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/recovery.rs @@ -0,0 +1,88 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +async fn recover_case(shorter: bool, corrupt: bool) { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let make = || { + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap() + }; + let writer = make(); + let name = StreamName { high: 801, low: 91 }; + let active = writer + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let mut encoded = TestGroupedStrip::default(); + let first_end = encoded.append(&writer, name, 9, active.chunk_id, b"first").await; + let final_end = encoded.append(&writer, name, 9, active.chunk_id, b"second").await; + assert_eq!(allocator.advance_calls.load(Ordering::Acquire), 0); + assert_eq!( + allocator + .chunk + .lock() + .unwrap() + .as_ref() + .unwrap() + .acknowledged_cursor, + 0 + ); + { + let mut bytes = disks.bytes.lock().unwrap(); + let replica = bytes.get_mut(&2).unwrap(); + if shorter { + replica[usize::try_from(first_end).unwrap()..].fill(0); + } + if corrupt { + replica[20] ^= 0x40; + } + } + drop(writer); + let cold = make(); + let memory = MemoryStreamStore::new(1024); + cold.bind_writer(name, 10, memory.claim_writer(name, 10).await.unwrap()); + let recovered = cold.recover_tail(name, 10, &active).await.unwrap(); + assert_eq!( + recovered.physical_end, + if shorter { first_end } else { final_end } + ); + assert_eq!(recovered.logical_end, if shorter { 5 } else { 11 }); + assert_eq!(recovered.extents.len(), if shorter { 1 } else { 2 }); + assert_eq!(allocator.advance_calls.load(Ordering::Acquire), 0); + cold.seal(active.chunk_id, 10, recovered.physical_end) + .await + .unwrap(); + assert_eq!( + allocator + .chunk + .lock() + .unwrap() + .as_ref() + .unwrap() + .acknowledged_cursor, + recovered.physical_end + ); +} + +#[tokio::test] +async fn cold_adapter_recovers_complete_writes_beyond_zero_checkpoint() { + recover_case(false, false).await; +} + +#[tokio::test] +async fn cold_adapter_uses_the_shortest_healthy_complete_prefix() { + recover_case(true, false).await; +} + +#[tokio::test] +async fn cold_adapter_does_not_turn_a_corrupt_replica_into_an_empty_tail() { + recover_case(false, true).await; +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/scope.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/scope.rs new file mode 100644 index 000000000..b07f9d7ac --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/scope.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; + +#[tokio::test] +async fn allocation_rejects_fewer_than_the_configured_copies() { + for copies in [3, 5] { + let allocator = Arc::new(Allocator::new()); + allocator.allocation_copy_limit.store(2, Ordering::Release); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new_with_mirror_copies( + allocator, + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + copies, + ) + .unwrap(); + let result = store + .allocate_mirrored( + StreamName { high: 41, low: 43 }, + 9, + crowdb_chunk_stream::StreamPurpose::Stream, + ) + .await; + assert!( + matches!(result, Err(crowdb_chunk_stream::StreamError::ReadUnavailable(_))), + "{copies}-copy allocation accepted an incomplete replica set: {result:?}" + ); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 0); + } +} + +#[tokio::test] +async fn another_stream_cannot_write_or_checkpoint_an_attributed_chunk() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new(allocator, disks.clone(), 30_000, ChunkReadPolicy::default()) + .unwrap(); + let name = StreamName { high: 81, low: 83 }; + let other = StreamName { high: 81, low: 84 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let data = Bytes::from( + crowdb_protocol::frame::encode_append_group_views( + active.chunk_id, + 0, + &Bytes::from_static(b"retained"), + 1, + ) + .unwrap() + .concat(), + ); + let end = data.len() as u64; + let images = [MirrorStripImage { + block_offset: 0, + data: data.clone(), + full_image: data.clone(), + }]; + assert!(matches!( + store + .write_mirrors( + other, + 9, + active.chunk_id, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: 0, + complete_end: end, + images: &images + } + ) + .await, + Err(crowdb_chunk_stream::StreamError::StaleWriter) + )); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 2); + store + .write_mirrors( + name, + 9, + active.chunk_id, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: 0, + complete_end: end, + images: &images, + }, + ) + .await + .unwrap(); + assert!(matches!( + store.advance_cursor(other, 9, active.chunk_id, 0, end, 7).await, + Err(crowdb_chunk_stream::StreamError::StaleWriter) + )); + assert_eq!(store.durable_cursor(active.chunk_id, 9).await.unwrap().offset, 0); + assert_eq!( + store + .advance_cursor(name, 9, active.chunk_id, 0, end, 7) + .await + .unwrap(), + CursorAdvance::Committed + ); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 4); +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/sealing.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/sealing.rs new file mode 100644 index 000000000..08feede86 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/sealing.rs @@ -0,0 +1,60 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use std::time::Duration; + +#[tokio::test(start_paused = true)] +async fn seal_bounds_parallel_segment_drains_and_waits_for_every_completion() { + for fail in [false, true] { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let memory = MemoryStreamStore::new(1024); + let name = StreamName::generate(); + store.bind_writer(name, 1, memory.claim_writer(name, 1).await.unwrap()); + let active = store + .allocate_mirrored(name, 1, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + store + .grow_mirrored(name, 1, active.chunk_id, active.capacity * 16) + .await + .unwrap() + .unwrap(); + let expected = allocator.chunk.lock().unwrap().as_ref().unwrap().strips.len() * 2; + assert!(expected > 8); + disks.drain_delay_ms.store(10, Ordering::Release); + if fail { + disks.fail_fsync_once.store(1, Ordering::Release); + } + let sealing = store.seal(active.chunk_id, 1, 0); + tokio::pin!(sealing); + assert!(tokio::time::timeout(Duration::from_millis(1), sealing.as_mut()) + .await + .is_err()); + assert_eq!(disks.active_drains.load(Ordering::Acquire), 8); + assert_eq!( + allocator.chunk.lock().unwrap().as_ref().unwrap().state, + ChunkState::Active as i32 + ); + assert_eq!(sealing.await.is_err(), fail); + assert_eq!(disks.max_active_drains.load(Ordering::Acquire), 8); + assert_eq!(disks.completed_drains.load(Ordering::Acquire), expected); + assert_eq!(disks.active_drains.load(Ordering::Acquire), 0); + assert_eq!( + allocator.chunk.lock().unwrap().as_ref().unwrap().state, + if fail { + ChunkState::Active + } else { + ChunkState::Sealed + } as i32 + ); + } +} diff --git a/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs b/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs new file mode 100644 index 000000000..24a2b3127 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/production_chunk_test/unwritten.rs @@ -0,0 +1,313 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_chunk_client::{select_frame_prefix, MirroredFrameReaders}; +use crowdb_protocol::frame::{encode_append_group_views, MAX_FRAME_GROUP_PAYLOAD_BYTES}; +use tokio::time::{Duration, Instant}; + +fn prepared_group(chunk: ChunkId, payload: &Bytes, padding: usize) -> (u64, MirrorStripImage) { + let mut data = encode_append_group_views(chunk, 0, payload, 1).unwrap().concat(); + let complete_end = data.len() as u64; + data.resize(data.len() + padding, 0); + let full_image = Bytes::from(data); + ( + complete_end, + MirrorStripImage { + block_offset: 0, + data: full_image.clone(), + full_image, + }, + ) +} + +async fn observed_end(allocator: &Allocator, disks: &Arc, name: StreamName) -> u64 { + let chunk = Arc::new(allocator.chunk.lock().unwrap().clone().unwrap()); + let deadline = Instant::now() + Duration::from_secs(5); + let readers = MirroredFrameReaders::new(chunk, name, disks.clone(), deadline).unwrap(); + let config = readers.scan_config(0, 0, 0, 64 * 1024 * 1024).unwrap(); + select_frame_prefix(readers.readers(), config, 1024 * 1024, deadline) + .await + .unwrap() + .observation() + .summary() + .physical_end +} + +#[tokio::test] +async fn recycled_bytes_stop_at_the_synchronized_unwritten_header() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + for disk in [1, 2] { + disks.bytes.lock().unwrap().insert(disk, vec![0xcc; 1024 * 1024]); + } + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let name = StreamName { high: 101, low: 103 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + assert_eq!(observed_end(&allocator, &disks, name).await, 0); + { + let media = disks.bytes.lock().unwrap(); + assert_eq!(&media[&1][..14], &[0; 14]); + assert!(media[&1].iter().all(|byte| *byte == 0)); + } + let mut strip = TestGroupedStrip::default(); + let end = strip.append(&store, name, 9, active.chunk_id, b"retained").await; + assert_eq!(store.durable_cursor(active.chunk_id, 9).await.unwrap().offset, 0); + assert_eq!(observed_end(&allocator, &disks, name).await, end); +} + +#[tokio::test] +async fn bytes_beyond_the_complete_range_are_rejected_before_consuming_the_fresh_writer() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new(allocator, disks.clone(), 30_000, ChunkReadPolicy::default()) + .unwrap(); + let name = StreamName { high: 107, low: 109 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let payload = Bytes::from_static(b"retained"); + for padding in [1, 14] { + let (end, mut image) = prepared_group(active.chunk_id, &payload, padding); + if padding > 0 { + let mut bytes = image.full_image.to_vec(); + *bytes.last_mut().unwrap() = 1; + image.full_image = Bytes::from(bytes); + image.data = image.full_image.clone(); + } + let result = store + .write_mirrors( + name, + 9, + active.chunk_id, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: 0, + complete_end: end, + images: &[image], + }, + ) + .await; + assert!( + matches!(result, Err(crowdb_chunk_stream::StreamError::InvalidRequest(_))), + "{result:?}" + ); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 2); + } + let mut strip = TestGroupedStrip::default(); + strip.append(&store, name, 9, active.chunk_id, b"retained").await; + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 4); +} + +#[tokio::test] +async fn exact_hard_capacity_needs_no_extra_header() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new_with_mirror_copies_and_capacity( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + 2, + 1024 * 1024, + ) + .unwrap(); + let name = StreamName { high: 113, low: 127 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let (end, image) = prepared_group( + active.chunk_id, + &Bytes::from(vec![0x51; 16 * MAX_FRAME_GROUP_PAYLOAD_BYTES]), + 0, + ); + assert_eq!(end, active.capacity); + store + .write_mirrors( + name, + 9, + active.chunk_id, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: 0, + complete_end: end, + images: &[image], + }, + ) + .await + .unwrap(); + assert_eq!(observed_end(&allocator, &disks, name).await, end); +} + +#[tokio::test] +async fn cross_strip_tail_header_survives_later_growth_and_the_next_append() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + for disk in [1, 2, 11, 12, 21, 22] { + disks.bytes.lock().unwrap().insert(disk, vec![0xcc; 1024 * 1024]); + } + let chunks = Arc::new( + ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(), + ); + let metadata = Arc::new(MemoryStreamStore::new(1)); + let name = StreamName { high: 139, low: 149 }; + let stream = ChunkStream::create( + StreamRecord { + purpose: crowdb_chunk_stream::StreamPurpose::Stream, + stream_name: name, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 9, + StreamConfig::default(), + metadata.clone(), + metadata.clone(), + chunks.clone(), + ) + .await + .unwrap(); + let prefix = Bytes::from(vec![0x51; 16 * MAX_FRAME_GROUP_PAYLOAD_BYTES - 7]); + let range = stream.append(std::slice::from_ref(&prefix)).await.unwrap(); + let chunk_id = range.chunk_id.unwrap(); + let complete_end = 1024 * 1024 - 7; + assert_eq!(chunks.durable_cursor(chunk_id, 9).await.unwrap().offset, 0); + assert_eq!(chunks.current_capacity(chunk_id).await.unwrap(), 8 * 1024 * 1024); + assert_eq!(observed_end(&allocator, &disks, name).await, complete_end); + let publications = metadata.metadata_publish_count(); + chunks + .grow_mirrored(name, 9, chunk_id, 9 * 1024 * 1024) + .await + .unwrap() + .unwrap(); + assert_eq!(metadata.metadata_publish_count(), publications); + assert_eq!(observed_end(&allocator, &disks, name).await, complete_end); + stream.append(&[Bytes::from_static(b"next")]).await.unwrap(); + assert_eq!(stream.read_at(0, prefix.len()).await.unwrap(), prefix); + assert_eq!( + stream.read_at(range.end, 4).await.unwrap(), + Bytes::from_static(b"next") + ); + assert_eq!(observed_end(&allocator, &disks, name).await, complete_end + 51); + assert_eq!(allocator.advance_calls.load(Ordering::Acquire), 0); + stream.checkpoint().await.unwrap(); + assert_eq!( + chunks.durable_cursor(chunk_id, 9).await.unwrap().offset, + complete_end + 51 + ); + assert_eq!(allocator.advance_calls.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn failed_empty_initialization_cannot_expose_a_writable_allocation() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + disks.fail_fsync_once.store(1, Ordering::Release); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let name = StreamName { high: 131, low: 137 }; + assert!(store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .is_err()); + let chunk = allocator.chunk.lock().unwrap().clone().unwrap(); + let (end, image) = prepared_group(chunk.id.unwrap(), &Bytes::from_static(b"blocked"), 0); + let result = store + .write_mirrors( + name, + 9, + chunk.id.unwrap(), + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: 0, + complete_end: end, + images: &[image], + }, + ) + .await; + assert!( + matches!(result, Err(crowdb_chunk_stream::StreamError::InvalidRequest(_))), + "{result:?}" + ); + assert_eq!( + store.durable_cursor(chunk.id.unwrap(), 9).await.unwrap().offset, + 0 + ); + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn uncertain_growth_initialization_cannot_be_retried_or_written() { + let allocator = Arc::new(Allocator::new()); + let disks = Arc::new(Disks::default()); + let store = ProductionStreamChunkStore::new( + allocator.clone(), + disks.clone(), + 30_000, + ChunkReadPolicy::default(), + ) + .unwrap(); + let name = StreamName { high: 163, low: 167 }; + let active = store + .allocate_mirrored(name, 9, crowdb_chunk_stream::StreamPurpose::Stream) + .await + .unwrap(); + let mut strip = TestGroupedStrip::default(); + let end = strip.append(&store, name, 9, active.chunk_id, b"retained").await; + disks.fail_fsync_once.store(11, Ordering::Release); + assert!(store + .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .await + .is_err()); + // All three added strips complete their submitted work, including the healthy + // replica of the failing strip, before uncertain initialization is returned. + assert_eq!(disks.fsyncs.load(Ordering::Acquire), 9); + let submissions = disks.writes.lock().unwrap().len(); + assert!(store + .grow_mirrored(name, 9, active.chunk_id, active.capacity + 1) + .await + .is_err()); + assert_eq!(disks.writes.lock().unwrap().len(), submissions); + assert_eq!(observed_end(&allocator, &disks, name).await, end); + let (complete_end, image) = prepared_group(active.chunk_id, &Bytes::from(vec![7; 1024 * 1024]), 0); + assert!(store + .write_mirrors( + name, + 9, + active.chunk_id, + crowdb_chunk_stream::StreamMirrorWrite { + authority: None, + physical_offset: end, + complete_end: end + complete_end, + images: &[image], + } + ) + .await + .is_err()); + assert_eq!(disks.writes.lock().unwrap().len(), submissions); +} diff --git a/lib/crowdb-chunk-stream/tests/purpose_test.rs b/lib/crowdb-chunk-stream/tests/purpose_test.rs index 0f7c4f77b..9f2b00b4f 100644 --- a/lib/crowdb-chunk-stream/tests/purpose_test.rs +++ b/lib/crowdb-chunk-stream/tests/purpose_test.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use bytes::Bytes; use crowdb_chunk_stream::memory::MemoryStreamStore; use crowdb_chunk_stream::{ - ChunkStream, StreamBinding, StreamBindingState, StreamConfig, StreamMetadataStore, StreamName, - StreamPurpose, + ChunkStream, StreamConfig, StreamMetadataStore, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, }; #[tokio::test] @@ -15,10 +15,10 @@ async fn purpose_survives_rollover_failed_write_and_cold_writer_recovery() { for purpose in [StreamPurpose::Wal, StreamPurpose::Stream] { let store = Arc::new(MemoryStreamStore::new(128)); let name = StreamName::generate(); - let binding = StreamBinding { + let binding = StreamRecord { stream_name: name, - state: StreamBindingState::Active, - ..StreamBinding::creating(name, None, purpose) + state: StreamRecordState::Active, + ..StreamRecord::creating(name, None, purpose) }; let stream = ChunkStream::create( binding, @@ -56,19 +56,18 @@ async fn purpose_survives_rollover_failed_write_and_cold_writer_recovery() { ); let next = recovered.append(&[bytes]).await.unwrap(); assert!(purpose.matches(next.chunk_id.unwrap())); - let manifest = store.load_current(name).await.unwrap().unwrap(); - assert_eq!(manifest.purpose, purpose); - let mut changed = manifest.clone(); + let snapshot = store.load_tail_for_tests(name).await.unwrap().unwrap(); + assert_eq!(snapshot.record().purpose, purpose); + let mut changed = snapshot.record().clone(); changed.purpose = match purpose { StreamPurpose::Wal => StreamPurpose::Stream, StreamPurpose::Stream => StreamPurpose::Wal, }; + changed.generation += 1; assert!(store - .publish( - Some((manifest.writer_epoch, manifest.generation)), - changed, - Vec::new() - ) + .record_store(1) + .unwrap() + .publish(Some(&snapshot), changed, None, None) .await .is_err()); } diff --git a/lib/crowdb-chunk-stream/tests/read_publication_test.rs b/lib/crowdb-chunk-stream/tests/read_publication_test.rs deleted file mode 100644 index 0580129c1..000000000 --- a/lib/crowdb-chunk-stream/tests/read_publication_test.rs +++ /dev/null @@ -1,203 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -use async_trait::async_trait; -use bytes::Bytes; -use crowdb_chunk_stream::memory::MemoryStreamStore; -use crowdb_chunk_stream::{ - ChunkStream, Result, StreamBinding, StreamBindingState, StreamConfig, StreamExtentPage, StreamManifest, - StreamMetadataStore, StreamName, -}; -use std::sync::{ - atomic::{AtomicBool, Ordering}, - Arc, -}; -use tokio::sync::Notify; - -struct TestPublicationGate { - store: Arc, - pause: AtomicBool, - entered: Notify, - release: Notify, -} - -#[async_trait] -impl StreamMetadataStore for TestPublicationGate { - async fn load_current(&self, name: StreamName) -> Result> { - self.store.load_current(name).await - } - async fn load_extent_page( - &self, - name: StreamName, - epoch: u64, - generation: u64, - page: u64, - ) -> Result> { - self.store.load_extent_page(name, epoch, generation, page).await - } - async fn publish( - &self, - expected: Option<(u64, u64)>, - manifest: StreamManifest, - pages: Vec, - ) -> Result<()> { - if self.pause.swap(false, Ordering::AcqRel) { - self.entered.notify_one(); - self.release.notified().await; - } - self.store.publish(expected, manifest, pages).await - } - async fn reclaim_extent_pages_before( - &self, - name: StreamName, - generation: u64, - max_pages: usize, - ) -> Result { - self.store - .reclaim_extent_pages_before(name, generation, max_pages) - .await - } -} - -#[tokio::test] -async fn concurrent_readers_keep_the_published_extent_boundary_until_append_publication() { - let store = Arc::new(MemoryStreamStore::new(1024)); - let metadata = Arc::new(TestPublicationGate { - store: store.clone(), - pause: AtomicBool::new(false), - entered: Notify::new(), - release: Notify::new(), - }); - let name = StreamName { high: 1, low: 1 }; - let writer = ChunkStream::create( - StreamBinding { - purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, - stream_name: name, - metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("test".into()), - }, - 9, - StreamConfig::default(), - store.clone(), - metadata.clone(), - store.clone(), - ) - .await - .unwrap(); - writer.append(&[Bytes::from_static(b"old")]).await.unwrap(); - metadata.pause.store(true, Ordering::Release); - let append_writer = writer.clone(); - let append = tokio::spawn(async move { append_writer.append(&[Bytes::from_static(b"new")]).await }); - tokio::time::timeout(std::time::Duration::from_secs(1), metadata.entered.notified()) - .await - .unwrap(); - // Mirror cursor has advanced; the corresponding directory is still unpublished. - let reader = ChunkStream::open_read_only( - name, - 9, - StreamConfig::default(), - store.clone(), - metadata.clone(), - store.clone(), - ) - .await - .unwrap(); - assert_eq!(writer.tail(), 3); - assert_eq!(reader.tail(), 3); - assert_eq!(writer.read_at(0, 3).await.unwrap(), Bytes::from_static(b"old")); - assert_eq!(reader.read_at(0, 3).await.unwrap(), Bytes::from_static(b"old")); - metadata.release.notify_one(); - append.await.unwrap().unwrap(); - assert_eq!(writer.tail(), 6); - assert_eq!(writer.read_at(0, 6).await.unwrap(), Bytes::from_static(b"oldnew")); - assert_eq!(reader.tail(), 3); -} - -#[tokio::test] -async fn writer_reopen_recovers_verified_frames_after_cursor_advance_before_publication() { - verify_crash_recovery(false).await; -} - -#[tokio::test] -async fn writer_reopen_rejects_corrupt_unpublished_durable_frames() { - verify_crash_recovery(true).await; -} - -async fn verify_crash_recovery(corrupt: bool) { - let store = Arc::new(MemoryStreamStore::new(1024)); - let metadata = Arc::new(TestPublicationGate { - store: store.clone(), - pause: AtomicBool::new(false), - entered: Notify::new(), - release: Notify::new(), - }); - let name = StreamName { high: 1, low: 1 }; - let writer = ChunkStream::create( - StreamBinding { - purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, - stream_name: name, - metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("test".into()), - }, - 9, - StreamConfig::default(), - store.clone(), - metadata.clone(), - store.clone(), - ) - .await - .unwrap(); - writer.append(&[Bytes::from_static(b"old")]).await.unwrap(); - metadata.pause.store(true, Ordering::Release); - let append_writer = writer.clone(); - let append = tokio::spawn(async move { append_writer.append(&[Bytes::from_static(b"new")]).await }); - tokio::time::timeout(std::time::Duration::from_secs(1), metadata.entered.notified()) - .await - .unwrap(); - if corrupt { - let active = writer.observe_metadata(None, 0).unwrap().active.unwrap(); - store - .flip_durable_byte( - active.chunk_id, - usize::try_from(active.acknowledged_cursor).unwrap() - + crowdb_protocol::frame::FRAME_HEADER_PREFIX_BYTES, - ) - .await; - } - // Simulate process loss after the durable cursor, before directory publication. - let recovered = ChunkStream::open( - name, - 10, - StreamConfig::default(), - store.clone(), - metadata.clone(), - store.clone(), - ) - .await; - if corrupt { - assert!(matches!( - recovered, - Err(crowdb_chunk_stream::StreamError::Corruption(_)) - )); - metadata.release.notify_one(); - append.await.unwrap().unwrap(); - return; - } - let recovered = recovered.unwrap(); - assert_eq!(recovered.tail(), 6); - assert_eq!( - recovered.read_at(0, 6).await.unwrap(), - Bytes::from_static(b"oldnew") - ); - metadata.release.notify_one(); - assert!(append.await.unwrap().is_err()); - recovered.append(&[Bytes::from_static(b"end")]).await.unwrap(); - assert_eq!( - recovered.read_at(0, 9).await.unwrap(), - Bytes::from_static(b"oldnewend") - ); -} diff --git a/lib/crowdb-chunk-stream/tests/rollover_growth_test.rs b/lib/crowdb-chunk-stream/tests/rollover_growth_test.rs new file mode 100644 index 000000000..80031fa32 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/rollover_growth_test.rs @@ -0,0 +1,274 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ActiveChunkDescriptor, ChunkStream, CursorAdvance, DurableCursor, Result, + StreamChunkStore, StreamConfig, StreamMetadataStore, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, StreamRegistry, TrimmedChunk, +}; +use crowdb_protocol::common::ChunkId; + +const INITIAL: u64 = 1024 * 1024; +const CAPACITY: u64 = 2 * INITIAL; + +/// Models production allocation: one strip initially, bounded growth afterward. +struct TestGrowingChunks { + store: Arc, +} + +#[async_trait] +impl StreamChunkStore for TestGrowingChunks { + fn bind_writer( + &self, + name: StreamName, + epoch: u64, + session: Arc, + ) { + self.store.bind_writer(name, epoch, session); + } + fn mirror_copies(&self) -> u32 { + self.store.mirror_copies() + } + async fn current_capacity(&self, chunk: ChunkId) -> Result { + self.store.current_capacity(chunk).await + } + async fn allocate_mirrored( + &self, + name: StreamName, + epoch: u64, + purpose: StreamPurpose, + ) -> Result { + let mut active = self.store.allocate_mirrored(name, epoch, purpose).await?; + active.capacity = INITIAL; + Ok(active) + } + + async fn grow_mirrored( + &self, + _name: StreamName, + epoch: u64, + chunk: ChunkId, + required: u64, + ) -> Result> { + if required > CAPACITY { + return Ok(None); + } + let cursor = self.store.durable_cursor(chunk, epoch).await?; + Ok(Some(ActiveChunkDescriptor { + chunk_id: chunk, + physical_start: cursor.offset, + logical_start: 0, + acknowledged_cursor: cursor.offset, + capacity: CAPACITY, + })) + } + + async fn write_mirrors( + &self, + name: StreamName, + epoch: u64, + chunk: ChunkId, + write: crowdb_chunk_stream::StreamMirrorWrite<'_>, + ) -> Result<()> { + self.store.write_mirrors(name, epoch, chunk, write).await + } + + async fn advance_cursor( + &self, + name: StreamName, + epoch: u64, + chunk: ChunkId, + expected: u64, + new: u64, + checksum: u32, + ) -> Result { + self.store + .advance_cursor(name, epoch, chunk, expected, new, checksum) + .await + } + + async fn durable_cursor(&self, chunk: ChunkId, epoch: u64) -> Result { + self.store.durable_cursor(chunk, epoch).await + } + + async fn recover_tail( + &self, + name: StreamName, + epoch: u64, + active: &ActiveChunkDescriptor, + ) -> Result { + self.store.recover_tail(name, epoch, active).await + } + + async fn renew_liveness(&self, chunk: ChunkId, epoch: u64) -> Result<()> { + self.store.renew_liveness(chunk, epoch).await + } + + async fn seal(&self, chunk: ChunkId, epoch: u64, cursor: u64) -> Result<()> { + self.store.seal(chunk, epoch, cursor).await + } + + async fn read(&self, chunk: ChunkId, offset: u64, length: usize) -> Result { + self.store.read(chunk, offset, length).await + } + + async fn release_trimmed(&self, chunk: ChunkId, end: u64, physical_end: u64) -> Result { + self.store.release_trimmed(chunk, end, physical_end).await + } +} + +#[tokio::test] +async fn rollover_grows_a_fresh_successor_before_admitting_the_waiting_append() { + let store = Arc::new(MemoryStreamStore::new(CAPACITY)); + let chunks = Arc::new(TestGrowingChunks { store: store.clone() }); + let name = StreamName { high: 31, low: 41 }; + let config = StreamConfig { + chunk_capacity_bytes: CAPACITY, + ..StreamConfig::default() + }; + let stream = ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Stream, + stream_name: name, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 1, + config.clone(), + store.clone(), + store.clone(), + chunks.clone(), + ) + .await + .unwrap(); + let payload = Bytes::from(vec![0x5a; usize::try_from(INITIAL + INITIAL / 2).unwrap()]); + let first = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + let second = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + assert_ne!(first.chunk_id, second.chunk_id); + assert_eq!(second.begin, first.end); + assert_eq!(stream.metrics().rollovers, 1); + assert_eq!( + store.metadata_publish_count(), + 2, + "only initial attachment and rollover publish chunk records" + ); + stream.checkpoint().await.unwrap(); + let reopened = ChunkStream::open_read_only(name, 1, config, store.clone(), store, chunks) + .await + .unwrap(); + assert_eq!(reopened.tail(), second.end); + for range in [first, second] { + assert_eq!( + reopened.read_at(range.begin, payload.len()).await.unwrap(), + payload + ); + } +} + +#[tokio::test] +async fn stale_stream_capacity_does_not_reject_a_newer_durable_chunk_layout() { + let store = Arc::new(MemoryStreamStore::new(CAPACITY)); + let chunks = Arc::new(TestGrowingChunks { store: store.clone() }); + let name = StreamName { high: 73, low: 79 }; + let config = StreamConfig { + chunk_capacity_bytes: CAPACITY, + ..StreamConfig::default() + }; + StreamRegistry::create( + store.as_ref(), + StreamRecord { + purpose: StreamPurpose::Stream, + stream_name: name, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + ) + .await + .unwrap(); + let active = chunks + .allocate_mirrored(name, 1, StreamPurpose::Stream) + .await + .unwrap(); + let record = crowdb_protocol::chunk_stream::StreamChunkRecord { + stream_id: name, + purpose: StreamPurpose::Stream, + chunk_id: active.chunk_id, + state: crowdb_protocol::chunk_stream::StreamChunkState::Active, + logical_start: Some(0), + logical_end: Some(0), + sequence: 0, + writer_epoch: 1, + generation: 1, + owner_session: [1; 16], + trim_offset: 0, + closed: false, + mappings: crowdb_protocol::chunk_stream::StreamMappings::default(), + }; + let metadata = store.record_store(1).unwrap(); + let mut reservation = record.clone(); + reservation.state = crowdb_protocol::chunk_stream::StreamChunkState::Candidate; + reservation.logical_start = None; + reservation.logical_end = None; + reservation.generation = 0; + let reservation = metadata.reserve_initial(&reservation, None).await.unwrap(); + metadata + .publish(None, record.clone(), None, Some(&reservation)) + .await + .unwrap(); + let payload = Bytes::from(vec![0x5a; usize::try_from(INITIAL + INITIAL / 2).unwrap()]); + let frames = crowdb_protocol::frame::encode_append_group_views(active.chunk_id, 0, &payload, 1).unwrap(); + let data = Bytes::from(frames.concat()); + let end = data.len() as u64; + assert!(end > INITIAL); + chunks + .grow_mirrored(name, 1, active.chunk_id, end) + .await + .unwrap() + .unwrap(); + chunks + .store + .write_groups_for_tests(name, 1, active.chunk_id, 0, data.clone()) + .await + .unwrap(); + chunks + .advance_cursor(name, 1, active.chunk_id, 0, end, crc32fast::hash(&data)) + .await + .unwrap(); + assert_eq!( + store.load_tail_for_tests(name).await.unwrap().unwrap().record(), + &record + ); + let reader = ChunkStream::open_read_only( + name, + 1, + config.clone(), + store.clone(), + store.clone(), + chunks.clone(), + ) + .await + .unwrap(); + assert_eq!( + reader.tail(), + 0, + "the captured record view remains the finite read boundary" + ); + let reopened = ChunkStream::open(name, 2, config, store.clone(), store.clone(), chunks) + .await + .unwrap(); + assert_eq!(reopened.tail(), payload.len() as u64); + assert_eq!(reopened.read_at(0, payload.len()).await.unwrap(), payload); + assert_eq!(reader.tail(), 0); +} diff --git a/lib/crowdb-chunk-stream/tests/stream_test.rs b/lib/crowdb-chunk-stream/tests/stream_test.rs index fb45eeedb..7d3c09fd0 100644 --- a/lib/crowdb-chunk-stream/tests/stream_test.rs +++ b/lib/crowdb-chunk-stream/tests/stream_test.rs @@ -7,18 +7,20 @@ use std::time::Duration; use bytes::Bytes; use crowdb_chunk_stream::memory::MemoryStreamStore; use crowdb_chunk_stream::{ - ChunkStream, CursorAdvance, ReadHint, StreamBinding, StreamBindingState, StreamChunkStore, StreamConfig, - StreamError, StreamMetadataStore, StreamName, StreamRegistry, + ChunkStream, CursorAdvance, ReadHint, StreamChunkStore, StreamConfig, StreamError, StreamMetadataStore, + StreamName, StreamRecord, StreamRecordState, StreamRegistry, }; -fn binding(name: StreamName) -> StreamBinding { - StreamBinding { +fn stream_record(name: StreamName) -> StreamRecord { + StreamRecord { purpose: crowdb_protocol::chunk_stream::StreamPurpose::Stream, stream_name: name, metadata_group_id: 7, - binding_generation: 1, - state: StreamBindingState::Active, - owner_kind: Some("test".into()), + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), } } @@ -27,7 +29,7 @@ async fn create_stream(store: &Arc, capacity: u64, config: St let metadata: Arc = store.clone(); let chunks: Arc = store.clone(); ChunkStream::create( - binding(StreamName { + stream_record(StreamName { high: 1, low: capacity, }), @@ -42,7 +44,7 @@ async fn create_stream(store: &Arc, capacity: u64, config: St } #[tokio::test] -async fn first_append_has_no_timer_and_publishes_framed_extent() { +async fn first_append_has_no_timer_and_publishes_only_chunk_attachment() { let store = Arc::new(MemoryStreamStore::new(64)); let stream = create_stream(&store, 64, StreamConfig::default()).await; @@ -56,8 +58,8 @@ async fn first_append_has_no_timer_and_publishes_framed_extent() { assert_eq!((range.begin, range.end), (0, 4)); assert_eq!(store.chunk_write_count(), 1); - assert_eq!(store.cursor_advance_count(), 1); - assert_eq!(store.metadata_publish_count(), 3); + assert_eq!(store.cursor_advance_count(), 0); + assert_eq!(store.metadata_publish_count(), 1); assert_eq!(stream.read_at(0, 4).await.unwrap(), Bytes::from_static(b"abcd")); } @@ -66,11 +68,15 @@ async fn idle_active_chunk_renews_liveness_without_advancing_cursor() { let store = Arc::new(MemoryStreamStore::new(64)); let stream = create_stream(&store, 64, StreamConfig::default()).await; stream.append(&[Bytes::from_static(b"idle")]).await.unwrap(); - assert_eq!(store.cursor_advance_count(), 1); + assert_eq!(store.cursor_advance_count(), 0); tokio::time::advance(Duration::from_secs(12 * 60)).await; tokio::task::yield_now().await; - assert_eq!(store.liveness_renew_count(), 1); - assert_eq!(store.cursor_advance_count(), 1); + assert_eq!( + store.liveness_renew_count(), + 2, + "active and candidate chunks renew independently" + ); + assert_eq!(store.cursor_advance_count(), 0); } #[tokio::test(start_paused = true)] @@ -84,11 +90,15 @@ async fn idle_liveness_uses_the_configured_interval() { stream.append(&[Bytes::from_static(b"idle")]).await.unwrap(); tokio::time::advance(Duration::from_secs(10)).await; tokio::task::yield_now().await; - assert_eq!(store.liveness_renew_count(), 1); + assert_eq!( + store.liveness_renew_count(), + 2, + "active and candidate chunks use the configured interval" + ); } #[tokio::test(start_paused = true)] -async fn idle_liveness_failures_rotate_without_fencing_the_stream() { +async fn idle_liveness_failures_reconfirm_without_fencing_the_stream() { let store = Arc::new(MemoryStreamStore::new(64)); let stream = create_stream(&store, 64, StreamConfig::default()).await; stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); @@ -104,10 +114,11 @@ async fn idle_liveness_failures_rotate_without_fencing_the_stream() { } #[tokio::test] -async fn registered_binding_initializes_metadata_without_recreating_registry_record() { +async fn registered_stream_initializes_metadata_without_recreating_identity_record() { let store = Arc::new(MemoryStreamStore::new(64)); let name = StreamName { high: 2, low: 1 }; - store.create(binding(name)).await.unwrap(); + let original = stream_record(name); + store.create(original.clone()).await.unwrap(); let registry: Arc = store.clone(); let metadata: Arc = store.clone(); let chunks: Arc = store.clone(); @@ -117,7 +128,7 @@ async fn registered_binding_initializes_metadata_without_recreating_registry_rec .unwrap(); assert_eq!(stream.tail(), 0); assert_eq!(store.metadata_publish_count(), 1); - assert_eq!(store.load(name).await.unwrap(), Some(binding(name))); + assert_eq!(store.load(name).await.unwrap(), Some(original)); } #[tokio::test] @@ -256,29 +267,28 @@ async fn queued_requests_aggregate_after_an_inflight_batch() { vec![(1, 2), (2, 3), (3, 4)] ); assert_eq!(store.chunk_write_count(), 3); - assert_eq!(store.cursor_advance_count(), 3); + assert_eq!(store.cursor_advance_count(), 0); assert_eq!(stream.read_at(0, 4).await.unwrap(), Bytes::from_static(b"abcd")); } #[tokio::test] -async fn extent_pages_are_cached_across_independent_reads() { +async fn chunk_records_are_cached_across_independent_reads() { let store = Arc::new(MemoryStreamStore::new(2)); let stream = create_stream(&store, 2, StreamConfig::default()).await; for bytes in [b"ab", b"cd", b"ef"] { stream.append(&[Bytes::copy_from_slice(bytes)]).await.unwrap(); } - let loads = store.extent_page_load_count(); assert_eq!(stream.read_at(0, 1).await.unwrap(), Bytes::from_static(b"a")); - assert_eq!(store.extent_page_load_count(), loads + 1); + assert_eq!(stream.metrics().record_cache_misses, 1); assert_eq!(stream.read_at(1, 1).await.unwrap(), Bytes::from_static(b"b")); - assert_eq!(store.extent_page_load_count(), loads + 1); + assert_eq!(stream.metrics().record_cache_misses, 1); let metrics = stream.metrics(); - assert_eq!(metrics.extent_page_cache_misses, 1); - assert_eq!(metrics.extent_page_cache_hits, 1); + assert_eq!(metrics.record_cache_misses, 1); + assert_eq!(metrics.record_cache_hits, 1); assert_eq!(metrics.physical_read_requests, 2); - assert_eq!(metrics.metadata_publications, 6); + assert_eq!(metrics.metadata_publications, 3); } #[tokio::test] @@ -345,7 +355,7 @@ async fn cross_chunk_read_runs_with_bounded_concurrency_and_ordered_output() { } #[tokio::test] -async fn ambiguous_cursor_is_resolved_without_resubmission() { +async fn ambiguous_checkpoint_is_resolved_without_resubmitting_append() { let committed_store = Arc::new(MemoryStreamStore::new(32)); committed_store .queue_cursor_outcome(CursorAdvance::Ambiguous, true) @@ -355,6 +365,8 @@ async fn ambiguous_cursor_is_resolved_without_resubmission() { committed.append(&[Bytes::from_static(b"yes")]).await.unwrap().end, 3 ); + assert_eq!(committed_store.cursor_advance_count(), 0); + committed.checkpoint().await.unwrap(); assert_eq!(committed_store.chunk_write_count(), 1); let absent_store = Arc::new(MemoryStreamStore::new(31)); @@ -372,7 +384,9 @@ async fn ambiguous_cursor_is_resolved_without_resubmission() { .begin, 2 ); - assert_eq!(absent_store.chunk_write_count(), 3); + assert_eq!(absent_store.cursor_advance_count(), 0); + absent.checkpoint().await.unwrap(); + assert_eq!(absent_store.chunk_write_count(), 2); } #[tokio::test] @@ -433,7 +447,7 @@ async fn trim_hides_prefix_before_reclaiming_complete_chunks() { let first = stream.append(&[Bytes::from_static(b"abcd")]).await.unwrap(); stream.append(&[Bytes::from_static(b"ef")]).await.unwrap(); - assert_eq!(stream.trim_prefix(4).await.unwrap(), 38); + assert_eq!(stream.trim_prefix(4).await.unwrap(), 51); assert!(store.is_released(first.chunk_id.unwrap()).await); assert!(matches!( stream.read_at(0, 1).await, @@ -444,10 +458,9 @@ async fn trim_hides_prefix_before_reclaiming_complete_chunks() { } #[tokio::test] -async fn near_tail_seek_loads_only_the_target_extent_page() { +async fn near_tail_seek_loads_only_the_target_chunk_record() { let store = Arc::new(MemoryStreamStore::new(1)); let config = StreamConfig { - extent_page_entries: 2, ..StreamConfig::default() }; let stream = create_stream(&store, 1, config).await; @@ -455,16 +468,15 @@ async fn near_tail_seek_loads_only_the_target_extent_page() { stream.append(&[Bytes::copy_from_slice(&[byte])]).await.unwrap(); } - let before = store.extent_page_load_count(); + let before = stream.metrics().record_cache_misses; assert_eq!(stream.read_at(3, 1).await.unwrap(), Bytes::from_static(b"d")); - assert_eq!(store.extent_page_load_count() - before, 1); + assert_eq!(stream.metrics().record_cache_misses - before, 1); } #[tokio::test] -async fn trim_keeps_nonzero_extent_page_identity() { +async fn trim_keeps_nonzero_chunk_record_identity() { let store = Arc::new(MemoryStreamStore::new(1)); let config = StreamConfig { - extent_page_entries: 2, ..StreamConfig::default() }; let stream = create_stream(&store, 1, config).await; @@ -472,34 +484,30 @@ async fn trim_keeps_nonzero_extent_page_identity() { stream.append(&[Bytes::copy_from_slice(&[byte])]).await.unwrap(); } stream.trim_prefix(2).await.unwrap(); - let manifest = store - .load_current(StreamName { high: 1, low: 1 }) - .await + let record = store + .record_store(7) .unwrap() + .locate(StreamName { high: 1, low: 1 }, 2) + .await .unwrap(); - assert_eq!(manifest.extent_pages[0].page_index, 2); + assert_eq!(record.record().logical_start, Some(2)); assert_eq!(stream.read_at(2, 3).await.unwrap(), Bytes::from_static(b"cde")); } #[tokio::test] -async fn metadata_reclaim_is_watermark_driven_bounded_and_keeps_current_pages() { +async fn metadata_reclaim_is_watermark_driven_bounded_and_keeps_retained_records() { let store = Arc::new(MemoryStreamStore::new(1)); - let config = StreamConfig { - extent_page_entries: 2, - ..StreamConfig::default() - }; - let stream = create_stream(&store, 1, config).await; + let stream = create_stream(&store, 1, StreamConfig::default()).await; for byte in *b"abcde" { stream.append(&[Bytes::copy_from_slice(&[byte])]).await.unwrap(); } - let name = StreamName { high: 1, low: 1 }; - let current = store.load_current(name).await.unwrap().unwrap().generation; - let before = store.extent_page_count().await; - - assert_eq!(stream.reclaim_metadata_before(current, 1).await.unwrap(), 1); - assert_eq!(store.extent_page_count().await, before - 1); - while stream.reclaim_metadata_before(current, 64).await.unwrap() != 0 {} - + let current = stream.record_generation(); + let records = store.chunk_records_for_tests(stream.stream_name()).await.unwrap(); + assert_eq!(stream.reclaim_metadata_before(current, 1).await.unwrap(), 0); + assert_eq!( + store.chunk_records_for_tests(stream.stream_name()).await.unwrap(), + records + ); assert_eq!(stream.read_at(0, 5).await.unwrap(), Bytes::from_static(b"abcde")); assert!(matches!( stream.reclaim_metadata_before(current + 1, 1).await, @@ -543,6 +551,7 @@ async fn read_only_open_observes_tail_without_changing_writer_authority() { let writer = create_stream(&store, 32, StreamConfig::default()).await; let name = StreamName { high: 1, low: 32 }; writer.append(&[Bytes::from_static(b"old")]).await.unwrap(); + writer.checkpoint().await.unwrap(); let publications = store.metadata_publish_count(); let reader = ChunkStream::open_read_only( @@ -575,7 +584,16 @@ async fn append_rotates_an_active_chunk_sealed_by_its_writer_lease() { let stream = create_stream(&store, 32, StreamConfig::default()).await; let first = stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); let active_chunk = first.chunk_id.unwrap(); - store.seal(active_chunk, 9, 37).await.unwrap(); + let active = store + .active_checkpoint_for_tests(stream.stream_name()) + .await + .unwrap(); + let physical_end = store + .recover_tail(stream.stream_name(), 9, &active) + .await + .unwrap() + .physical_end; + store.seal(active_chunk, 9, physical_end).await.unwrap(); let range = stream.append(&[Bytes::from_static(b"new")]).await.unwrap(); @@ -638,7 +656,7 @@ async fn higher_epoch_reopens_same_bytes_and_fences_old_writer() { } #[tokio::test] -async fn rollover_retries_failed_manifest_publication_without_reopen() { +async fn rollover_retries_failed_record_publication_without_reopen() { let store = Arc::new(MemoryStreamStore::new(4)); let stream = create_stream(&store, 4, StreamConfig::default()).await; let name = StreamName { high: 1, low: 4 }; @@ -666,7 +684,7 @@ async fn rollover_retries_failed_manifest_publication_without_reopen() { } #[tokio::test] -async fn reopen_repartitions_directory_and_reads_before_another_append() { +async fn reopen_recovers_tail_record_and_reads_before_another_append() { let store = Arc::new(MemoryStreamStore::new(32)); let stream = create_stream(&store, 32, StreamConfig::default()).await; for bytes in [b"one", b"two", b"end"] { @@ -678,7 +696,6 @@ async fn reopen_repartitions_directory_and_reads_before_another_append() { StreamName { high: 1, low: 32 }, 9, StreamConfig { - extent_page_entries: 1, ..StreamConfig::default() }, store.clone(), @@ -692,5 +709,6 @@ async fn reopen_repartitions_directory_and_reads_before_another_append() { Bytes::from_static(b"onetwoend") ); let observed = reopened.observe_metadata(None, 0).unwrap(); - assert_eq!(observed.extent_pages.len(), 3); + assert_eq!(observed.chunk.logical_end, 9); + assert!(observed.chunk.mapping_count <= 3); } diff --git a/lib/crowdb-chunk-stream/tests/successor_preparation_test.rs b/lib/crowdb-chunk-stream/tests/successor_preparation_test.rs new file mode 100644 index 000000000..03f0aedc6 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/successor_preparation_test.rs @@ -0,0 +1,305 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::{sync::Arc, time::Duration}; + +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamConfig, StreamName, StreamPurpose, StreamRecord, + StreamRecordState, +}; +use tokio::time::timeout; + +async fn stream(store: Arc, interval: Duration) -> ChunkStream { + ChunkStream::create( + StreamRecord { + state: StreamRecordState::Active, + ..StreamRecord::creating(StreamName::generate(), None, StreamPurpose::Stream) + }, + 1, + StreamConfig { + liveness_interval: interval, + ..StreamConfig::default() + }, + store.clone(), + store.clone(), + store, + ) + .await + .unwrap() +} + +#[tokio::test] +async fn pending_successor_only_backpressures_writes_that_require_rotation() { + let store = Arc::new(MemoryStreamStore::new(128)); + store.pause_allocation_for_tests(2); + let stream = stream(store.clone(), Duration::from_secs(60)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + timeout(Duration::from_secs(1), store.wait_for_allocation_for_tests()) + .await + .unwrap(); + timeout( + Duration::from_secs(1), + stream.append(&[Bytes::from_static(b"new")]), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(stream.tail(), 6); + let writer = stream.clone(); + let mut rollover = tokio::spawn(async move { writer.append(&[Bytes::from(vec![0x61; 100])]).await }); + assert!(timeout(Duration::from_millis(20), &mut rollover).await.is_err()); + assert_eq!(stream.tail(), 6); + store.resume_allocation_for_tests(); + let range = timeout(Duration::from_secs(1), rollover) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(range.chunk_id.unwrap().low, 2); + assert_eq!(range.begin, 6); + assert_eq!(stream.read_at(0, 6).await.unwrap(), Bytes::from_static(b"oldnew")); + stream.close().await.unwrap(); +} + +#[tokio::test] +async fn rotation_uses_the_prepared_candidate_while_its_replacement_is_blocked() { + let store = Arc::new(MemoryStreamStore::new(128)); + store.pause_allocation_for_tests(3); + let stream = stream(store.clone(), Duration::from_secs(60)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + timeout(Duration::from_secs(1), async { + loop { + if store.chunks_for_tests().await.len() == 2 { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!(store.chunks_for_tests().await.len(), 2); + let range = timeout( + Duration::from_secs(1), + stream.append(&[Bytes::from(vec![0x61; 100])]), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(range.chunk_id.unwrap().low, 2); + timeout(Duration::from_secs(1), store.wait_for_allocation_for_tests()) + .await + .unwrap(); + assert_eq!(stream.metrics().rollovers, 1); + store.resume_allocation_for_tests(); + stream.close().await.unwrap(); + let chunks = store.chunks_for_tests().await; + assert_eq!(chunks.len(), 3); + assert_eq!(chunks[2].1, 0); + assert!( + chunks[2].2 && chunks[2].3, + "unused replacement must be sealed and released" + ); + assert!(!chunks[1].3, "the attached chunk remains retained"); +} + +#[tokio::test] +async fn close_drains_candidate_initialization_and_reclaims_it_without_publication() { + let store = Arc::new(MemoryStreamStore::new(128)); + store.pause_allocation_for_tests(2); + let stream = stream(store.clone(), Duration::from_secs(60)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + timeout(Duration::from_secs(1), store.wait_for_allocation_for_tests()) + .await + .unwrap(); + assert_eq!( + store.metadata_publish_count(), + 1, + "preparing a candidate must not publish a new tail checkpoint" + ); + let mut closing = tokio::spawn(async move { stream.close().await }); + assert!(timeout(Duration::from_millis(20), &mut closing).await.is_err()); + store.resume_allocation_for_tests(); + timeout(Duration::from_secs(1), closing) + .await + .unwrap() + .unwrap() + .unwrap(); + let chunks = store.chunks_for_tests().await; + assert_eq!(chunks.len(), 2); + assert!(chunks[1].2 && chunks[1].3); + assert_eq!(chunks[1].1, 0); + assert_eq!(store.metadata_publish_count(), 2); +} + +#[tokio::test(start_paused = true)] +async fn idle_candidate_is_renewed_without_publishing_or_rotating_the_stream() { + let store = Arc::new(MemoryStreamStore::new(128)); + let stream = stream(store.clone(), Duration::from_secs(10)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + tokio::task::yield_now().await; + let publications = store.metadata_publish_count(); + for _ in 0..3 { + tokio::time::advance(Duration::from_secs(10)).await; + tokio::task::yield_now().await; + } + assert!( + store.liveness_renew_count() >= 6, + "both attached and prepared chunks require renewal" + ); + assert_eq!(store.metadata_publish_count(), publications); + assert_eq!(stream.metrics().rollovers, 0); + stream.close().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn failed_candidate_renewal_replaces_only_the_unused_chunk() { + let store = Arc::new(MemoryStreamStore::new(128)); + let stream = stream(store.clone(), Duration::from_secs(10)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + tokio::task::yield_now().await; + let candidate = store.chunks_for_tests().await[1].0; + store.fail_chunk_renewal_for_tests(candidate); + let publications = store.metadata_publish_count(); + tokio::time::advance(Duration::from_secs(10)).await; + timeout(Duration::from_secs(1), async { + while store.chunks_for_tests().await.len() < 3 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let chunks = store.chunks_for_tests().await; + assert!(chunks[1].2 && chunks[1].3, "failed candidate must be reclaimed"); + assert_eq!(chunks[2].1, 0); + assert!(!chunks[2].2 && !chunks[2].3); + assert_eq!(stream.metrics().rollovers, 0); + assert_eq!(store.metadata_publish_count(), publications); + stream.append(&[Bytes::from_static(b"new")]).await.unwrap(); + assert_eq!(stream.read_at(0, 6).await.unwrap(), Bytes::from_static(b"oldnew")); + stream.close().await.unwrap(); +} + +#[tokio::test] +async fn explicit_owner_release_reclaims_the_candidate_and_does_not_replenish_it() { + let store = Arc::new(MemoryStreamStore::new(128)); + let stream = stream(store.clone(), Duration::from_secs(10)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + stream.release_writer().await.unwrap(); + tokio::task::yield_now().await; + let chunks = store.chunks_for_tests().await; + assert_eq!(chunks.len(), 2); + assert!(chunks[1].2 && chunks[1].3); + assert!(!chunks[0].3); + assert!(stream.append(&[Bytes::from_static(b"late")]).await.is_err()); +} + +#[tokio::test] +async fn candidate_allocation_outage_backpressures_instead_of_poisoning_the_writer() { + let store = Arc::new(MemoryStreamStore::new(128)); + store.fail_allocation_for_tests(2); + let stream = stream(store.clone(), Duration::from_secs(10)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + timeout(Duration::from_secs(1), store.wait_for_allocation_for_tests()) + .await + .unwrap(); + let writer = stream.clone(); + let mut pending = tokio::spawn(async move { writer.append(&[Bytes::from(vec![0x61; 100])]).await }); + assert!(timeout(Duration::from_millis(20), &mut pending).await.is_err()); + assert_eq!(stream.tail(), 3); + let range = timeout(Duration::from_secs(1), pending) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(range.chunk_id.unwrap().low, 3); + assert_eq!(range.begin, 3); + assert_eq!(stream.metrics().rollovers, 1); + assert_eq!(stream.metrics().failed, 0); + stream.close().await.unwrap(); +} + +#[tokio::test(start_paused = true)] +async fn active_liveness_outage_backpressures_without_rotating_a_healthy_tail() { + let store = Arc::new(MemoryStreamStore::new(128)); + let stream = stream(store.clone(), Duration::from_secs(10)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + tokio::task::yield_now().await; + store.fail_next_renewals(100); + tokio::time::advance(Duration::from_secs(10)).await; + for _ in 0..4 { + tokio::time::advance(Duration::from_millis(100)).await; + tokio::task::yield_now().await; + } + let writer = stream.clone(); + let mut pending = tokio::spawn(async move { writer.append(&[Bytes::from_static(b"new")]).await }); + assert!( + timeout(Duration::from_secs(1), &mut pending).await.is_err(), + "unconfirmed chunk liveness must apply backpressure without sealing the tail" + ); + assert_eq!(stream.tail(), 3); + store.fail_next_renewals(0); + timeout(Duration::from_secs(1), pending) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(stream.metrics().rollovers, 0); + assert_eq!(stream.read_at(0, 6).await.unwrap(), Bytes::from_static(b"oldnew")); + stream.close().await.unwrap(); +} + +#[tokio::test] +async fn unused_candidate_cleanup_failure_does_not_prevent_durable_owner_release() { + let store = Arc::new(MemoryStreamStore::new(128)); + let stream = stream(store.clone(), Duration::from_secs(60)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + let name = stream.stream_name(); + store.fail_empty_release_for_tests(); + stream.release_writer().await.unwrap(); + let chunks = store.chunks_for_tests().await; + assert!(!chunks[0].3, "attached data remains retained"); + assert!( + chunks[1].2 && !chunks[1].3, + "failed candidate cleanup remains retained for expiry: {chunks:?}" + ); + let next = ChunkStream::open( + name, + 2, + StreamConfig::default(), + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + next.append(&[Bytes::from_static(b"new")]).await.unwrap(); + assert_eq!(next.read_at(0, 6).await.unwrap(), Bytes::from_static(b"oldnew")); + next.close().await.unwrap(); +} + +#[tokio::test] +async fn unexpected_candidate_data_blocks_owner_release_instead_of_becoming_cleanup_backlog() { + let store = Arc::new(MemoryStreamStore::new(128)); + let stream = stream(store.clone(), Duration::from_secs(60)).await; + stream.append(&[Bytes::from_static(b"old")]).await.unwrap(); + timeout(Duration::from_secs(1), async { + while store.chunks_for_tests().await.len() < 2 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let chunks = store.chunks_for_tests().await; + let encoded = store.synchronized_bytes_for_tests(chunks[0].0).await; + store + .write_groups_for_tests(stream.stream_name(), 1, chunks[1].0, 0, encoded) + .await + .unwrap(); + let released = stream.release_writer().await; + assert!( + matches!(released, Err(crowdb_chunk_stream::StreamError::Corruption(_))), + "a candidate with data violates the empty-candidate contract: {released:?}" + ); + assert_eq!(stream.read_at(0, 3).await.unwrap(), Bytes::from_static(b"old")); +} diff --git a/lib/crowdb-chunk-stream/tests/trim_chunk_test.rs b/lib/crowdb-chunk-stream/tests/trim_chunk_test.rs new file mode 100644 index 000000000..67d15acb9 --- /dev/null +++ b/lib/crowdb-chunk-stream/tests/trim_chunk_test.rs @@ -0,0 +1,129 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_chunk_stream::{ + memory::MemoryStreamStore, ChunkStream, StreamChunkStore, StreamConfig, StreamName, StreamPurpose, + StreamRecord, StreamRecordState, +}; +use std::sync::Arc; + +async fn create(config: StreamConfig) -> (Arc, ChunkStream) { + let store = Arc::new(MemoryStreamStore::new(150_000)); + let stream = ChunkStream::create( + StreamRecord { + purpose: StreamPurpose::Stream, + stream_name: StreamName { high: 97, low: 101 }, + metadata_group_id: 1, + record_version: 1, + state: StreamRecordState::Active, + consumer_kind: Some("test".into()), + consumer_id: None, + created_at_ms: StreamRecord::creation_time_ms(), + }, + 9, + config, + store.clone(), + store.clone(), + store.clone(), + ) + .await + .unwrap(); + (store, stream) +} + +#[tokio::test] +async fn trimming_one_frame_cannot_release_a_sealed_chunk_with_a_retained_suffix() { + let (store, stream) = create(StreamConfig::default()).await; + let payload = Bytes::from(vec![0x5a; 100_000]); + let first = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + let second = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + assert_ne!(first.chunk_id, second.chunk_id); + assert_eq!(stream.trim_prefix(70_000).await.unwrap(), 0); + assert!(!store.is_released(first.chunk_id.unwrap()).await); + assert_eq!( + stream.read_at(70_000, 30_000).await.unwrap(), + payload.slice(70_000..) + ); + let checkpoint = store + .durable_cursor(first.chunk_id.unwrap(), 9) + .await + .unwrap() + .offset; + assert_eq!(stream.trim_prefix(first.end).await.unwrap(), checkpoint); + assert!(store.is_released(first.chunk_id.unwrap()).await); + assert_eq!( + stream.read_at(second.begin, payload.len()).await.unwrap(), + payload + ); +} + +#[tokio::test] +async fn trimming_a_whole_active_chunk_keeps_it_available_for_later_append() { + let (store, stream) = create(StreamConfig::default()).await; + let payload = Bytes::from(vec![0x5a; 50_000]); + let first = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + let second = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + assert_eq!(first.chunk_id, second.chunk_id); + assert_eq!(stream.trim_prefix(first.end).await.unwrap(), 0); + assert_eq!( + stream.read_at(second.begin, payload.len()).await.unwrap(), + payload + ); + assert_eq!(stream.trim_prefix(second.end).await.unwrap(), 0); + assert!(!store.is_released(first.chunk_id.unwrap()).await); + let tail = stream.append(&[Bytes::from_static(b"tail")]).await.unwrap(); + assert_eq!(tail.chunk_id, first.chunk_id); + assert_eq!( + stream.read_at(tail.begin, 4).await.unwrap(), + Bytes::from_static(b"tail") + ); + let physical = store + .durable_cursor(first.chunk_id.unwrap(), 9) + .await + .unwrap() + .offset; + let next = stream.append(&[Bytes::from(vec![0x33; 100_000])]).await.unwrap(); + assert_ne!(next.chunk_id, first.chunk_id); + let sealed_physical = store + .durable_cursor(first.chunk_id.unwrap(), 9) + .await + .unwrap() + .offset; + assert!( + sealed_physical > physical, + "rollover seals the uncheckpointed final append too" + ); + assert_eq!(stream.trim_prefix(second.end).await.unwrap(), 0); + assert_eq!( + stream.read_at(tail.begin, 4).await.unwrap(), + Bytes::from_static(b"tail") + ); + assert_eq!(stream.trim_prefix(tail.end).await.unwrap(), sealed_physical); + assert!(store.is_released(first.chunk_id.unwrap()).await); +} + +#[tokio::test] +async fn a_gc_budget_smaller_than_one_chunk_still_reclaims_only_complete_chunks() { + let (store, stream) = create(StreamConfig { + gc_bytes_per_pass: 1, + ..StreamConfig::default() + }) + .await; + let payload = Bytes::from(vec![0x5a; 100_000]); + let first = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + let second = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + let third = stream.append(std::slice::from_ref(&payload)).await.unwrap(); + let physical = store + .durable_cursor(first.chunk_id.unwrap(), 9) + .await + .unwrap() + .offset; + assert_eq!(stream.trim_prefix(second.end).await.unwrap(), physical); + assert!(store.is_released(first.chunk_id.unwrap()).await); + assert!(!store.is_released(second.chunk_id.unwrap()).await); + assert_eq!(stream.trim_prefix(second.end).await.unwrap(), physical); + assert!(store.is_released(second.chunk_id.unwrap()).await); + assert_eq!(stream.trim_prefix(second.end).await.unwrap(), 0); + assert_eq!(stream.read_at(third.begin, payload.len()).await.unwrap(), payload); +} diff --git a/lib/crowdb-chunkdb-client/src/client.rs b/lib/crowdb-chunkdb-client/src/client.rs index 3e950fea3..49ad7bd7f 100644 --- a/lib/crowdb-chunkdb-client/src/client.rs +++ b/lib/crowdb-chunkdb-client/src/client.rs @@ -29,7 +29,7 @@ use crowdb_protocol::chunkdb::rpc::{ RelocateSegmentHandoffResponse, ReplaceChunkStripRangeRequest, ReplaceChunkStripRangeResponse, ReserveStripGroupRequest, ReserveStripGroupResponse, SealChunkRequest, SealChunkResponse, TriggerConversionBatchRequest, TriggerConversionBatchResponse, TriggerConversionRequest, - TriggerConversionResponse, UpdateChunkStripRequest, UpdateChunkStripResponse, + TriggerConversionResponse, }; use crowdb_protocol::common::ChunkId; use crowdb_protocol::InstanceId; @@ -171,11 +171,22 @@ impl ChunkdbClient { /// Refresh `ChunkDB` service endpoints and range ownership bindings. pub async fn refresh_routes(&self) -> Result<()> { - self.refresh_endpoints().await?; self.range_binding .refresh() .await .map_err(|error| ChunkdbClientError::Unreachable(format!("slot refresh failed: {error}")))?; + // The validated binding snapshot already contains the current registry + // endpoints. Do not read that same registry a second time per refresh. + let endpoints = self + .range_binding + .snapshot() + .into_iter() + .filter(|binding| !binding.rpc_endpoint.is_empty()) + .map(|binding| (binding.instance_id, binding.rpc_endpoint)) + .collect(); + self.endpoint_cache.store(Arc::new(endpoints)); + self.last_registry_refresh_ms + .store(unix_time_ms(), Ordering::Release); Ok(()) } @@ -338,16 +349,6 @@ impl ChunkdbClient { .await } - /// Update a single strip within a chunk. - pub async fn update_chunk_strip(&self, req: UpdateChunkStripRequest) -> Result { - let chunk_id = req.chunk_id; - self.with_rpc_retry(chunk_id.as_ref(), |t, ep| { - let req = req.clone(); - async move { t.send_update_chunk_strip(&ep, &req).await } - }) - .await - } - pub async fn allocate_replacement_segment( &self, req: AllocateReplacementSegmentRequest, diff --git a/lib/crowdb-chunkdb-client/src/lib.rs b/lib/crowdb-chunkdb-client/src/lib.rs index 711e5b2b9..4072dad89 100644 --- a/lib/crowdb-chunkdb-client/src/lib.rs +++ b/lib/crowdb-chunkdb-client/src/lib.rs @@ -12,6 +12,7 @@ #![allow(clippy::must_use_candidate, clippy::missing_errors_doc)] pub mod client; +mod readiness; pub mod rpc_transport; pub use client::{ChunkdbClient, NativeChunkRoutes, RetryConfig}; diff --git a/lib/crowdb-chunkdb-client/src/readiness.rs b/lib/crowdb-chunkdb-client/src/readiness.rs new file mode 100644 index 000000000..d5d9fa913 --- /dev/null +++ b/lib/crowdb-chunkdb-client/src/readiness.rs @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Read-only point observations whose work does not grow with catalog size. + +use crowdb_protocol::{ + chunk_id::{CHUNK_TYPE_S3, CHUNK_TYPE_WAL}, + chunkdb::rpc::QueryChunkRequest, + common::ChunkId, +}; + +use crate::{ChunkdbClient, ChunkdbClientError, Result}; + +impl ChunkdbClient { + /// Read one key through each maintenance domain's current owner and storage route. + /// An authoritative not-found response is valid evidence for an empty catalog. + /// This neither creates probe objects nor enumerates unrelated chunks. + /// + /// # Errors + /// Returns routing, ownership, storage, transport and malformed response errors. + pub async fn observe_readiness(&self) -> Result<()> { + tokio::try_join!( + self.observe_domain(CHUNK_TYPE_WAL), + self.observe_domain(CHUNK_TYPE_S3), + )?; + Ok(()) + } + + async fn observe_domain(&self, purpose: u8) -> Result<()> { + let id = ChunkId { + high: u64::from(purpose) << 56, + low: 0, + }; + match self.query_chunk(QueryChunkRequest { chunk_id: Some(id) }).await { + Ok(reply) if reply.chunk.as_ref().is_some_and(|chunk| chunk.id == Some(id)) => Ok(()), + Err(ChunkdbClientError::NotFound(_)) => Ok(()), + Ok(_) => Err(ChunkdbClientError::Rpc( + "readiness query returned a missing or different chunk identity".into(), + )), + Err(error) => Err(error), + } + } +} diff --git a/lib/crowdb-chunkdb-client/src/rpc_transport.rs b/lib/crowdb-chunkdb-client/src/rpc_transport.rs index 291d4c865..abf1109d0 100644 --- a/lib/crowdb-chunkdb-client/src/rpc_transport.rs +++ b/lib/crowdb-chunkdb-client/src/rpc_transport.rs @@ -1,18 +1,10 @@ // Copyright 2026-present Gian . #![allow(clippy::missing_errors_doc)] -#![allow(dead_code)] // Wired in Phase 5 (ChunkdbClient transport selection) - -//! crowdb-rpc client transport for `ChunkdbService` (R116 migration). -//! Mirrors `kv_rpc_transport.rs` (R117): builds flatbuffer requests, -//! sends via `RpcClient::call`, awaits `CallFuture`, parses flatbuffer -//! responses via the zero-copy `Ref` wrappers, and maps them to the -//! existing tonic proto response types so the retry/routing logic in -//! `client.rs` is unchanged. -//! -//! Runs alongside the tonic channel pool during the mixed-rollout -//! window; `ChunkdbClient` selects the transport via -//! `with_rpc_transport`. + +//! Current `ChunkDB` control transport over crowdb-rpc. +//! Builds typed `FlatBuffer` requests, awaits correlated replies and validates +//! response records before returning them to the routing/retry client. use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -28,7 +20,7 @@ use crowdb_protocol::chunkdb::rpc::{ QuerySegmentOwnerResponse, RelocateSegmentHandoffResponse, ReplaceChunkStripRangeResponse, ReserveStripGroupResponse, SealChunkResponse, StripCleanupIntent, StripReservationAction, StripReservationGroup, StripReservationState, StripType as ProtoStripType, - TriggerConversionBatchResponse, TriggerConversionResponse, UpdateChunkStripResponse, + TriggerConversionBatchResponse, TriggerConversionResponse, }; use crowdb_protocol::chunkdb::rpc::{EcState as ProtoEcState, EcStrip, MirrorStrip, Strip as ProtoStrip}; use crowdb_protocol::chunkdb_fb::{ @@ -50,8 +42,7 @@ use crowdb_protocol::chunkdb_fb::{ FBSealChunkRequest, FBSealChunkRequestArgs, FBSegment, FBStripBody, FBStripReservationAction, FBStripReservationGroup, FBStripReservationState, FBStripType, FBTriggerConversionBatchRequest, FBTriggerConversionBatchRequestArgs, FBTriggerConversionBatchResponse, FBTriggerConversionRequest, - FBTriggerConversionRequestArgs, FBTriggerConversionResponse, FBUpdateChunkStripRequest, - FBUpdateChunkStripRequestArgs, + FBTriggerConversionRequestArgs, FBTriggerConversionResponse, }; use crowdb_protocol::common::{ChunkId, DiskId}; use crowdb_protocol::fb::FBMsgType; @@ -241,8 +232,7 @@ impl ChunkdbRpcTransport { // ── AllocateChunk ───────────────────────────────────────────── - /// Send an `AllocateChunk` request via crowdb-rpc. Returns the tonic - /// `AllocateChunkResponse` so the caller's retry logic is unchanged. + /// Send an allocation request and decode its typed response. pub async fn send_allocate_chunk( &self, rpc_endpoint: &str, @@ -616,7 +606,6 @@ impl ChunkdbRpcTransport { id: req_id, rpc_create_nano: 0, chunk_id: chunk_id_off.as_ref(), - seal_length: req.seal_length, seal_bytes: req.seal_bytes, }; let fb_req = FBSealChunkRequest::create(&mut builder, &args); @@ -700,46 +689,6 @@ impl ChunkdbRpcTransport { Ok(DeleteChunkRangeResponse {}) } - // ── UpdateChunkStrip ────────────────────────────────────────── - - /// Send an `UpdateChunkStrip` request via crowdb-rpc. - pub async fn send_update_chunk_strip( - &self, - rpc_endpoint: &str, - req: &crowdb_protocol::chunkdb::rpc::UpdateChunkStripRequest, - ) -> Result { - let req_id = self.next_id(); - let conn = self.conn_for(rpc_endpoint)?; - let mut builder = FlatBufferBuilder::new(); - let chunk_id_off = req.chunk_id.as_ref().map(|id| FBInt128::new(id.high, id.low)); - let strip_off = req - .strip - .as_ref() - .map(|s| build_chunk_strip_offset(&mut builder, s)); - let args = FBUpdateChunkStripRequestArgs { - id: req_id, - rpc_create_nano: 0, - chunk_id: chunk_id_off.as_ref(), - strip_index: req.strip_index, - strip: strip_off, - }; - let fb_req = FBUpdateChunkStripRequest::create(&mut builder, &args); - builder.finish(fb_req, None); - let control = Buffer::from_bytes(builder.finished_data()); - let msg_type = FBMsgType::EUpdateChunkStripRequest.0 as u16; - let resp = call_rpc(self, &conn, req_id, control, msg_type, rpc_endpoint).await?; - let r = FBAllocateChunkResponseRef::new(resp.bytes()); - if !r.valid() { - return Err(ChunkdbClientError::Rpc( - "update_chunk_strip response malformed".into(), - )); - } - check_ret_code(r.ret_code(), r.error_msg())?; - Ok(UpdateChunkStripResponse { - chunk: r.chunk().map(|fb_chunk| parse_fb_chunk(&fb_chunk)).transpose()?, - }) - } - pub async fn send_allocate_replacement_segment( &self, rpc_endpoint: &str, @@ -1393,7 +1342,7 @@ fn build_fb_segment(segment: &crowdb_protocol::diskdb::rpc::Segment) -> FBSegmen // ── Request building: ChunkStrip offset ─────────────────────────── /// Build a `FBChunkStrip` `WIPOffset` from a proto `ChunkStrip` for -/// request construction (`UpdateChunkStrip`). +/// fenced range replacement requests. fn build_chunk_strip_offset<'a>( fbb: &mut FlatBufferBuilder<'a>, strip: &ChunkStrip, @@ -1580,14 +1529,6 @@ fn fb_ec_state_to_proto(s: crowdb_protocol::chunkdb_fb::FBEcState) -> ProtoEcSta // ── Endpoint helpers ────────────────────────────────────────────── -fn rpc_error_to_client(e: RpcError) -> ChunkdbClientError { - if e.is_retryable() { - ChunkdbClientError::Unavailable(format!("rpc transient: {e:?}")) - } else { - ChunkdbClientError::Rpc(format!("rpc error: {e:?}")) - } -} - fn normalize_endpoint(endpoint: &str) -> String { let with_scheme = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { endpoint.to_string() diff --git a/lib/crowdb-common/cpp/include/crowdb-common/diskio_uring.h b/lib/crowdb-common/cpp/include/crowdb-common/diskio_uring.h index 3b506c46c..198c69804 100644 --- a/lib/crowdb-common/cpp/include/crowdb-common/diskio_uring.h +++ b/lib/crowdb-common/cpp/include/crowdb-common/diskio_uring.h @@ -188,6 +188,9 @@ class DiskIOUring std::function cb; int fd{-1}; // for in_flight decrement on dispatch CallbackEntry *next_free{nullptr}; // poll-thread-only: deferred-delete list + uint64_t queued_nanos{0}; + uint64_t published_nanos{0}; + uint8_t opcode{0}; }; // fd_table entry: pipeline index + in-flight counter. diff --git a/lib/crowdb-common/cpp/include/crowdb-common/metrics/system_metrics.h b/lib/crowdb-common/cpp/include/crowdb-common/metrics/system_metrics.h index d4aaf78cd..5672c6410 100644 --- a/lib/crowdb-common/cpp/include/crowdb-common/metrics/system_metrics.h +++ b/lib/crowdb-common/cpp/include/crowdb-common/metrics/system_metrics.h @@ -13,11 +13,26 @@ #include #include +#include #include +#include namespace crowdb::common::metrics { +namespace detail +{ +class C6Counters; +} + +// A host hardware domain, identified by its PMU's representative logical CPU. +// Missing pct means the interval could not be read; zero means measured zero. +struct C6Residency +{ + int cpu = 0; + std::optional pct; +}; + // Snapshot of system-level metrics at a single point in time. struct SystemMetricsSnapshot { @@ -40,6 +55,10 @@ struct SystemMetricsSnapshot // Aggregate DRAM bandwidth. Available when an AMD PMU cannot split // reads and writes. std::optional dram_total_mib; + // Hardware C6 percentages per physical core/package. Empty if unsupported + // or access is denied. Host-wide, independent of this process's CPU%. + std::vector cpu_core_c6; + std::vector cpu_package_c6; }; // Collects OS-level metrics by reading /proc (Linux) or using ps (macOS). @@ -70,7 +89,8 @@ class SystemCollector // Opaque DRAM bandwidth counter (Linux perf_event_open fds). struct DramBwImpl; - DramBwImpl *dram_bw_ = nullptr; + DramBwImpl *dram_bw_ = nullptr; + std::unique_ptr c6_; }; // Write a system snapshot to the writer in the "misc" section format, diff --git a/lib/crowdb-common/cpp/src/diskio_uring.cpp b/lib/crowdb-common/cpp/src/diskio_uring.cpp index cf54065e9..24fbe0c28 100644 --- a/lib/crowdb-common/cpp/src/diskio_uring.cpp +++ b/lib/crowdb-common/cpp/src/diskio_uring.cpp @@ -12,7 +12,9 @@ #include #include +#include #include +#include #include #include @@ -23,6 +25,43 @@ namespace { std::atomic uring_in_flight{0}; +struct UringLatencyMetrics +{ + metrics::LatencyHistogram *queue; + metrics::LatencyHistogram *completion; +}; + +const UringLatencyMetrics *uring_latency_metrics(uint8_t opcode) +{ + static const auto read = UringLatencyMetrics{ + .queue = metrics::MetricsRegistry::global().register_histogram("diskio.uring.read.queue.lh"), + .completion = metrics::MetricsRegistry::global().register_histogram("diskio.uring.read.completion.lh")}; + static const auto write = UringLatencyMetrics{ + .queue = metrics::MetricsRegistry::global().register_histogram("diskio.uring.write.queue.lh"), + .completion = metrics::MetricsRegistry::global().register_histogram("diskio.uring.write.completion.lh")}; + static const auto fsync = UringLatencyMetrics{ + .queue = metrics::MetricsRegistry::global().register_histogram("diskio.uring.fsync.queue.lh"), + .completion = metrics::MetricsRegistry::global().register_histogram("diskio.uring.fsync.completion.lh")}; + switch (opcode) { + case IORING_OP_READ: + return &read; + case IORING_OP_WRITE: + case IORING_OP_WRITEV: + return &write; + case IORING_OP_FSYNC: + return &fsync; + default: + return nullptr; + } +} + +uint64_t steady_nanos() +{ + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + crowdb::common::metrics::Counter &uring_sq_full_metric() { static auto *counter = @@ -36,6 +75,7 @@ void register_uring_metrics() "diskio.uring.inflight.g", [] { return uring_in_flight.load(std::memory_order_relaxed); }); (void)in_flight; (void)uring_sq_full_metric(); + (void)uring_latency_metrics(IORING_OP_WRITE); } } // namespace @@ -430,7 +470,8 @@ void DiskIOUring::submit_lockfree(Pipeline &p, int fd, std::function total_in_flight_.fetch_add(1, std::memory_order_relaxed); uring_in_flight.fetch_add(1, std::memory_order_relaxed); - auto *entry = new CallbackEntry{.cb = std::move(on_complete), .fd = fd, .next_free = {}}; + auto *entry = + new CallbackEntry{.cb = std::move(on_complete), .fd = fd, .next_free = {}, .queued_nanos = steady_nanos()}; // CAS loop: check capacity BEFORE claiming a slot. bool observed_full = false; @@ -455,6 +496,7 @@ void DiskIOUring::submit_lockfree(Pipeline &p, int fd, std::function struct io_uring_sqe *sqe = &p.ring.sq.sqes[idx << p.sq_shift]; io_uring_initialize_sqe(sqe); prep(sqe); + entry->opcode = sqe->opcode; io_uring_sqe_set_data(sqe, entry); p.sqe_ready[idx].store(true, std::memory_order_release); mark_pending(p); @@ -482,6 +524,9 @@ void DiskIOUring::publish_ready_sqes(Pipeline &p) if (!p.sqe_ready[idx].load(std::memory_order_acquire)) { break; } + auto *entry = + std::bit_cast(static_cast(p.ring.sq.sqes[idx << p.sq_shift].user_data)); + entry->published_nanos = steady_nanos(); p.sqe_ready[idx].store(false, std::memory_order_relaxed); if (p.ring.sq.array != nullptr) { p.ring.sq.array[idx] = idx; @@ -623,6 +668,12 @@ bool DiskIOUring::drain_cqes(Pipeline &p) ::io_uring_cqe_seen(&p.ring, cqe); if (entry != nullptr) { + if (const auto *latency = uring_latency_metrics(entry->opcode); latency != nullptr) { + latency->queue->observe(entry->published_nanos - entry->queued_nanos); + // Includes submission, kernel execution and CQ dispatch; it + // is not a measurement of physical-device service time. + latency->completion->observe(steady_nanos() - entry->published_nanos); + } if (entry->fd >= 0 && entry->fd < fd_table_size_) { fd_in_flight_[entry->fd].fetch_sub(1, std::memory_order_acq_rel); } diff --git a/lib/crowdb-common/cpp/src/metrics/cstate_metrics.cpp b/lib/crowdb-common/cpp/src/metrics/cstate_metrics.cpp new file mode 100644 index 000000000..88e67a3a2 --- /dev/null +++ b/lib/crowdb-common/cpp/src/metrics/cstate_metrics.cpp @@ -0,0 +1,161 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "cstate_metrics.h" + +#include +#include +#include +#include +#include +#include + +namespace crowdb::common::metrics::detail +{ + +static std::optional parse_cpu(std::string_view text) +{ + int cpu = 0; + auto result = std::from_chars(text.data(), text.data() + text.size(), cpu); + if (result.ec != std::errc{} || result.ptr != text.data() + text.size() || cpu < 0) { + return std::nullopt; + } + return cpu; +} + +std::optional> parse_cpu_list(std::string_view text) +{ + auto last = text.find_last_not_of(" \t\r\n"); + text = last == std::string_view::npos ? std::string_view{} : text.substr(0, last + 1); + std::set cpus; + while (!text.empty()) { + auto comma = text.find(','); + auto part = text.substr(0, comma); + auto dash = part.find('-'); + auto first = parse_cpu(part.substr(0, dash)); + auto end = dash == std::string_view::npos ? first : parse_cpu(part.substr(dash + 1)); + if (!first || !end || *end < *first) { + return std::nullopt; + } + for (int cpu = *first;; ++cpu) { + cpus.insert(cpu); + if (cpu == *end) { + break; + } + } + if (comma == std::string_view::npos) { + return std::vector(cpus.begin(), cpus.end()); + } + text.remove_prefix(comma + 1); + } + return std::nullopt; +} + +#ifdef __linux__ +std::optional C6Window::sample(std::optional c6, std::optional tsc) +{ + auto baseline = std::exchange(previous, std::nullopt); + if (!c6 || !tsc) { + return std::nullopt; + } + previous = std::pair{*c6, *tsc}; + if (!baseline) { + return std::nullopt; + } + auto c6_delta = scaled_delta(*c6, baseline->first); + auto tsc_delta = scaled_delta(*tsc, baseline->second); + if (!c6_delta || !tsc_delta || *tsc_delta <= 0 || *c6_delta > *tsc_delta * 1.01) { + return std::nullopt; + } + // Separately read PMUs can have a small boundary skew near 100%. + return std::min(100.0, 100.0 * *c6_delta / *tsc_delta); +} + +C6Residency C6Pair::collect() +{ + auto c6_sample = c6.read(); + auto tsc_sample = tsc.read(); + return {.cpu = cpu, .pct = window.sample(c6_sample, tsc_sample)}; +} + +static std::optional read_pmu_file(std::string_view pmu, std::string_view file) +{ + std::ifstream input("/sys/bus/event_source/devices/" + std::string(pmu) + "/" + std::string(file)); + std::string text; + if (!std::getline(input, text)) { + return std::nullopt; + } + return text; +} + +static std::optional read_pmu_number(std::string_view pmu, std::string_view file, bool event) +{ + auto text = read_pmu_file(pmu, file); + if (!text) { + return std::nullopt; + } + std::string_view number(*text); + if (event) { + if (!number.starts_with("event=0x")) { + return std::nullopt; + } + number.remove_prefix(8); + } + uint64_t value = 0; + auto result = std::from_chars(number.data(), number.data() + number.size(), value, event ? 16 : 10); + if (result.ec != std::errc{} || result.ptr != number.data() + number.size()) { + return std::nullopt; + } + return value; +} + +std::vector C6Counters::open_scope(std::string_view pmu) +{ + auto type = read_pmu_number(pmu, "type", false); + auto config = read_pmu_number(pmu, "events/c6-residency", true); + auto tsc_type = read_pmu_number("msr", "type", false); + auto tsc_config = read_pmu_number("msr", "events/tsc", true); + auto mask = read_pmu_file(pmu, "cpumask"); + auto cpus = mask ? parse_cpu_list(*mask) : std::nullopt; + if (!type || !config || !tsc_type || !tsc_config || !cpus) { + return {}; + } + std::vector pairs; + for (int cpu : *cpus) { + int c6_fd = PerfCounter::open_fd(static_cast(*type), *config, cpu); + if (c6_fd < 0) { + return {}; + } + PerfCounter c6(c6_fd); + int tsc_fd = PerfCounter::open_fd(static_cast(*tsc_type), *tsc_config, cpu); + if (tsc_fd < 0) { + return {}; + } + C6Pair pair{.cpu = cpu, .c6 = std::move(c6), .tsc = PerfCounter(tsc_fd), .window = {}}; + pair.collect(); // Establish both baselines before the first flush. + pairs.push_back(std::move(pair)); + } + return pairs; +} + +C6Counters::C6Counters() : cores_(open_scope("cstate_core")), packages_(open_scope("cstate_pkg")) +{ +} +#else +C6Counters::C6Counters() = default; +#endif + +std::pair, std::vector> C6Counters::collect() +{ + std::vector cores; + std::vector packages; +#ifdef __linux__ + std::transform(cores_.begin(), cores_.end(), std::back_inserter(cores), + [](C6Pair &pair) { return pair.collect(); }); + std::transform(packages_.begin(), packages_.end(), std::back_inserter(packages), + [](C6Pair &pair) { return pair.collect(); }); +#endif + return {std::move(cores), std::move(packages)}; +} + +} // namespace crowdb::common::metrics::detail diff --git a/lib/crowdb-common/cpp/src/metrics/cstate_metrics.h b/lib/crowdb-common/cpp/src/metrics/cstate_metrics.h new file mode 100644 index 000000000..b184c6a09 --- /dev/null +++ b/lib/crowdb-common/cpp/src/metrics/cstate_metrics.h @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#pragma once + +#include "crowdb-common/metrics/system_metrics.h" +#include "perf_counter.h" + +#include +#include +#include +#include + +namespace crowdb::common::metrics::detail +{ + +std::optional> parse_cpu_list(std::string_view text); + +#ifdef __linux__ +struct C6Window +{ + std::optional> previous; + std::optional sample(std::optional c6, std::optional tsc); +}; + +struct C6Pair +{ + int cpu; + PerfCounter c6; + PerfCounter tsc; + C6Window window; + C6Residency collect(); +}; +#endif + +class C6Counters +{ + public: + C6Counters(); + std::pair, std::vector> collect(); + + private: +#ifdef __linux__ + static std::vector open_scope(std::string_view pmu); + std::vector cores_; + std::vector packages_; +#endif +}; + +} // namespace crowdb::common::metrics::detail diff --git a/lib/crowdb-common/cpp/src/metrics/perf_counter.h b/lib/crowdb-common/cpp/src/metrics/perf_counter.h new file mode 100644 index 000000000..8f101e0bd --- /dev/null +++ b/lib/crowdb-common/cpp/src/metrics/perf_counter.h @@ -0,0 +1,108 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#pragma once + +#ifdef __linux__ +# include +# include +# include +# include + +# include +# include +# include + +namespace crowdb::common::metrics::detail +{ + +struct PerfSample +{ + uint64_t value = 0; + uint64_t time_enabled = 0; + uint64_t time_running = 0; +}; + +class PerfCounter +{ + public: + explicit PerfCounter(int fd) : fd_(fd) + { + } + + ~PerfCounter() + { + if (fd_ >= 0) { + close(fd_); + } + } + + PerfCounter(const PerfCounter &) = delete; + PerfCounter &operator=(const PerfCounter &) = delete; + + PerfCounter(PerfCounter &&other) noexcept : fd_(std::exchange(other.fd_, -1)) + { + } + + PerfCounter &operator=(PerfCounter &&other) noexcept + { + if (this != &other) { + if (fd_ >= 0) { + close(fd_); + } + fd_ = std::exchange(other.fd_, -1); + } + return *this; + } + + // Shared by DRAM and C-state collectors. Enabled only after open succeeds. + static int open_fd(uint32_t type, uint64_t config, int cpu) + { + perf_event_attr attr{}; + attr.type = type; + attr.size = sizeof(attr); + attr.config = config; + attr.disabled = 1; + attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING; + int fd = static_cast(syscall(SYS_perf_event_open, &attr, -1, cpu, -1, PERF_FLAG_FD_CLOEXEC)); + if (fd >= 0 && ioctl(fd, PERF_EVENT_IOC_ENABLE, 0) != 0) { + close(fd); + return -1; + } + return fd; + } + + [[nodiscard]] static std::optional read_fd(int fd) + { + PerfSample sample; + if (::read(fd, &sample, sizeof(sample)) != sizeof(sample)) { + return std::nullopt; + } + return sample; + } + + [[nodiscard]] std::optional read() const + { + return read_fd(fd_); + } + + private: + int fd_; +}; + +inline std::optional scaled_delta(const PerfSample ¤t, const PerfSample &previous) +{ + if (current.value < previous.value || current.time_enabled < previous.time_enabled || + current.time_running < previous.time_running) { + return std::nullopt; + } + uint64_t enabled = current.time_enabled - previous.time_enabled; + uint64_t running = current.time_running - previous.time_running; + if (running == 0 || enabled < running) { + return std::nullopt; + } + return static_cast(current.value - previous.value) * static_cast(enabled) / + static_cast(running); +} + +} // namespace crowdb::common::metrics::detail +#endif diff --git a/lib/crowdb-common/cpp/src/metrics/system_metrics.cpp b/lib/crowdb-common/cpp/src/metrics/system_metrics.cpp index fedd8c5ae..4c36ffb10 100644 --- a/lib/crowdb-common/cpp/src/metrics/system_metrics.cpp +++ b/lib/crowdb-common/cpp/src/metrics/system_metrics.cpp @@ -2,6 +2,8 @@ #include "crowdb-common/metrics/system_metrics.h" +#include "cstate_metrics.h" + #include #include #include @@ -124,48 +126,11 @@ static std::pair read_tcp_stats() // ── DRAM bandwidth via perf_event_open ────────────────────────── -struct perf_event_attr_min -{ - uint32_t type; - uint32_t size; - uint64_t config; - uint64_t sample_period_or_freq; - uint64_t sample_type; - uint64_t read_format; - uint64_t flags; - uint32_t wakeup_events_or_watermark; - uint32_t bp_type; - uint64_t bp_addr_or_config1; - uint64_t bp_len_or_config2; - uint64_t branch_sample_type; - uint64_t sample_regs_user; - uint32_t sample_stack_user; - int32_t clockid; - uint64_t sample_regs_intr; - uint32_t aux_watermark; - uint16_t sample_max_stack; - uint16_t reserved2; - uint32_t aux_sample_size; - uint32_t reserved3; - uint64_t sig_data; -}; - -static constexpr uint64_t PERF_FLAG_DISABLED = 1ULL; -static constexpr uint64_t PERF_IOC_ENABLE = 0x2400ULL; -static constexpr uint64_t PERF_FLAG_FD_CLOEXEC = 8ULL; -static constexpr uint64_t PERF_FORMAT_TOTAL_TIME_ENABLED = 1ULL; -static constexpr uint64_t PERF_FORMAT_TOTAL_TIME_RUNNING = 2ULL; - struct SystemCollector::DramBwImpl { struct Fd { - struct Sample - { - uint64_t value = 0; - uint64_t time_enabled = 0; - uint64_t time_running = 0; - }; + using Sample = detail::PerfSample; int fd = -1; Sample prev; @@ -206,27 +171,12 @@ struct SystemCollector::DramBwImpl static int perf_open(uint32_t pmu_type, uint64_t config, int cpu) { - perf_event_attr_min attr{}; - attr.type = pmu_type; - attr.size = sizeof(attr); - attr.config = config; - attr.flags = PERF_FLAG_DISABLED; - attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING; - return static_cast(syscall(SYS_perf_event_open, &attr, -1, cpu, -1, PERF_FLAG_FD_CLOEXEC)); - } - - static void perf_enable(int fd) - { - ioctl(fd, PERF_IOC_ENABLE, 0); + return detail::PerfCounter::open_fd(pmu_type, config, cpu); } static std::optional read_fd(int fd) { - Fd::Sample sample; - if (read(fd, &sample, sizeof(sample)) != sizeof(sample)) { - return std::nullopt; - } - return sample; + return detail::PerfCounter::read_fd(fd); } static std::optional read_pmu_type(const std::string &name) @@ -327,7 +277,6 @@ struct SystemCollector::DramBwImpl close_fds(totals); return false; } - perf_enable(fd); totals.push_back({.fd = fd, .prev = {}}); } total_fds = std::move(totals); @@ -369,8 +318,6 @@ struct SystemCollector::DramBwImpl close_fds(writes); return false; } - perf_enable(rfd); - perf_enable(wfd); reads.push_back({.fd = rfd, .prev = {}}); writes.push_back({.fd = wfd, .prev = {}}); } @@ -502,7 +449,7 @@ static std::pair read_tcp_stats() // ── SystemCollector ───────────────────────────────────────────── -SystemCollector::SystemCollector() +SystemCollector::SystemCollector() : c6_(std::make_unique()) { #ifdef __linux__ auto [user_us, sys_us] = read_cpu_times(); @@ -533,7 +480,8 @@ SystemCollector::SystemCollector(SystemCollector &&other) noexcept prev_tcp_retransmits_(other.prev_tcp_retransmits_), prev_tcp_lost_(other.prev_tcp_lost_), prev_time_us_(other.prev_time_us_), - dram_bw_(std::exchange(other.dram_bw_, nullptr)) + dram_bw_(std::exchange(other.dram_bw_, nullptr)), + c6_(std::move(other.c6_)) { } @@ -547,6 +495,7 @@ SystemCollector &SystemCollector::operator=(SystemCollector &&other) noexcept prev_tcp_lost_ = other.prev_tcp_lost_; prev_time_us_ = other.prev_time_us_; dram_bw_ = std::exchange(other.dram_bw_, nullptr); + c6_ = std::move(other.c6_); } return *this; } @@ -598,9 +547,30 @@ SystemMetricsSnapshot SystemCollector::collect() } #endif + if (c6_) { + auto [cores, packages] = c6_->collect(); + snap.cpu_core_c6 = std::move(cores); + snap.cpu_package_c6 = std::move(packages); + } return snap; } +static void flush_c6(FILE *fp, const char *scope, const std::vector &values) +{ + if (values.empty()) { + std::fprintf(fp, " cpu.c6.%s=unsupported", scope); + } + for (const auto &value : values) { + std::fprintf(fp, " cpu.c6.%s@%d=", scope, value.cpu); + if (value.pct) { + std::fprintf(fp, "%.1f%%", *value.pct); + } + else { + std::fprintf(fp, "unsupported"); + } + } +} + void flush_system(FILE *fp, const SystemMetricsSnapshot &snap) { double rss_gb = static_cast(snap.rss_kb) / 1024.0 / 1024.0; @@ -616,6 +586,8 @@ void flush_system(FILE *fp, const SystemMetricsSnapshot &snap) print_bw(snap.dram_write_mib); std::fprintf(fp, " bw_total_mib="); print_bw(snap.dram_total_mib); + flush_c6(fp, "core", snap.cpu_core_c6); + flush_c6(fp, "package", snap.cpu_package_c6); std::fprintf(fp, "\n"); } diff --git a/lib/crowdb-common/cpp/tests/cstate_metrics_test.cpp b/lib/crowdb-common/cpp/tests/cstate_metrics_test.cpp new file mode 100644 index 000000000..5e99ff7aa --- /dev/null +++ b/lib/crowdb-common/cpp/tests/cstate_metrics_test.cpp @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "../src/metrics/cstate_metrics.h" + +#include + +using crowdb::common::metrics::detail::parse_cpu_list; + +TEST(C6MetricsTest, PmuRepresentativesAreUnique) +{ + EXPECT_EQ(parse_cpu_list("0-3,2,8,10-11\n"), (std::vector{0, 1, 2, 3, 8, 10, 11})); + for (const char *invalid : {"", "-1", "3-1", "0-", "a", "0,,2", "0,"}) { + EXPECT_FALSE(parse_cpu_list(invalid)) << invalid; + } +} + +#ifdef __linux__ +using crowdb::common::metrics::detail::C6Window; +using crowdb::common::metrics::detail::PerfSample; + +static PerfSample sample(uint64_t value, uint64_t enabled, uint64_t running) +{ + return {.value = value, .time_enabled = enabled, .time_running = running}; +} + +TEST(C6MetricsTest, ZeroIsDistinctFromMissingBaseline) +{ + C6Window window; + EXPECT_FALSE(window.sample(sample(100, 100, 100), sample(1000, 100, 100))); + EXPECT_EQ(window.sample(sample(100, 200, 200), sample(2000, 200, 200)), 0.0); + EXPECT_EQ(window.sample(sample(600, 300, 300), sample(3000, 300, 300)), 50.0); +} + +TEST(C6MetricsTest, UsesTscAndCorrectsMultiplexing) +{ + C6Window window; + window.sample(PerfSample{}, PerfSample{}); + EXPECT_EQ(window.sample(sample(250, 100, 50), sample(1000, 100, 100)), 50.0); +} + +TEST(C6MetricsTest, FailedReadRequiresFreshBaselines) +{ + C6Window window; + window.sample(PerfSample{}, PerfSample{}); + EXPECT_FALSE(window.sample(std::nullopt, sample(1000, 100, 100))); + EXPECT_FALSE(window.sample(sample(1000, 200, 200), sample(2000, 200, 200))); + EXPECT_EQ(window.sample(sample(1250, 300, 300), sample(3000, 300, 300)), 25.0); +} + +TEST(C6MetricsTest, CounterResetRecovers) +{ + C6Window window; + window.sample(sample(1000, 100, 100), sample(2000, 100, 100)); + EXPECT_FALSE(window.sample(PerfSample{}, PerfSample{})); + EXPECT_EQ(window.sample(sample(750, 100, 100), sample(1000, 100, 100)), 75.0); +} + +TEST(C6MetricsTest, InvalidIntervalsAreUnavailable) +{ + for (const auto &[c6, tsc] : { + std::pair{sample(1, 100, 0), sample(1000, 100, 100)}, + std::pair{sample(1, 50, 100), sample(1000, 100, 100)}, + std::pair{sample(1, 100, 100), sample(0, 100, 100)}, + std::pair{sample(2000, 100, 100), sample(1000, 100, 100)} + }) { + C6Window window; + window.sample(PerfSample{}, PerfSample{}); + EXPECT_FALSE(window.sample(c6, tsc)); + } +} +#endif diff --git a/lib/crowdb-common/cpp/tests/system_metrics_test.cpp b/lib/crowdb-common/cpp/tests/system_metrics_test.cpp index f3c54222d..7cb6b88e4 100644 --- a/lib/crowdb-common/cpp/tests/system_metrics_test.cpp +++ b/lib/crowdb-common/cpp/tests/system_metrics_test.cpp @@ -6,6 +6,7 @@ #include #include +#include #include using crowdb::common::metrics::flush_system; @@ -41,16 +42,22 @@ TEST(SystemMetricsTest, FlushWritesAllFields) snap.dram_read_mib = 512.5; snap.dram_write_mib = 128.3; snap.dram_total_mib = 640.8; + snap.cpu_core_c6 = { + {.cpu = 0, .pct = 83.25}, + {.cpu = 2, .pct = 0.0 } + }; + snap.cpu_package_c6 = { + {.cpu = 0, .pct = std::nullopt} + }; // Write to a temp file via FILE*. - FILE *fp = std::tmpfile(); + std::unique_ptr fp(std::tmpfile(), &std::fclose); ASSERT_NE(fp, nullptr); - flush_system(fp, snap); - std::fseek(fp, 0, SEEK_SET); + flush_system(fp.get(), snap); + std::fseek(fp.get(), 0, SEEK_SET); std::array buf{}; - size_t n = std::fread(buf.data(), 1, buf.size() - 1, fp); + size_t n = std::fread(buf.data(), 1, buf.size() - 1, fp.get()); buf[n] = '\0'; - std::fclose(fp); std::string out(buf.data()); EXPECT_TRUE(out.find("cpu.user=42%") != std::string::npos); @@ -61,6 +68,7 @@ TEST(SystemMetricsTest, FlushWritesAllFields) EXPECT_TRUE(out.find("bw_read_mib=512.5") != std::string::npos); EXPECT_TRUE(out.find("bw_write_mib=128.3") != std::string::npos); EXPECT_TRUE(out.find("bw_total_mib=640.8") != std::string::npos); + EXPECT_TRUE(out.find("cpu.c6.core@0=83.2% cpu.c6.core@2=0.0% cpu.c6.package@0=unsupported") != std::string::npos); } TEST(SystemMetricsTest, FlushWritesUnsupportedWhenNone) @@ -70,17 +78,17 @@ TEST(SystemMetricsTest, FlushWritesUnsupportedWhenNone) snap.dram_write_mib = std::nullopt; snap.dram_total_mib = std::nullopt; - FILE *fp = std::tmpfile(); + std::unique_ptr fp(std::tmpfile(), &std::fclose); ASSERT_NE(fp, nullptr); - flush_system(fp, snap); - std::fseek(fp, 0, SEEK_SET); + flush_system(fp.get(), snap); + std::fseek(fp.get(), 0, SEEK_SET); std::array buf{}; - size_t n = std::fread(buf.data(), 1, buf.size() - 1, fp); + size_t n = std::fread(buf.data(), 1, buf.size() - 1, fp.get()); buf[n] = '\0'; - std::fclose(fp); std::string out(buf.data()); EXPECT_TRUE(out.find("bw_read_mib=unsupported") != std::string::npos); EXPECT_TRUE(out.find("bw_write_mib=unsupported") != std::string::npos); EXPECT_TRUE(out.find("bw_total_mib=unsupported") != std::string::npos); + EXPECT_TRUE(out.find("cpu.c6.core=unsupported cpu.c6.package=unsupported") != std::string::npos); } diff --git a/lib/crowdb-common/rust/Cargo.toml b/lib/crowdb-common/rust/Cargo.toml index 3d54aad45..86672ed8b 100644 --- a/lib/crowdb-common/rust/Cargo.toml +++ b/lib/crowdb-common/rust/Cargo.toml @@ -11,6 +11,9 @@ autobenches = false [lints] workspace = true +[features] +test-util = [] + [dependencies] tokio = { workspace = true, features = ["rt", "macros", "time", "sync"] } tracing = { workspace = true } @@ -28,4 +31,5 @@ xxhash-rust = { version = "0.8", features = ["xxh64"] } libc = "0.2" [dev-dependencies] +crowdb-common = { path = ".", features = ["test-util"] } crowdb-test-harness = { path = "../../crowdb-test-harness" } diff --git a/lib/crowdb-common/rust/src/metrics/perf.rs b/lib/crowdb-common/rust/src/metrics/perf.rs index 2b9e48191..c4eb93052 100644 --- a/lib/crowdb-common/rust/src/metrics/perf.rs +++ b/lib/crowdb-common/rust/src/metrics/perf.rs @@ -28,6 +28,11 @@ clippy::items_after_statements )] +mod cstate; +pub use cstate::C6Counters; +#[cfg(feature = "test-util")] +pub use cstate::{parse_c6_cpu_list_for_tests, TestC6Window}; + #[cfg(target_os = "linux")] mod imp { use std::time::Instant; @@ -74,14 +79,14 @@ mod imp { #[repr(C)] #[derive(Clone, Copy, Default)] - struct PerfRead { - value: u64, - time_enabled: u64, - time_running: u64, + pub(super) struct PerfRead { + pub(super) value: u64, + pub(super) time_enabled: u64, + pub(super) time_running: u64, } /// A single opened perf counter fd + its previous reading. - struct PerfFd { + pub(super) struct PerfFd { fd: i32, prev: PerfRead, } @@ -92,7 +97,7 @@ mod imp { /// `pmu_type` is the type from /sys/bus/event_source/devices//type. /// `cpu` is the CPU from the PMU's cpumask (uncore PMUs require /// a specific CPU, not -1). - fn open(pmu_type: u32, config: u64, cpu: i32) -> Option { + pub(super) fn open(pmu_type: u32, config: u64, cpu: i32) -> Option { let attr = PerfEventAttr { type_: pmu_type, size: PERF_ATTR_SIZE, @@ -120,16 +125,16 @@ mod imp { // Enable the counter. const PERF_EVENT_IOC_ENABLE: u64 = 0x2400; - let _ = unsafe { libc::ioctl(fd, PERF_EVENT_IOC_ENABLE as libc::c_ulong, 0u64) }; - - Some(Self { + let counter = Self { fd, prev: PerfRead::default(), - }) + }; + let enabled = unsafe { libc::ioctl(fd, PERF_EVENT_IOC_ENABLE as libc::c_ulong, 0u64) }; + (enabled == 0).then_some(counter) } /// Read the current cumulative counter value. - fn read(&self) -> Option { + pub(super) fn read(&self) -> Option { let mut sample = PerfRead::default(); let size = std::mem::size_of::(); let read = unsafe { @@ -164,7 +169,7 @@ mod imp { } /// Read a PMU type from /sys/bus/event_source/devices//type. - fn read_pmu_type(name: &str) -> Option { + pub(super) fn read_pmu_type(name: &str) -> Option { let path = format!("/sys/bus/event_source/devices/{name}/type"); std::fs::read_to_string(path) .ok() @@ -188,7 +193,7 @@ mod imp { /// Returns None if the file doesn't exist (some PMUs like amd_df /// don't expose events/ in sysfs — their encodings come from perf's /// JSON metric database and must be hardcoded by the caller). - fn read_pmu_event_config(pmu: &str, event: &str) -> Option { + pub(super) fn read_pmu_event_config(pmu: &str, event: &str) -> Option { let path = format!("/sys/bus/event_source/devices/{pmu}/events/{event}"); let content = std::fs::read_to_string(path).ok()?; let mut event_val: u64 = 0; diff --git a/lib/crowdb-common/rust/src/metrics/perf/cstate.rs b/lib/crowdb-common/rust/src/metrics/perf/cstate.rs new file mode 100644 index 000000000..dd48a6e59 --- /dev/null +++ b/lib/crowdb-common/rust/src/metrics/perf/cstate.rs @@ -0,0 +1,135 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Hardware C6 residency, normalized by the invariant TSC rather than CPU frequency. + +use std::collections::BTreeSet; + +use super::imp::{read_pmu_event_config, read_pmu_type, PerfFd, PerfRead}; +use crate::metrics::system::C6Residency; + +fn parse_cpu_list(text: &str) -> Option> { + let mut cpus = BTreeSet::new(); + for part in text.trim().split(',') { + let (first, last) = part.split_once('-').unwrap_or((part, part)); + let first: i32 = first.parse().ok()?; + let last: i32 = last.parse().ok()?; + if first < 0 || last < first { + return None; + } + cpus.extend(first..=last); + } + Some(cpus.into_iter().collect()) +} + +#[derive(Default)] +struct ResidencyWindow { + previous: Option<(PerfRead, PerfRead)>, +} + +impl ResidencyWindow { + fn sample(&mut self, c6: Option, tsc: Option) -> Option { + let previous = self.previous.take(); + let current = (c6?, tsc?); + self.previous = Some(current); + let previous = previous?; + let c6 = scaled_delta(current.0, previous.0)?; + let tsc = scaled_delta(current.1, previous.1)?; + // Small boundary skew is possible because these PMUs are read separately. + // A substantial excess indicates an invalid sample, not valid residency. + (tsc > 0.0 && c6 <= tsc * 1.01).then(|| (100.0 * c6 / tsc).min(100.0)) + } +} + +fn scaled_delta(current: PerfRead, previous: PerfRead) -> Option { + let value = current.value.checked_sub(previous.value)?; + let enabled = current.time_enabled.checked_sub(previous.time_enabled)?; + let running = current.time_running.checked_sub(previous.time_running)?; + (running > 0 && enabled >= running).then(|| value as f64 * enabled as f64 / running as f64) +} + +struct ResidencyPair { + cpu: i32, + c6: PerfFd, + tsc: PerfFd, + window: ResidencyWindow, +} + +impl ResidencyPair { + fn collect(&mut self) -> C6Residency { + C6Residency { + cpu: self.cpu, + pct: self.window.sample(self.c6.read(), self.tsc.read()), + } + } +} + +/// System-wide hardware residency for each physical core and package. +/// PMU cpumasks select one representative logical CPU per hardware domain; +/// SMT siblings are therefore not counted twice. Missing permissions or PMUs +/// leave that scope empty. No software cpuidle fallback is substituted. +#[derive(Default)] +pub struct C6Counters { + cores: Vec, + packages: Vec, +} + +impl C6Counters { + #[must_use] + pub fn new() -> Self { + Self { + cores: open_scope("cstate_core").unwrap_or_default(), + packages: open_scope("cstate_pkg").unwrap_or_default(), + } + } + + /// Percentages since the previous collection, keyed by representative CPU. + pub fn collect(&mut self) -> (Vec, Vec) { + ( + self.cores.iter_mut().map(ResidencyPair::collect).collect(), + self.packages.iter_mut().map(ResidencyPair::collect).collect(), + ) + } +} + +fn open_scope(pmu: &str) -> Option> { + let pmu_type = read_pmu_type(pmu)?; + let config = read_pmu_event_config(pmu, "c6-residency")?; + let tsc_type = read_pmu_type("msr")?; + let tsc_config = read_pmu_event_config("msr", "tsc")?; + let mask = std::fs::read_to_string(format!("/sys/bus/event_source/devices/{pmu}/cpumask")).ok()?; + let mut pairs = Vec::new(); + for cpu in parse_cpu_list(&mask)? { + let mut pair = ResidencyPair { + cpu, + c6: PerfFd::open(pmu_type, config, cpu)?, + tsc: PerfFd::open(tsc_type, tsc_config, cpu)?, + window: ResidencyWindow::default(), + }; + let _ = pair.collect(); // Establish both baselines before the first flush. + pairs.push(pair); + } + Some(pairs) +} + +#[cfg(feature = "test-util")] +#[must_use] +pub fn parse_c6_cpu_list_for_tests(text: &str) -> Option> { + parse_cpu_list(text) +} + +#[cfg(feature = "test-util")] +#[derive(Default)] +pub struct TestC6Window(ResidencyWindow); + +#[cfg(feature = "test-util")] +impl TestC6Window { + pub fn sample_for_tests(&mut self, c6: Option<[u64; 3]>, tsc: Option<[u64; 3]>) -> Option { + let sample = |[value, time_enabled, time_running]: [u64; 3]| PerfRead { + value, + time_enabled, + time_running, + }; + self.0.sample(c6.map(sample), tsc.map(sample)) + } +} diff --git a/lib/crowdb-common/rust/src/metrics/registry.rs b/lib/crowdb-common/rust/src/metrics/registry.rs index 3ca7dd67f..b1d6a16e1 100644 --- a/lib/crowdb-common/rust/src/metrics/registry.rs +++ b/lib/crowdb-common/rust/src/metrics/registry.rs @@ -121,6 +121,23 @@ impl MetricsRegistry { h } + /// Include an existing histogram in this registry's periodic output. + /// + /// # Panics + /// Panics if the same name already identifies a different histogram. + pub fn register_shared_histogram(&mut self, handle: Arc) { + let name = handle.name().to_owned(); + if let Some(existing) = self.histograms.iter().find(|entry| entry.name == name) { + assert!( + Arc::ptr_eq(&existing.handle, &handle), + "histogram name collision: {name}" + ); + return; + } + self.max_name_len = self.max_name_len.max(name.len()); + self.histograms.push(HistogramEntry { handle, name }); + } + /// Register a latency summary, returning a handle for `observe(ns)`. pub fn register_summary(&mut self, name: impl Into) -> Arc { let name = name.into(); diff --git a/lib/crowdb-common/rust/src/metrics/system.rs b/lib/crowdb-common/rust/src/metrics/system.rs index 1709d45eb..0cc69f556 100644 --- a/lib/crowdb-common/rust/src/metrics/system.rs +++ b/lib/crowdb-common/rust/src/metrics/system.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian //! OS-level system metrics: CPU time, memory RSS, TCP retransmits, and -//! DRAM read/write bandwidth. +//! DRAM read/write bandwidth and hardware C6 residency. //! //! On Linux, reads `/proc/self/stat` for CPU jiffies, `/proc/self/status` //! for RSS, `/proc/net/snmp` for TCP retransmit/lost counters, and @@ -19,7 +19,15 @@ use serde::{Deserialize, Serialize}; use std::fs; #[cfg(target_os = "linux")] -use super::perf::DramBwCounter; +use super::perf::{C6Counters, DramBwCounter}; + +/// Hardware C6 residency of a core or package, identified by its PMU's +/// representative logical CPU. `None` means the interval could not be read. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct C6Residency { + pub cpu: i32, + pub pct: Option, +} /// Snapshot of system-level metrics at a single point in time. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] @@ -44,6 +52,11 @@ pub struct SystemMetrics { /// Average aggregate DRAM bandwidth. Available on AMD even when the /// platform PMU cannot distinguish reads from writes. pub dram_total_mib: Option, + /// Host physical-core C6 percentages, one entry per PMU representative CPU. + /// Empty when unsupported or access is denied; independent of process CPU%. + pub cpu_core_c6: Vec, + /// Host package C6 percentages, keyed by PMU representative CPU. + pub cpu_package_c6: Vec, } /// Collects OS-level metrics by reading `/proc` (Linux) or using @@ -60,6 +73,8 @@ pub struct SystemCollector { prev_instant: Instant, #[cfg(target_os = "linux")] dram_bw: Option, + #[cfg(target_os = "linux")] + c6: C6Counters, } impl SystemCollector { @@ -79,6 +94,8 @@ impl SystemCollector { prev_instant: Instant::now(), #[cfg(target_os = "linux")] dram_bw: DramBwCounter::new(), + #[cfg(target_os = "linux")] + c6: C6Counters::new(), } } @@ -126,6 +143,11 @@ impl SystemCollector { #[cfg(not(target_os = "linux"))] let (dram_read_mib, dram_write_mib, dram_total_mib) = (None, None, None); + #[cfg(target_os = "linux")] + let (cpu_core_c6, cpu_package_c6) = self.c6.collect(); + #[cfg(not(target_os = "linux"))] + let (cpu_core_c6, cpu_package_c6) = (Vec::new(), Vec::new()); + SystemMetrics { cpu_user_pct, cpu_sys_pct, @@ -135,6 +157,8 @@ impl SystemCollector { dram_read_mib, dram_write_mib, dram_total_mib, + cpu_core_c6, + cpu_package_c6, } } } @@ -156,11 +180,31 @@ pub fn flush_system(writer: &mut W, snap: &SystemMetrics) { let bw_total = snap .dram_total_mib .map_or_else(|| "unsupported".to_string(), |v| format!("{v:.1}")); - let _ = writeln!( + let _ = write!( writer, "sys cpu.user={}% cpu.sys={}% rss_gb={rss_gb:.2} tcp_retrans={} tcp_lost={} bw_read_mib={bw_read} bw_write_mib={bw_write} bw_total_mib={bw_total}", snap.cpu_user_pct, snap.cpu_sys_pct, snap.tcp_retransmits, snap.tcp_lost, ); + flush_c6(writer, "core", &snap.cpu_core_c6); + flush_c6(writer, "package", &snap.cpu_package_c6); + let _ = writeln!(writer); +} + +fn flush_c6(writer: &mut W, scope: &str, values: &[C6Residency]) { + if values.is_empty() { + let _ = write!(writer, " cpu.c6.{scope}=unsupported"); + } + for value in values { + let _ = write!(writer, " cpu.c6.{scope}@{}=", value.cpu); + match value.pct { + Some(pct) => { + let _ = write!(writer, "{pct:.1}%"); + } + None => { + let _ = write!(writer, "unsupported"); + } + } + } } // ── Platform-specific readers ─────────────────────────────────── @@ -267,71 +311,3 @@ fn read_tcp_stats() -> (u64, u64) { // TCP stats are not available on macOS without private APIs. (0, 0) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn collect_returns_snapshot() { - let mut collector = SystemCollector::new(); - let snap = collector.collect(); - // CPU deltas should be small but non-negative. - // RSS should be non-zero (the test process has memory). - assert!(snap.rss_kb > 0, "RSS should be non-zero, got {}", snap.rss_kb); - } - - #[test] - fn collect_delta_is_non_negative() { - let mut collector = SystemCollector::new(); - let _snap1 = collector.collect(); - let snap2 = collector.collect(); - // Utilization should be non-negative (can exceed 100% on multi-core). - assert!(snap2.cpu_user_pct <= 100 * 1024); - } - - #[test] - fn flush_system_writes_all_fields() { - let snap = SystemMetrics { - cpu_user_pct: 42, - cpu_sys_pct: 17, - rss_kb: 4096, - tcp_retransmits: 3, - tcp_lost: 1, - dram_read_mib: Some(512.5), - dram_write_mib: Some(128.3), - dram_total_mib: Some(640.8), - }; - let mut buf = Vec::new(); - flush_system(&mut buf, &snap); - let out = String::from_utf8(buf).unwrap(); - assert!(out.contains("cpu.user=42%")); - assert!(out.contains("cpu.sys=17%")); - assert!(out.contains("rss_gb=0.00")); - assert!(out.contains("tcp_retrans=3")); - assert!(out.contains("tcp_lost=1")); - assert!(out.contains("bw_read_mib=512.5")); - assert!(out.contains("bw_write_mib=128.3")); - assert!(out.contains("bw_total_mib=640.8")); - } - - #[test] - fn flush_system_writes_unsupported_when_none() { - let snap = SystemMetrics { - cpu_user_pct: 0, - cpu_sys_pct: 0, - rss_kb: 0, - tcp_retransmits: 0, - tcp_lost: 0, - dram_read_mib: None, - dram_write_mib: None, - dram_total_mib: None, - }; - let mut buf = Vec::new(); - flush_system(&mut buf, &snap); - let out = String::from_utf8(buf).unwrap(); - assert!(out.contains("bw_read_mib=unsupported")); - assert!(out.contains("bw_total_mib=unsupported")); - assert!(out.contains("bw_write_mib=unsupported")); - } -} diff --git a/lib/crowdb-common/rust/tests/cstate_metrics_test.rs b/lib/crowdb-common/rust/tests/cstate_metrics_test.rs new file mode 100644 index 000000000..82c889840 --- /dev/null +++ b/lib/crowdb-common/rust/tests/cstate_metrics_test.rs @@ -0,0 +1,84 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. +#![cfg(target_os = "linux")] + +use crowdb_common::metrics::perf::{parse_c6_cpu_list_for_tests, TestC6Window}; + +#[test] +fn pmu_representatives_are_sorted_and_unique() { + assert_eq!( + parse_c6_cpu_list_for_tests("0-3,2,8,10-11\n"), + Some(vec![0, 1, 2, 3, 8, 10, 11]) + ); + for invalid in ["", "-1", "3-1", "0-", "a", "0,,2", "0,"] { + assert_eq!(parse_c6_cpu_list_for_tests(invalid), None, "{invalid}"); + } +} + +#[test] +fn hardware_zero_is_distinct_from_missing_baseline() { + let mut window = TestC6Window::default(); + assert_eq!( + window.sample_for_tests(Some([100, 100, 100]), Some([1000, 100, 100])), + None + ); + assert_eq!( + window.sample_for_tests(Some([100, 200, 200]), Some([2000, 200, 200])), + Some(0.0) + ); + assert_eq!( + window.sample_for_tests(Some([600, 300, 300]), Some([3000, 300, 300])), + Some(50.0) + ); +} + +#[test] +fn residency_uses_tsc_and_corrects_multiplexing() { + let mut window = TestC6Window::default(); + window.sample_for_tests(Some([0, 0, 0]), Some([0, 0, 0])); + // C6 ran half the interval: 250 measured ticks scale to 500 out of 1000 TSC ticks. + assert_eq!( + window.sample_for_tests(Some([250, 100, 50]), Some([1000, 100, 100])), + Some(50.0) + ); +} + +#[test] +fn failed_read_requires_a_fresh_pair_of_baselines() { + let mut window = TestC6Window::default(); + window.sample_for_tests(Some([0, 0, 0]), Some([0, 0, 0])); + assert_eq!(window.sample_for_tests(None, Some([1000, 100, 100])), None); + assert_eq!( + window.sample_for_tests(Some([1000, 200, 200]), Some([2000, 200, 200])), + None + ); + assert_eq!( + window.sample_for_tests(Some([1250, 300, 300]), Some([3000, 300, 300])), + Some(25.0) + ); +} + +#[test] +fn counter_reset_is_unavailable_then_recovers() { + let mut window = TestC6Window::default(); + window.sample_for_tests(Some([1000, 100, 100]), Some([2000, 100, 100])); + assert_eq!(window.sample_for_tests(Some([0, 0, 0]), Some([0, 0, 0])), None); + assert_eq!( + window.sample_for_tests(Some([750, 100, 100]), Some([1000, 100, 100])), + Some(75.0) + ); +} + +#[test] +fn unscheduled_or_inconsistent_intervals_are_unavailable() { + for (c6, tsc) in [ + ([1, 100, 0], [1000, 100, 100]), + ([1, 50, 100], [1000, 100, 100]), + ([1, 100, 100], [0, 100, 100]), + ([2000, 100, 100], [1000, 100, 100]), + ] { + let mut window = TestC6Window::default(); + window.sample_for_tests(Some([0, 0, 0]), Some([0, 0, 0])); + assert_eq!(window.sample_for_tests(Some(c6), Some(tsc)), None); + } +} diff --git a/lib/crowdb-common/rust/tests/system_metrics_test.rs b/lib/crowdb-common/rust/tests/system_metrics_test.rs new file mode 100644 index 000000000..c1fdd457b --- /dev/null +++ b/lib/crowdb-common/rust/tests/system_metrics_test.rs @@ -0,0 +1,81 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_common::metrics::system::{flush_system, C6Residency, SystemCollector, SystemMetrics}; + +#[test] +fn collect_returns_snapshot() { + let mut collector = SystemCollector::new(); + let snap = collector.collect(); + // CPU deltas should be small but non-negative. + // RSS should be non-zero (the test process has memory). + assert!(snap.rss_kb > 0, "RSS should be non-zero, got {}", snap.rss_kb); +} + +#[test] +fn collect_delta_is_non_negative() { + let mut collector = SystemCollector::new(); + let _snap1 = collector.collect(); + let snap2 = collector.collect(); + // Utilization should be non-negative (can exceed 100% on multi-core). + assert!(snap2.cpu_user_pct <= 100 * 1024); +} + +#[test] +fn flush_system_writes_all_fields() { + let snap = SystemMetrics { + cpu_user_pct: 42, + cpu_sys_pct: 17, + rss_kb: 4096, + tcp_retransmits: 3, + tcp_lost: 1, + dram_read_mib: Some(512.5), + dram_write_mib: Some(128.3), + dram_total_mib: Some(640.8), + cpu_core_c6: vec![ + C6Residency { + cpu: 0, + pct: Some(83.25), + }, + C6Residency { + cpu: 2, + pct: Some(0.0), + }, + ], + cpu_package_c6: vec![C6Residency { cpu: 0, pct: None }], + }; + let mut buf = Vec::new(); + flush_system(&mut buf, &snap); + let out = String::from_utf8(buf).unwrap(); + assert!(out.contains("cpu.user=42%")); + assert!(out.contains("cpu.sys=17%")); + assert!(out.contains("rss_gb=0.00")); + assert!(out.contains("tcp_retrans=3")); + assert!(out.contains("tcp_lost=1")); + assert!(out.contains("bw_read_mib=512.5")); + assert!(out.contains("bw_write_mib=128.3")); + assert!(out.contains("bw_total_mib=640.8")); + assert!(out.contains("cpu.c6.core@0=83.2% cpu.c6.core@2=0.0% cpu.c6.package@0=unsupported")); +} + +#[test] +fn flush_system_writes_unsupported_when_none() { + let snap = SystemMetrics { + cpu_user_pct: 0, + cpu_sys_pct: 0, + rss_kb: 0, + tcp_retransmits: 0, + tcp_lost: 0, + dram_read_mib: None, + dram_write_mib: None, + dram_total_mib: None, + ..SystemMetrics::default() + }; + let mut buf = Vec::new(); + flush_system(&mut buf, &snap); + let out = String::from_utf8(buf).unwrap(); + assert!(out.contains("bw_read_mib=unsupported")); + assert!(out.contains("bw_total_mib=unsupported")); + assert!(out.contains("bw_write_mib=unsupported")); + assert!(out.contains("cpu.c6.core=unsupported cpu.c6.package=unsupported")); +} diff --git a/lib/crowdb-console-shared/Cargo.toml b/lib/crowdb-console-shared/Cargo.toml index 160866975..b89b527da 100644 --- a/lib/crowdb-console-shared/Cargo.toml +++ b/lib/crowdb-console-shared/Cargo.toml @@ -21,6 +21,9 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls" form_urlencoded = "1" hex = "0.4" sha2 = "0.10" +hmac = "0.12" +percent-encoding = "2" +chrono = { version = "0.4", default-features = false, features = ["clock"] } uuid = { version = "1", features = ["v4", "serde"] } tokio = { workspace = true, features = ["rt", "macros", "time", "process", "io-util", "net"] } futures = "0.3" @@ -32,9 +35,12 @@ crowdb-protocol = { path = "../crowdb-protocol" } crowdb-kv-client = { path = "../crowdb-kv-client" } [dev-dependencies] +bincode = "1" +clap = { version = "4", features = ["derive"] } +crowdb-kv-server = { path = "../../app/crowdb-kv-server" } crowdb-kv = { path = "../crowdb-kv" } crowdb-console-shared = { path = ".", features = ["test-util"] } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "net"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "net", "signal"] } axum = "0.7" # KV data-plane crowdb-rpc client for `tests/kv_e2e_test.rs` (C6's own crowdb-rpc # `KvClient` wrapper is gone; that test now exercises the real @@ -43,3 +49,8 @@ crowdb-kv-client = { path = "../crowdb-kv-client" } crowdb-chunkdb-client = { path = "../crowdb-chunkdb-client" } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } crowdb-test-harness = { path = "../crowdb-test-harness" } + +[[example]] +name = "regression-fixture" +path = "tests/common/regression_fixture.rs" +required-features = ["test-util"] diff --git a/lib/crowdb-console-shared/src/bootstrap_intent.rs b/lib/crowdb-console-shared/src/bootstrap_intent.rs index 1e68ed6e0..bf5261276 100644 --- a/lib/crowdb-console-shared/src/bootstrap_intent.rs +++ b/lib/crowdb-console-shared/src/bootstrap_intent.rs @@ -259,7 +259,7 @@ impl BootstrapIntent { .servers .iter() .map(|server| ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: server.id.clone(), url: server.management_url.clone(), node_id: Some(server.node_id), diff --git a/lib/crowdb-console-shared/src/config.rs b/lib/crowdb-console-shared/src/config.rs index 0688d507c..ef1408ebc 100644 --- a/lib/crowdb-console-shared/src/config.rs +++ b/lib/crowdb-console-shared/src/config.rs @@ -198,9 +198,9 @@ pub struct ServerEntry { pub rpc_url: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub rest_port: Option, - /// Configured Dataset origin for an Access service, projected from its intent. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub dataset_origin: Option, + /// Internal protocol origins for Console requests, projected from service listeners. + #[serde(default, skip_serializing_if = "std::collections::BTreeMap::is_empty")] + pub access_origins: std::collections::BTreeMap, #[serde(default, skip_serializing_if = "Option::is_none")] pub rpc_port: Option, #[serde(default)] @@ -260,7 +260,7 @@ impl ServerEntry { node_id: None, rpc_url: None, rest_port: None, - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), rpc_port: None, auto_start: false, binary: None, diff --git a/lib/crowdb-console-shared/src/deployment.rs b/lib/crowdb-console-shared/src/deployment.rs index d7430979c..48a81f9a8 100644 --- a/lib/crowdb-console-shared/src/deployment.rs +++ b/lib/crowdb-console-shared/src/deployment.rs @@ -6,7 +6,10 @@ pub mod admission; mod bootstrap; pub mod node_update; +pub mod preparation; +pub mod recovery; pub mod registry; pub mod services; +pub mod target; pub use bootstrap::{PreparedBootstrap, CLUSTER_KEY}; diff --git a/lib/crowdb-console-shared/src/deployment/admission.rs b/lib/crowdb-console-shared/src/deployment/admission.rs index 12447488d..270b7e689 100644 --- a/lib/crowdb-console-shared/src/deployment/admission.rs +++ b/lib/crowdb-console-shared/src/deployment/admission.rs @@ -61,6 +61,34 @@ pub async fn remote_control(node: &NodeEntry, command: &NodeControl) -> Result Result { + if node.id != member.node_id || node.ssh_password.is_some() || node.ssh_key.is_none() { + return Err(Error::Config( + "pinned member control requires its identity and local key".into(), + )); + } + let creds = SshCreds::resolve(node)?; + let key = member.key()?; + let operation = async { + let mut session = Session::connect_pinned(node, &creds, key).await?; + let result = control(&mut session, command).await; + session.close().await; + result + }; + tokio::time::timeout(Duration::from_secs(10), operation) + .await + .map_err(|_| Error::Config("pinned member control timed out".into()))? +} + /// Execute ordered commands through one authenticated session, stopping at the /// first rejection and retaining already completed durable stages for retry. /// @@ -273,14 +301,15 @@ async fn control(session: &mut Session, request: &NodeControl) -> Result "LD_LIBRARY_PATH=/opt/crowdb/lib /opt/crowdb/bin/crowdb-monitor control --json {}", quote(&json) ); - let output = tokio::time::timeout(Duration::from_secs(45), session.exec(&command)) + let output = tokio::time::timeout(Duration::from_secs(45), session.exec_bounded(&command, 65536)) .await .map_err(config_error)??; if !output.success() { - return Err(Error::Config(format!( - "monitor SSH command failed: {}", - output.stderr_str() - ))); + let message = output.stderr_str(); + if let Some(reason) = message.trim().strip_prefix("Error: Waiting: ") { + return Err(Error::DependenciesNotReady(reason.into())); + } + return Err(Error::Config(format!("monitor SSH command failed: {message}"))); } if output.stdout.len() > 65536 { return Err(Error::Config("monitor reply too large".into())); diff --git a/lib/crowdb-console-shared/src/deployment/node_update.rs b/lib/crowdb-console-shared/src/deployment/node_update.rs index b00953578..629382737 100644 --- a/lib/crowdb-console-shared/src/deployment/node_update.rs +++ b/lib/crowdb-console-shared/src/deployment/node_update.rs @@ -41,6 +41,7 @@ pub async fn apply(ctx: &OpContext, cluster: &str, target: NodeRecord) -> Result || source.node_id != target.node_id || source.physical_host_id != target.physical_host_id || source.operation_id != target.operation_id + || source.host_key != target.host_key { return Err(Error::Config( "node update changes its authority or identity".into(), diff --git a/lib/crowdb-console-shared/src/deployment/preparation.rs b/lib/crowdb-console-shared/src/deployment/preparation.rs new file mode 100644 index 000000000..86a3022cf --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/preparation.rs @@ -0,0 +1,259 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! One Group-0-owned default preparation, shared by all authorized node runners. + +mod binding; +mod record; +pub use binding::{chunk_kv_binding, metadata_binding, storage_binding}; + +pub use record::{ + DefaultChunkKvBootstrap, DefaultMetadataGroup, DefaultPreparation, DefaultReplica, DefaultStorageBinding, +}; + +use crowdb_kv_client::{Error as KvError, GetOutcome, ReadMode}; +use crowdb_protocol::mgmt::SystemBootstrapIdentity; + +use super::{PreparedBootstrap, CLUSTER_KEY}; +use crate::{ + error::{Error, Result}, + ops::OpContext, +}; + +pub const DEFAULT_PREPARATION_KEY: &[u8] = b"/deployment/default-preparation"; + +pub(crate) async fn check_group_creation(ctx: &OpContext, store: u64, group: u64) -> Result<()> { + if for_group(ctx, store, group) + .await? + .is_some_and(|current| !current.retired) + { + return Err(Error::Config( + "group is reserved by accepted default preparation".into(), + )); + } + Ok(()) +} + +pub(crate) async fn retire_group(ctx: &OpContext, store: u64, group: u64) -> Result<()> { + if let Some(current) = for_group(ctx, store, group).await? { + retire(ctx, ¤t.bootstrap, ¤t.operation_id).await?; + } + Ok(()) +} + +async fn for_group(ctx: &OpContext, store: u64, group: u64) -> Result> { + if (store, group) != (0, 1) { + return Ok(None); + } + let GetOutcome::Found { value, .. } = ctx + .kv() + .get(0, 0, DEFAULT_PREPARATION_KEY, ReadMode::Linearizable, None) + .await? + else { + return Ok(None); + }; + if value.len() > 32768 { + return Err(Error::Config("default preparation exceeds its size bound".into())); + } + let current: DefaultPreparation = serde_json::from_slice(&value).map_err(config_error)?; + let bootstrap = confirmed(ctx, ¤t.bootstrap).await?; + current.validate(&bootstrap)?; + Ok(Some(current)) +} + +/// Reserve fixed identities before creating any ordinary group, disks or services. +/// +/// # Errors +/// Rejects unconfirmed/changed authority, malformed or retired operations and +/// preexisting objects without this operation's ownership. CAS ambiguity is +/// resolved through authoritative readback, never a new local operation. +pub async fn reserve(ctx: &OpContext, identity: &SystemBootstrapIdentity) -> Result { + tokio::time::timeout(std::time::Duration::from_secs(3), reserve_inner(ctx, identity)) + .await + .map_err(|_| Error::Config("default preparation authority reservation timed out".into()))? +} + +async fn reserve_inner(ctx: &OpContext, identity: &SystemBootstrapIdentity) -> Result { + let bootstrap = confirmed(ctx, identity).await?; + let candidate = DefaultPreparation::new(&bootstrap)?; + let bytes = serde_json::to_vec(&candidate).map_err(config_error)?; + for _ in 0..10 { + if let Some((current, _)) = read_record(ctx, &bootstrap).await? { + if current.retired { + return Err(Error::Config("default preparation was explicitly retired".into())); + } + return Ok(current); + } + if let Err(error) = check_unowned_objects(ctx, &candidate).await { + // A winning runner may have published and begun creating objects + // between our absent read and the ownership check. + if let Some((current, _)) = read_record(ctx, &bootstrap).await? { + if !current.retired { + return Ok(current); + } + } + return Err(error); + } + match ctx.kv().put_cas(0, 0, DEFAULT_PREPARATION_KEY, &bytes, 0).await { + Ok(_) => return Ok(candidate), + Err(KvError::CasFailed { .. } | KvError::CasBusy | KvError::OutcomeUnknown) => {} + Err(error) => return Err(error.into()), + } + } + Err(KvError::OutcomeUnknown.into()) +} + +/// Read the accepted preparation without reserving or repairing anything. +/// +/// # Errors +/// Rejects unavailable/conflicting authority and invalid committed records. +pub async fn read( + ctx: &OpContext, + identity: &SystemBootstrapIdentity, +) -> Result> { + let bootstrap = confirmed(ctx, identity).await?; + read_record(ctx, &bootstrap).await +} + +/// Permanently retire this default operation before explicit topology deletion. +/// +/// # Errors +/// Rejects changed operation identity, missing authority and failed publication. +pub async fn retire(ctx: &OpContext, identity: &SystemBootstrapIdentity, operation: &str) -> Result<()> { + let bootstrap = confirmed(ctx, identity).await?; + for _ in 0..10 { + let (mut current, revision) = read_record(ctx, &bootstrap) + .await? + .ok_or_else(|| Error::Config("default preparation has not been accepted".into()))?; + if current.operation_id != operation { + return Err(Error::Config("default preparation operation differs".into())); + } + if current.retired { + return Ok(()); + } + current.retired = true; + let bytes = serde_json::to_vec(¤t).map_err(config_error)?; + match ctx + .kv() + .put_cas(0, 0, DEFAULT_PREPARATION_KEY, &bytes, revision) + .await + { + Ok(_) => return Ok(()), + Err(KvError::CasFailed { .. } | KvError::CasBusy | KvError::OutcomeUnknown) => {} + Err(error) => return Err(error.into()), + } + } + Err(KvError::OutcomeUnknown.into()) +} + +async fn confirmed(ctx: &OpContext, identity: &SystemBootstrapIdentity) -> Result { + let GetOutcome::Found { value, .. } = ctx + .kv() + .get(0, 0, CLUSTER_KEY, ReadMode::Linearizable, None) + .await? + else { + return Err(Error::Config( + "default preparation requires committed cluster authority".into(), + )); + }; + if value.len() > 1024 * 1024 { + return Err(Error::Config( + "committed cluster identity exceeds its size bound".into(), + )); + } + let bootstrap: PreparedBootstrap = serde_json::from_slice(&value).map_err(config_error)?; + bootstrap.validate()?; + if bootstrap.identity != *identity { + return Err(Error::Config( + "default preparation cluster identity differs".into(), + )); + } + Ok(bootstrap) +} + +async fn read_record( + ctx: &OpContext, + bootstrap: &PreparedBootstrap, +) -> Result> { + let GetOutcome::Found { value, revision } = ctx + .kv() + .get(0, 0, DEFAULT_PREPARATION_KEY, ReadMode::Linearizable, None) + .await? + else { + return Ok(None); + }; + if value.len() > 32768 { + return Err(Error::Config("default preparation exceeds its size bound".into())); + } + let current: DefaultPreparation = serde_json::from_slice(&value).map_err(config_error)?; + current.validate(bootstrap)?; + Ok(Some((current, revision))) +} + +async fn check_unowned_objects(ctx: &OpContext, candidate: &DefaultPreparation) -> Result<()> { + let groups = ctx.sysmd().list_disk_groups().await?; + for binding in &candidate.storage { + let reservation = + crowdb_protocol::disk_storage::DiskGroupIdReservation::key_for(binding.disk_group_id); + if groups.iter().any(|group| group.dg_id == binding.disk_group_id) + || matches!( + ctx.kv() + .get(0, 0, reservation.as_bytes(), ReadMode::Linearizable, None) + .await?, + GetOutcome::Found { .. } + ) + { + return Err(Error::Config( + "default disk group ID already has a global owner".into(), + )); + } + } + let metadata = &candidate.metadata; + if ctx + .membership() + .read(metadata.store_id, metadata.group_id) + .await? + .is_some() + { + return Err(Error::Config( + "default metadata membership already has an owner".into(), + )); + } + if ctx + .sysmd() + .get_group(metadata.store_id, metadata.group_id) + .await? + .is_some() + || !ctx + .sysmd() + .list_replicas_in_group(metadata.store_id, metadata.group_id) + .await? + .is_empty() + { + return Err(Error::Config( + "default metadata group already exists without accepted ownership".into(), + )); + } + for binding in &candidate.storage { + if ctx + .sysmd() + .get_disk_group(binding.rack_id, binding.node_id, binding.disk_group_id) + .await? + .is_some() + || ctx + .sysmd() + .get_bind(binding.rack_id, binding.node_id, binding.disk_group_id) + .await? + .is_some() + { + return Err(Error::Config( + "default storage binding already exists without accepted ownership".into(), + )); + } + } + Ok(()) +} + +fn config_error(error: impl std::fmt::Display) -> Error { + Error::Config(error.to_string()) +} diff --git a/lib/crowdb-console-shared/src/deployment/preparation/binding.rs b/lib/crowdb-console-shared/src/deployment/preparation/binding.rs new file mode 100644 index 000000000..63c6c6fe5 --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/preparation/binding.rs @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Resolve only the accepted metadata binding; never select an unrelated group. + +use crowdb_protocol::mgmt::SystemBootstrapIdentity; + +use super::{DefaultChunkKvBootstrap, DefaultMetadataGroup, DefaultPreparation, DefaultStorageBinding}; +use crate::{ + error::{Error, Result}, + ops::OpContext, +}; + +async fn active(ctx: &OpContext, identity: &SystemBootstrapIdentity) -> Result { + let (accepted, _) = super::read(ctx, identity) + .await? + .ok_or_else(|| Error::Config("ordinary metadata group preparation has not been accepted".into()))?; + if accepted.retired { + return Err(Error::Config( + "ordinary metadata group preparation was explicitly retired".into(), + )); + } + let metadata = &accepted.metadata; + let actual = ctx + .sysmd() + .get_group(metadata.store_id, metadata.group_id) + .await? + .ok_or_else(|| Error::Config("accepted ordinary metadata group is not published".into()))?; + if actual.store_id != metadata.store_id || actual.group_id != metadata.group_id { + return Err(Error::Config( + "ordinary metadata group identity differs from its accepted binding".into(), + )); + } + Ok(accepted) +} + +/// Resolve the fixed ordinary group without reserving or recreating it. +/// +/// # Errors +/// Rejects absent, retired or conflicting preparation/publication and lost authority. +pub async fn metadata_binding( + ctx: &OpContext, + identity: &SystemBootstrapIdentity, +) -> Result { + Ok(active(ctx, identity).await?.metadata) +} + +/// Resolve the single initial catalog creator and durable partition identities. +/// +/// # Errors +/// Rejects absent, retired or conflicting default preparation and metadata. +pub async fn chunk_kv_binding( + ctx: &OpContext, + identity: &SystemBootstrapIdentity, +) -> Result { + Ok(active(ctx, identity).await?.chunk_kv) +} + +/// Resolve one admitted node's storage metadata and initial disk-group reservation. +/// +/// # Errors +/// Rejects changed rack/node mapping and unavailable or retired metadata authority. +pub async fn storage_binding( + ctx: &OpContext, + identity: &SystemBootstrapIdentity, + rack: u64, + node: u64, +) -> Result { + active(ctx, identity) + .await? + .storage + .into_iter() + .find(|binding| binding.rack_id == rack && binding.node_id == node) + .ok_or_else(|| Error::Config("node storage is outside the accepted default binding".into())) +} diff --git a/lib/crowdb-console-shared/src/deployment/preparation/record.rs b/lib/crowdb-console-shared/src/deployment/preparation/record.rs new file mode 100644 index 000000000..b60d58586 --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/preparation/record.rs @@ -0,0 +1,139 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::{chunk_kv::Id128, chunk_stream::StreamName, mgmt::SystemBootstrapIdentity}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::{ + deployment::PreparedBootstrap, + error::{Error, Result}, +}; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DefaultReplica { + pub node_id: u64, + pub replica_id: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DefaultMetadataGroup { + pub store_id: u64, + pub group_id: u64, + pub replicas: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DefaultStorageBinding { + pub rack_id: u64, + pub node_id: u64, + /// Initial automatic-medium placement. Supplied media retain compatible + /// per-class placement in the target's sealed journal under this metadata. + pub disk_group_id: u64, + pub store_id: u64, + pub group_id: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DefaultChunkKvBootstrap { + pub node_id: u64, + pub partition_id: Id128, + pub tree_id: u64, + pub stream_name: StreamName, + pub owner_epoch: u64, + pub metadata_group_id: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DefaultPreparation { + pub version: u32, + pub bootstrap: SystemBootstrapIdentity, + pub operation_id: String, + pub metadata: DefaultMetadataGroup, + pub storage: Vec, + pub chunk_kv: DefaultChunkKvBootstrap, + /// Reference only; access secrets are never included in progress/status. + pub credential_ref: String, + pub retired: bool, +} + +impl DefaultPreparation { + pub(super) fn new(bootstrap: &PreparedBootstrap) -> Result { + Self::with_operation(bootstrap, Uuid::new_v4().to_string()) + } + + fn with_operation(bootstrap: &PreparedBootstrap, operation_id: String) -> Result { + let members = bootstrap.intent.members(); + if !matches!(members.len(), 1 | 3) { + return Err(Error::Config( + "default preparation requires one or three admitted voters".into(), + )); + } + let config = bootstrap.intent.to_config(); + let (high, low) = Uuid::parse_str(&operation_id) + .map_err(|_| Error::Config("default preparation operation is invalid".into()))? + .as_u64_pair(); + let high = (high & i64::MAX as u64).max(1); + let low = (low & i64::MAX as u64).max(1); + let mut replicas = Vec::new(); + let mut storage = Vec::new(); + for (index, node_id) in members.iter().enumerate() { + let node = config + .node(*node_id) + .ok_or_else(|| Error::Config("default preparation member is not admitted".into()))?; + replicas.push(DefaultReplica { + node_id: *node_id, + replica_id: index as u64 + 1, + }); + storage.push(DefaultStorageBinding { + rack_id: node.rack_id, + node_id: *node_id, + disk_group_id: index as u64 + 1, + store_id: 0, + group_id: 1, + }); + } + Ok(Self { + version: 1, + bootstrap: bootstrap.identity.clone(), + credential_ref: format!("access/default/{operation_id}"), + operation_id, + metadata: DefaultMetadataGroup { + store_id: 0, + group_id: 1, + replicas, + }, + storage, + chunk_kv: DefaultChunkKvBootstrap { + node_id: members[0], + partition_id: Id128 { high, low }, + tree_id: low, + stream_name: StreamName { high, low: low ^ 1 }, + owner_epoch: 1, + metadata_group_id: 1, + }, + retired: false, + }) + } + + pub(super) fn validate(&self, bootstrap: &PreparedBootstrap) -> Result<()> { + let operation = Uuid::parse_str(&self.operation_id) + .map_err(|_| Error::Config("default preparation operation is invalid".into()))?; + if operation.is_nil() || operation.to_string() != self.operation_id { + return Err(Error::Config("default preparation operation is invalid".into())); + } + let mut expected = Self::with_operation(bootstrap, self.operation_id.clone())?; + expected.retired = self.retired; + if self != &expected { + return Err(Error::Config( + "default preparation differs from committed membership".into(), + )); + } + Ok(()) + } +} diff --git a/lib/crowdb-console-shared/src/deployment/recovery.rs b/lib/crowdb-console-shared/src/deployment/recovery.rs new file mode 100644 index 000000000..78e19c455 --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/recovery.rs @@ -0,0 +1,169 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Identity-bound trust for rediscovery; addresses are never trust anchors. + +mod endpoints; +mod services; +pub use services::rebind_service; +mod confirmed; +mod persistence; +mod registry; +pub use confirmed::ConfirmedMemberTrust; + +use std::collections::BTreeSet; + +use russh::keys::{key::PublicKey, parse_public_key_base64}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::PreparedBootstrap; +use crate::config::NodeEntry; +use crate::error::{Error, Result}; +use crowdb_protocol::mgmt::node::{observation::NodeRuntimeObservation, NodeControl}; +use crowdb_protocol::mgmt::SystemBootstrapIdentity; + +/// Public identity pins established through authenticated bootstrap/admission. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MemberTrust { + pub version: u32, + pub bootstrap: SystemBootstrapIdentity, + pub members: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TrustedMember { + pub discovery_id: String, + pub node_id: u64, + pub admission_operation_id: String, + pub host_key: String, +} + +impl MemberTrust { + /// Read a member's retained identity over SSH pinned independently of its IP. + /// + /// # Errors + /// Rejects unknown members, SSH failures and conflicting durable identities. + pub async fn observe(&self, node: &NodeEntry) -> Result { + self.validate()?; + let member = self + .members + .iter() + .find(|member| member.node_id == node.id) + .ok_or_else(|| invalid("node is not an accepted recovery member"))?; + let value = + super::admission::remote_control_pinned(node, member, &NodeControl::RuntimeObservation).await?; + let observed: NodeRuntimeObservation = + serde_json::from_value(value).map_err(|_| invalid("invalid authenticated member observation"))?; + if observed.version != 1 + || observed.discovery_id != member.discovery_id + || !observed.binding.as_ref().is_some_and(|binding| { + binding.node_id == member.node_id && binding.bootstrap == self.bootstrap + }) + { + return Err(invalid("authenticated peer differs from accepted member binding")); + } + Ok(observed) + } + + /// Validate pins against the exact accepted initial membership. + /// + /// # Errors + /// Rejects mismatched, missing, duplicate or malformed identity bindings. + pub fn validate_bootstrap(&self, accepted: &PreparedBootstrap) -> Result<()> { + accepted.validate()?; + self.validate()?; + if self.bootstrap != accepted.identity + || self.members.len() != accepted.nodes.len() + || self.members.len() != accepted.intent.members().len() + { + return Err(invalid("member trust differs from accepted bootstrap")); + } + for member in &self.members { + let expected = member.key()?; + if !accepted.intent.members().contains(&member.node_id) + || !accepted.nodes.iter().any(|node| { + node.node_id == member.node_id + && node.discovery_id == member.discovery_id + && node.operation_id == member.admission_operation_id + && !node.cancelled + && node.host_key.as_ref().map_or(true, |key| { + host_key(key).is_ok_and(|key| key.fingerprint() == expected.fingerprint()) + }) + }) + { + return Err(invalid("member trust changes accepted membership")); + } + } + Ok(()) + } + + /// # Errors + /// Rejects unsupported versions, invalid identities and shared host keys. + pub fn validate(&self) -> Result<()> { + if self.version != 1 || self.members.is_empty() || self.members.len() > 4096 { + return Err(invalid("invalid member trust version or size")); + } + valid_uuid(&self.bootstrap.cluster_id)?; + valid_uuid(&self.bootstrap.operation_id)?; + if self.bootstrap.configuration_digest.len() != 64 + || !self + .bootstrap + .configuration_digest + .bytes() + .all(|b| b.is_ascii_hexdigit()) + { + return Err(invalid("invalid member trust bootstrap digest")); + } + let mut ids = BTreeSet::new(); + let mut uuids = BTreeSet::new(); + let mut keys = BTreeSet::new(); + for member in &self.members { + valid_uuid(&member.discovery_id)?; + valid_uuid(&member.admission_operation_id)?; + let key = member.key()?; + if member.node_id == 0 + || !ids.insert(member.node_id) + || !uuids.insert(&member.discovery_id) + || !keys.insert(key.fingerprint()) + { + return Err(invalid("duplicate or invalid trusted member identity")); + } + } + Ok(()) + } +} + +impl TrustedMember { + pub(crate) fn key(&self) -> Result { + host_key(&self.host_key) + } +} + +pub(crate) fn host_key(text: &str) -> Result { + let fields: Vec<_> = text.split_whitespace().collect(); + if text.len() > 4096 || text.lines().count() != 1 || fields.len() < 2 || fields[0] != "ssh-ed25519" { + return Err(invalid("expected one Ed25519 SSH host key")); + } + let key = parse_public_key_base64(fields[1]).map_err(|_| invalid("invalid SSH host key"))?; + if key.name() != "ssh-ed25519" { + return Err(invalid("SSH host key algorithm differs")); + } + Ok(key) +} + +fn valid_uuid(value: &str) -> Result<()> { + if Uuid::parse_str(value) + .map_err(|_| invalid("invalid member trust UUID"))? + .is_nil() + { + return Err(invalid("nil member trust UUID")); + } + Ok(()) +} + +fn invalid(message: &str) -> Error { + Error::Config(message.into()) +} diff --git a/lib/crowdb-console-shared/src/deployment/recovery/confirmed.rs b/lib/crowdb-console-shared/src/deployment/recovery/confirmed.rs new file mode 100644 index 000000000..fbb7b0df9 --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/recovery/confirmed.rs @@ -0,0 +1,130 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Confirmed non-voter recovery contacts retain the original voting authority. + +use super::{invalid, MemberTrust, TrustedMember}; +use crate::{ + config::{NodeEntry, ServerEntry}, + deployment::{registry::NodeRegistry, PreparedBootstrap}, + error::Result, + ConsoleConfig, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ConfirmedMemberTrust { + pub version: u32, + pub initial: MemberTrust, + pub current: MemberTrust, + pub registry: NodeRegistry, +} + +impl MemberTrust { + /// Read the initial public pins sealed into the authenticated manifest. + /// + /// # Errors + /// Rejects an invalid manifest or any missing authenticated host key. + pub fn from_bootstrap(operation: &PreparedBootstrap) -> Result { + operation.validate()?; + let mut members = operation + .nodes + .iter() + .map(|node| { + Ok(TrustedMember { + discovery_id: node.discovery_id.clone(), + node_id: node.node_id, + admission_operation_id: node.operation_id.clone(), + host_key: node + .host_key + .clone() + .ok_or_else(|| invalid("bootstrap has no authenticated member key"))?, + }) + }) + .collect::>>()?; + members.sort_by_key(|member| member.node_id); + let trust = Self { + version: 1, + bootstrap: operation.identity.clone(), + members, + }; + trust.validate_bootstrap(operation)?; + Ok(trust) + } +} + +impl ConfirmedMemberTrust { + /// # Errors + /// Requires the current registry and the immutable initial pins to agree. + pub fn new(initial: MemberTrust, registry: NodeRegistry) -> Result { + let current = initial.confirmed_registry(®istry)?; + Ok(Self { + version: 1, + initial, + current, + registry, + }) + } + + /// # Errors + /// Rejects changed initial votes/pins, pending members and mismatched topology. + pub fn validate(&self, operation: &PreparedBootstrap) -> Result<()> { + self.initial.validate_bootstrap(operation)?; + if self.version != 1 + || self.current != self.initial.confirmed_registry(&self.registry)? + || self + .initial + .members + .iter() + .any(|member| !self.current.members.contains(member)) + { + return Err(invalid( + "confirmed recovery contacts change the accepted initial authority", + )); + } + Ok(()) + } + + /// Retain original voting/store configuration and add confirmed contact hints. + /// + /// # Errors + /// Rejects malformed identities or an incomplete confirmed registry snapshot. + pub fn topology(&self, operation: &PreparedBootstrap) -> Result { + self.validate(operation)?; + let mut config = operation.intent.to_config(); + for member in &self.current.members { + if config.node(member.node_id).is_some() { + continue; + } + let node = self + .registry + .nodes + .iter() + .find(|node| node.node_id == member.node_id) + .ok_or_else(|| invalid("confirmed member topology is absent"))?; + config.nodes.push(NodeEntry { + id: node.node_id, + rack_id: node.rack_id, + host: node.host.clone(), + ssh_port: node.ssh_port, + ssh_user: node.ssh_user.clone(), + ssh_key: None, + ssh_password: None, + ssh_credential_ref: None, + }); + let address = node + .host + .parse::() + .map_err(|_| invalid("invalid recovery contact address"))?; + let mut server = ServerEntry::new( + format!("kv-{}", node.node_id), + format!("http://{}", std::net::SocketAddr::new(address, 10000)), + ); + server.node_id = Some(node.node_id); + server.rpc_url = Some(std::net::SocketAddr::new(address, 10100).to_string()); + config.servers.push(server); + } + Ok(config) + } +} diff --git a/lib/crowdb-console-shared/src/deployment/recovery/endpoints.rs b/lib/crowdb-console-shared/src/deployment/recovery/endpoints.rs new file mode 100644 index 000000000..8d3b649c2 --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/recovery/endpoints.rs @@ -0,0 +1,50 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{invalid, valid_uuid, MemberTrust}; +use crate::{config::NodeEntry, error::Result}; +use crowdb_protocol::mgmt::node::{recovery::MemberRecoveryIdentity, NodeControl}; + +impl MemberTrust { + /// Verify durable identity and current addresses over the pinned SSH channel. + /// + /// # Errors + /// Rejects unknown members, foreign bindings, malformed addresses and keys. + pub async fn endpoint(&self, node: &NodeEntry) -> Result { + self.validate()?; + let member = self + .members + .iter() + .find(|member| member.node_id == node.id) + .ok_or_else(|| invalid("unknown recovery member"))?; + let value = + super::super::admission::remote_control_pinned(node, member, &NodeControl::RecoveryIdentity) + .await?; + let identity: MemberRecoveryIdentity = + serde_json::from_value(value).map_err(|_| invalid("invalid authenticated recovery identity"))?; + if identity.version != 1 + || identity.binding.bootstrap != self.bootstrap + || identity.binding.node_id != member.node_id + || identity.endpoint.node_id != member.node_id + || identity.endpoint.discovery_id != member.discovery_id + || identity.endpoint.addresses.is_empty() + || identity.endpoint.addresses.len() > 16 + || identity + .endpoint + .addresses + .iter() + .any(|address| address.is_unspecified() || address.is_multicast()) + || !identity + .endpoint + .addresses + .iter() + .any(|address| address.to_string() == node.host) + { + return Err(invalid( + "authenticated recovery endpoint differs from accepted member", + )); + } + valid_uuid(&identity.endpoint.boot_id)?; + Ok(identity) + } +} diff --git a/lib/crowdb-console-shared/src/deployment/recovery/persistence.rs b/lib/crowdb-console-shared/src/deployment/recovery/persistence.rs new file mode 100644 index 000000000..783d1c744 --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/recovery/persistence.rs @@ -0,0 +1,76 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Immutable publication; recovery never learns replacement keys from discovery. + +use std::{ + fs::{self, File, OpenOptions}, + io::{Read, Write}, + os::unix::fs::{OpenOptionsExt, PermissionsExt}, + path::Path, +}; + +use super::{invalid, MemberTrust}; +use crate::error::Result; + +const MAX_BYTES: u64 = 1024 * 1024; + +impl MemberTrust { + /// # Errors + /// Rejects missing, corrupt, exposed, oversized and symlinked trust records. + pub fn load(path: &Path) -> Result { + let metadata = fs::symlink_metadata(path)?; + if !metadata.is_file() || metadata.permissions().mode() & 0o777 != 0o600 || metadata.len() > MAX_BYTES + { + return Err(invalid("invalid durable member trust file")); + } + let mut bytes = Vec::new(); + File::open(path)?.take(MAX_BYTES + 1).read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_BYTES { + return Err(invalid("member trust too large")); + } + let trust: Self = serde_json::from_slice(&bytes).map_err(|_| invalid("corrupt member trust"))?; + trust.validate()?; + Ok(trust) + } + + /// Seal authenticated inputs once; concurrent callers must agree exactly. + /// + /// # Errors + /// Rejects changed pins, invalid files and publication failures. + pub fn seal(&self, path: &Path) -> Result<()> { + self.validate()?; + let mut canonical = self.clone(); + canonical.members.sort_by_key(|member| member.node_id); + let bytes = serde_json::to_vec(&canonical).map_err(|_| invalid("invalid member trust"))?; + if bytes.len() as u64 > MAX_BYTES { + return Err(invalid("member trust too large")); + } + let parent = path + .parent() + .ok_or_else(|| invalid("member trust has no parent"))?; + let temporary = parent.join(format!(".member-trust-{}.tmp", uuid::Uuid::new_v4())); + let result = (|| { + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + match fs::hard_link(&temporary, path) { + Ok(()) => File::open(parent)?.sync_all()?, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + if Self::load(path)? != canonical { + return Err(invalid("member trust differs from accepted pins")); + } + Ok(()) + })(); + let cleanup = fs::remove_file(temporary); + result?; + cleanup?; + Ok(()) + } +} diff --git a/lib/crowdb-console-shared/src/deployment/recovery/registry.rs b/lib/crowdb-console-shared/src/deployment/recovery/registry.rs new file mode 100644 index 000000000..1373c1a32 --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/recovery/registry.rs @@ -0,0 +1,66 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Extend initial recovery pins only from a current confirmed Group-0 registry. + +use super::{invalid, MemberTrust, TrustedMember}; +use crate::{deployment::registry::NodeRegistry, error::Result}; + +impl MemberTrust { + /// Build the current recovery destinations from confirmed authority. + /// + /// The caller must obtain this registry through a current linearizable read. + /// This projection cannot be used as a cached admission or voting grant. + /// + /// # Errors + /// Rejects another cluster, changed initial identities/keys and unpinned + /// confirmed members. Pending and cancelled admissions grant no trust. + pub fn confirmed_registry(&self, registry: &NodeRegistry) -> Result { + self.validate()?; + if registry.cluster_id != self.bootstrap.cluster_id || registry.nodes.len() > 4096 { + return Err(invalid( + "recovery registry belongs to another cluster or exceeds its bound", + )); + } + let mut members = Vec::new(); + for node in registry + .nodes + .iter() + .filter(|node| node.confirmed && !node.cancelled) + { + if let Some(initial) = self.members.iter().find(|member| member.node_id == node.node_id) { + if initial.discovery_id != node.discovery_id + || initial.admission_operation_id != node.operation_id + || node.host_key.as_ref().is_some_and(|key| { + super::host_key(key).map_or(true, |key| { + initial + .key() + .map_or(true, |accepted| accepted.fingerprint() != key.fingerprint()) + }) + }) + { + return Err(invalid("confirmed registry changes an initial recovery pin")); + } + members.push(initial.clone()); + } else { + members.push(TrustedMember { + discovery_id: node.discovery_id.clone(), + node_id: node.node_id, + admission_operation_id: node.operation_id.clone(), + host_key: node + .host_key + .clone() + .ok_or_else(|| invalid("confirmed member has no authenticated host key"))?, + }); + } + } + members.sort_by_key(|member| member.node_id); + let current = Self { + version: self.version, + bootstrap: self.bootstrap.clone(), + members, + }; + current.validate()?; + Ok(current) + } +} diff --git a/lib/crowdb-console-shared/src/deployment/recovery/services.rs b/lib/crowdb-console-shared/src/deployment/recovery/services.rs new file mode 100644 index 000000000..5796d9fab --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/recovery/services.rs @@ -0,0 +1,68 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Rebind node-owned internal listeners without changing public client URIs. + +use crate::error::{Error, Result}; +use std::net::{IpAddr, SocketAddr}; + +/// # Errors +/// Rejects malformed retained service configuration and invalid node addresses. +pub fn rebind_service(configuration: &str, old: IpAddr, current: IpAddr, seeds: &[String]) -> Result { + let mut config: toml::Value = toml::from_str(configuration).map_err(|e| Error::Config(e.to_string()))?; + for path in [ + "server.listen_addr", + "server.http_listen_addr", + "server.rpc_listen_addr", + "server.bind_address", + "http_listen_addr", + "rpc_listen_addr", + "http_advertise_addr", + "rpc_advertise_addr", + "dataset.listen", + "iceberg.listen", + "s3.listen", + "health.listen", + ] { + let mut keys = path.split('.'); + let Some(first) = keys.next() else { + continue; + }; + let value = if let Some(second) = keys.next() { + config.get_mut(first).and_then(|table| table.get_mut(second)) + } else { + config.get_mut(first) + }; + if let Some(value) = value { + if value.as_str().and_then(|text| text.parse::().ok()) == Some(old) { + *value = toml::Value::String(current.to_string()); + continue; + } + if let Some(address) = value.as_str().and_then(|text| text.parse::().ok()) { + if address.ip() == old { + *value = toml::Value::String(SocketAddr::new(current, address.port()).to_string()); + } + } + } + } + if !seeds.is_empty() { + for path in [ + "server.kv_server_mgmt_seeds", + "group0.kv_seeds", + "group0_mgmt_seeds", + "common.management_seeds", + ] { + let mut keys = path.split('.'); + let first = keys.next().unwrap_or_default(); + let value = if let Some(second) = keys.next() { + config.get_mut(first).and_then(|table| table.get_mut(second)) + } else { + config.get_mut(first) + }; + if let Some(value) = value.filter(|value| value.is_array()) { + *value = toml::Value::Array(seeds.iter().cloned().map(toml::Value::String).collect()); + } + } + } + toml::to_string(&config).map_err(|e| Error::Config(e.to_string())) +} diff --git a/lib/crowdb-console-shared/src/deployment/registry.rs b/lib/crowdb-console-shared/src/deployment/registry.rs index 7528c874b..87b0cd3b7 100644 --- a/lib/crowdb-console-shared/src/deployment/registry.rs +++ b/lib/crowdb-console-shared/src/deployment/registry.rs @@ -19,6 +19,9 @@ pub struct NodeRecord { pub ssh_port: u16, pub ssh_user: String, pub operation_id: String, + /// Public key proved by the authenticated admission, independent of its address. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub host_key: Option, pub confirmed: bool, #[serde(default)] pub cancelled: bool, @@ -30,6 +33,26 @@ pub struct NodeRegistry { pub nodes: Vec, } +impl NodeRecord { + /// Retain the public SSH key proved by admission; an address update cannot replace it. + /// + /// # Errors + /// Rejects malformed keys or an attempted replacement of an accepted pin. + pub fn pin_host_key(&mut self, authenticated: &str) -> Result<()> { + use russh::keys::PublicKeyBase64; + let key = super::recovery::host_key(authenticated)?; + if let Some(previous) = &self.host_key { + if super::recovery::host_key(previous)?.fingerprint() != key.fingerprint() { + return Err(Error::Config( + "admission changes its authenticated host key".into(), + )); + } + } + self.host_key = Some(format!("ssh-ed25519 {}", key.public_key_base64())); + Ok(()) + } +} + /// # Errors /// Rejects conflicting bootstrap mapping or unavailable quorum. pub async fn publish(kv: &CrowdbKvClient, registry: &NodeRegistry) -> Result<()> { @@ -61,6 +84,7 @@ pub async fn publish(kv: &CrowdbKvClient, registry: &NodeRegistry) -> Result<()> /// Rejects duplicate UUID endpoints, changed clusters and unavailable authority. pub async fn admit(kv: &CrowdbKvClient, cluster: &str, mut node: NodeRecord) -> Result { super::node_update::check_admission(kv, &node.discovery_id).await?; + let mut last_conflict = None; for attempt in 0..32 { let (mut registry, revision) = read(kv) .await? @@ -93,6 +117,15 @@ pub async fn admit(kv: &CrowdbKvClient, cluster: &str, mut node: NodeRecord) -> node.node_id = existing.node_id; if !existing.cancelled { node.operation_id = existing.operation_id.clone(); + if existing.host_key.is_some() + && node.host_key.is_some() + && existing.host_key != node.host_key + { + return Err(Error::Config( + "admission changes its authenticated host key".into(), + )); + } + node.host_key = existing.host_key.clone().or(node.host_key); } if existing.confirmed { if existing.rack_id != node.rack_id || existing.ssh_user != node.ssh_user { @@ -117,15 +150,16 @@ pub async fn admit(kv: &CrowdbKvClient, cluster: &str, mut node: NodeRecord) -> let bytes = serde_json::to_vec(®istry).map_err(config_error)?; match kv.put_cas(0, 0, NODES_KEY, &bytes, revision).await { Ok(_) => return Ok(node), - Err(KvError::CasFailed { .. } | KvError::OutcomeUnknown | KvError::CasBusy) => { + Err(conflict @ (KvError::CasFailed { .. } | KvError::OutcomeUnknown | KvError::CasBusy)) => { + last_conflict = Some((revision, conflict)); tokio::time::sleep(std::time::Duration::from_millis(5 + attempt * 2)).await; } Err(error) => return Err(error.into()), } } - Err(Error::Config( - "concurrent admission did not converge; retry".into(), - )) + Err(Error::Config(format!( + "concurrent admission did not converge; last conditional outcome: {last_conflict:?}" + ))) } fn cancellation_key(operation: &str) -> Vec { diff --git a/lib/crowdb-console-shared/src/deployment/services.rs b/lib/crowdb-console-shared/src/deployment/services.rs index 8f82ce9a5..92d87e041 100644 --- a/lib/crowdb-console-shared/src/deployment/services.rs +++ b/lib/crowdb-console-shared/src/deployment/services.rs @@ -12,6 +12,8 @@ use serde_json::Value; mod identity; mod planned_start; pub use planned_start::publish_planned_start; +mod readiness; +pub use readiness::diskdb_storage_ready; /// Commit a current service intent before asking its monitor to execute it. /// Identical retries retain the accepted operation identity. diff --git a/lib/crowdb-console-shared/src/deployment/services/readiness.rs b/lib/crowdb-console-shared/src/deployment/services/readiness.rs new file mode 100644 index 000000000..0758ae363 --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/services/readiness.rs @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::error::{Error, Result}; +use crowdb_kv_client::CrowdbKvClient; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +/// Confirm that loaded `DiskDB` group counts cover the registered online disks. +/// # Errors +/// Rejects unavailable registry/hardware reads and incomplete storage publication. +pub async fn diskdb_storage_ready(kv: Arc) -> Result<()> { + let registry = crowdb_kv_client::ServiceRegistryClient::from_shared(kv.clone()); + let hardware = crowdb_kv_client::HardwareClient::from_shared(kv); + let (instances, disks) = tokio::time::timeout(Duration::from_secs(3), async { + tokio::join!(registry.read_all_diskdb_instances(), hardware.list_all_disks()) + }) + .await + .map_err(|_| Error::Config("DiskDB readiness probe timed out".into()))?; + let instances = instances?; + let mut expected = BTreeMap::::new(); + for disk in disks? { + if matches!( + crowdb_protocol::common::HwStatus::try_from(disk.value.status), + Ok(crowdb_protocol::common::HwStatus::Init | crowdb_protocol::common::HwStatus::Offline) + ) { + continue; + } + *expected.entry(disk.disk_group_id).or_default() += 1; + } + for (group_id, disk_count) in expected { + let owners: Vec<_> = instances + .iter() + .filter_map(|(_, value)| { + let extra = value.extra.as_ref()?.diskdb.as_ref()?; + (!value.rpc_endpoint.is_empty() && extra.owned_dg_ids.contains(&group_id)).then_some(extra) + }) + .collect(); + if owners.len() != 1 { + return Err(Error::Config(format!( + "disk group {group_id} has no unique published DiskDB route" + ))); + } + if !owners[0].group_usages.iter().any(|usage| { + usage.disk_group_id == group_id && usage.usage_known && usage.disk_count >= disk_count + }) { + return Err(Error::Config(format!( + "disk group {group_id} has not loaded its {disk_count} registered disks" + ))); + } + } + Ok(()) +} diff --git a/lib/crowdb-console-shared/src/deployment/target.rs b/lib/crowdb-console-shared/src/deployment/target.rs new file mode 100644 index 000000000..19a60fb4a --- /dev/null +++ b/lib/crowdb-console-shared/src/deployment/target.rs @@ -0,0 +1,277 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Read-only target resolution, separate from runtime resource ownership. + +use std::time::Duration; + +use crowdb_protocol::mgmt::cluster::ClusterTargetIdentity; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RuntimeSelection { + pub runtime: String, + pub endpoint: String, + pub context: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RuntimeOwnership { + pub selection: RuntimeSelection, + pub owner: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClusterTargetBinding { + pub version: u32, + pub name: String, + pub cluster_id: String, + pub management_endpoints: Vec, + pub runtime: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum TargetError { + #[error("select a cluster binding or at least one management endpoint")] + Missing, + #[error("invalid cluster binding or management endpoint")] + Invalid, + #[error("management endpoint is unavailable or has no confirmed cluster identity")] + Unavailable, + #[error("management endpoints and selected binding disagree on cluster identity")] + IdentityConflict, + #[error("selected runtime endpoint/context differs from the cluster binding")] + RuntimeConflict, + #[error( + "native RPC endpoint is unreachable; publishing only HTTP ports does not expose native transport" + )] + RpcUnreachable, +} + +#[derive(Clone, Debug)] +pub struct ResolvedClusterTarget { + pub identity: ClusterTargetIdentity, + pub management_endpoints: Vec, + pub runtime: Option, +} + +/// Observe every selected endpoint before returning a workload target. +/// +/// # Errors +/// Rejects missing/malformed inputs, unreachable or mixed identities, stale +/// bindings and changed runtime selection. Never retries through another target. +pub async fn resolve( + binding: Option<&ClusterTargetBinding>, + explicit: &[String], + runtime: Option<&RuntimeSelection>, +) -> Result { + validate_binding(binding, runtime)?; + let selected = if explicit.is_empty() { + binding.map_or(&[][..], |binding| binding.management_endpoints.as_slice()) + } else { + explicit + }; + let management_endpoints = management_origins(selected)?; + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(3)) + .build() + .map_err(|_| TargetError::Unavailable)?; + // A single bounded observation pass; the slowest endpoint bounds the pass. + let responses = + futures::future::join_all(management_endpoints.iter().map(|origin| observe(&client, origin))).await; + let mut identity: Option = None; + for response in responses { + let current = response?; + if identity + .as_ref() + .is_some_and(|seen| seen.cluster_id != current.cluster_id) + || binding.is_some_and(|binding| binding.cluster_id != current.cluster_id) + { + return Err(TargetError::IdentityConflict); + } + if let Some(seen) = &mut identity { + seen.group0_management_endpoints + .extend(current.group0_management_endpoints); + seen.group0_rpc_endpoints.extend(current.group0_rpc_endpoints); + } else { + identity = Some(current); + } + } + let mut identity = identity.ok_or(TargetError::Missing)?; + identity.group0_management_endpoints.sort(); + identity.group0_management_endpoints.dedup(); + identity.group0_rpc_endpoints.sort(); + identity.group0_rpc_endpoints.dedup(); + Ok(ResolvedClusterTarget { + identity, + management_endpoints, + runtime: binding.and_then(|binding| binding.runtime.clone()), + }) +} + +impl ResolvedClusterTarget { + /// Confirm native transport resolves to the same committed application UUID. + /// + /// # Errors + /// Rejects unreachable authority, missing publication and a different native + /// cluster, even if its TCP listener accepts connections. + pub async fn verify_native_authority( + &self, + kv: &crowdb_kv_client::CrowdbKvClient, + ) -> Result<(), TargetError> { + use crowdb_kv_client::{GetOutcome, ReadMode}; + let result = tokio::time::timeout( + Duration::from_secs(3), + kv.get(0, 0, super::CLUSTER_KEY, ReadMode::Linearizable, None), + ) + .await + .map_err(|_| TargetError::Unavailable)? + .map_err(|_| TargetError::Unavailable)?; + let GetOutcome::Found { value, .. } = result else { + return Err(TargetError::Unavailable); + }; + if value.len() > 1024 * 1024 { + return Err(TargetError::Invalid); + } + let prepared: super::PreparedBootstrap = + serde_json::from_slice(&value).map_err(|_| TargetError::Invalid)?; + prepared.validate().map_err(|_| TargetError::Invalid)?; + if prepared.identity.cluster_id != self.identity.cluster_id { + return Err(TargetError::IdentityConflict); + } + Ok(()) + } + + /// Verify native transport reachability separately from HTTP-only attach. + /// + /// # Errors + /// Rejects an unreachable advertised RPC endpoint before workload writes. + pub async fn require_native_transport(&self) -> Result<(), TargetError> { + let results = futures::future::join_all(self.identity.group0_rpc_endpoints.iter().map(|endpoint| { + tokio::time::timeout(Duration::from_secs(2), tokio::net::TcpStream::connect(endpoint)) + })) + .await; + if results.is_empty() || results.into_iter().any(|result| !matches!(result, Ok(Ok(_)))) { + return Err(TargetError::RpcUnreachable); + } + Ok(()) + } +} + +fn validate_binding( + binding: Option<&ClusterTargetBinding>, + runtime: Option<&RuntimeSelection>, +) -> Result<(), TargetError> { + let Some(binding) = binding else { return Ok(()) }; + if binding.version != 1 + || !uuid(&binding.cluster_id) + || binding.name.is_empty() + || binding.name.len() > 48 + || !binding.name.as_bytes()[0].is_ascii_alphanumeric() + || !binding + .name + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"_.-".contains(&b)) + { + return Err(TargetError::Invalid); + } + management_origins(&binding.management_endpoints)?; + if let Some(ownership) = &binding.runtime { + if !uuid(&ownership.owner) + || ownership.selection.endpoint.is_empty() + || !matches!(ownership.selection.runtime.as_str(), "docker" | "containerd") + { + return Err(TargetError::Invalid); + } + if runtime.is_some_and(|runtime| runtime != &ownership.selection) { + return Err(TargetError::RuntimeConflict); + } + } + Ok(()) +} + +fn uuid(value: &str) -> bool { + Uuid::parse_str(value).is_ok_and(|id| !id.is_nil() && id.to_string() == value) +} + +/// Validate and canonicalize explicit console management origins. +/// +/// # Errors +/// Rejects empty, excessive, malformed or credential-bearing endpoints. +pub fn management_origins(values: &[String]) -> Result, TargetError> { + if values.is_empty() { + return Err(TargetError::Missing); + } + if values.len() > 16 { + return Err(TargetError::Invalid); + } + let mut result = Vec::new(); + for value in values { + let url = reqwest::Url::parse(value).map_err(|_| TargetError::Invalid)?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || url.path() != "/" + || url.query().is_some() + || url.fragment().is_some() + || !url.username().is_empty() + || url.password().is_some() + || url.port() == Some(0) + { + return Err(TargetError::Invalid); + } + result.push(url.as_str().trim_end_matches('/').to_owned()); + } + result.sort(); + result.dedup(); + Ok(result) +} + +async fn observe(client: &reqwest::Client, origin: &str) -> Result { + let mut response = client + .get(format!("{origin}/api/cluster/target")) + .send() + .await + .map_err(|_| TargetError::Unavailable)? + .error_for_status() + .map_err(|_| TargetError::Unavailable)?; + let mut body = Vec::new(); + while let Some(chunk) = response.chunk().await.map_err(|_| TargetError::Unavailable)? { + if body.len() + chunk.len() > 32768 { + return Err(TargetError::Invalid); + } + body.extend_from_slice(&chunk); + } + let mut identity: ClusterTargetIdentity = + serde_json::from_slice(&body).map_err(|_| TargetError::Invalid)?; + if identity.version != 1 + || !uuid(&identity.cluster_id) + || identity.group0_rpc_endpoints.is_empty() + || identity.group0_rpc_endpoints.len() > 16 + { + return Err(TargetError::Invalid); + } + identity.group0_management_endpoints = + management_origins(&identity.group0_management_endpoints).map_err(|_| TargetError::Invalid)?; + for endpoint in &identity.group0_rpc_endpoints { + // Use the URL parser for DNS and bracketed IPv6, while rejecting paths, + // credentials and omitted/zero ports on the native transport address. + let url = reqwest::Url::parse(&format!("tcp://{endpoint}")).map_err(|_| TargetError::Invalid)?; + if url.host_str().is_none() + || url.port().map_or(true, |port| port == 0) + || !url.path().is_empty() + || url.query().is_some() + || url.fragment().is_some() + || !url.username().is_empty() + || url.password().is_some() + { + return Err(TargetError::Invalid); + } + } + Ok(identity) +} diff --git a/lib/crowdb-console-shared/src/error.rs b/lib/crowdb-console-shared/src/error.rs index 097a45453..dafe300a3 100644 --- a/lib/crowdb-console-shared/src/error.rs +++ b/lib/crowdb-console-shared/src/error.rs @@ -29,6 +29,9 @@ pub enum Error { #[error("{kind} {id} already exists")] Conflict { kind: String, id: String }, + #[error("Waiting: {0}")] + DependenciesNotReady(String), + #[error("config error: {0}")] Config(String), diff --git a/lib/crowdb-console-shared/src/lifecycle.rs b/lib/crowdb-console-shared/src/lifecycle.rs index 3e661ffe1..586c4c0fd 100644 --- a/lib/crowdb-console-shared/src/lifecycle.rs +++ b/lib/crowdb-console-shared/src/lifecycle.rs @@ -121,10 +121,13 @@ mod restart; pub use restart::restart_local_service; mod native; pub use native::prepare_native_launch; +mod chunkdb; +pub use chunkdb::chunkdb_config; /// Inputs for a local `ChunkDB` deployment. #[derive(Debug, Clone)] pub struct ChunkdbDeployRequest { + pub local_deployment: bool, pub dynamic_ownership: bool, pub server_id: String, pub instance_id: u64, @@ -878,38 +881,6 @@ pub fn crowdb_diskio_bin() -> Option { find_in_path(std::ffi::OsStr::new("crowdb-diskio")) } -/// Locate the standalone `crowdb-rpc-fb-server` C++ binary (built via -/// `pixi run build-cpp`). Search order: `$CROWDB_RPC_FB_SERVER_BIN`, -/// `lib/crowdb-rpc/build/crowdb-rpc-fb-server` relative to the repo -/// root (walked up from the current exe), then `$PATH`. -pub fn crowdb_rpc_fb_server_bin() -> Option { - if let Ok(p) = std::env::var("CROWDB_RPC_FB_SERVER_BIN") { - return Some(PathBuf::from(p)); - } - // The fb-server is a CMake target under lib/crowdb-rpc/build/. - // Walk up from the exe dir to find the repo root, then check the - // expected build path. - if let Ok(exe) = std::env::current_exe() { - if let Some(dir) = exe.parent() { - let mut p = dir.to_path_buf(); - for _ in 0..5 { - let candidate = p.join("lib/crowdb-rpc/build/crowdb-rpc-fb-server"); - if is_executable(&candidate) { - return Some(candidate); - } - if !p.pop() { - break; - } - } - } - } - if let Some(path) = find_in_path(std::ffi::OsStr::new("crowdb-rpc-fb-server")) { - return Some(path); - } - warn!("crowdb-rpc-fb-server binary not found via env, repo build, or $PATH"); - None -} - /// Resolve the `--config` path for a diskdb deploy. Uses the /// pre-copied config at `/conf/crowdb_diskdb_config.toml`. /// Falls back to a minimal auto-generated config with all required @@ -1291,7 +1262,7 @@ pub async fn deploy_chunkdb_local( "ChunkDB", ) .await?; - std::mem::forget(child); + tokio::spawn(reap_child(child, pid, "crowdb-chunkdb")); Ok(DeployedDiskdb { server_id: req.server_id.clone(), endpoint, @@ -1310,46 +1281,6 @@ pub async fn deploy_chunkdb_local( }) } -/// Render the node-specific `ChunkDB` configuration after deployment input validation. -#[must_use] -pub fn chunkdb_config(req: &ChunkdbDeployRequest, node: &NodeEntry) -> String { - let seeds = req - .kv_server_mgmt_seeds - .iter() - .map(|seed| format!("{seed:?}")) - .collect::>() - .join(", "); - let deployment_mode = if req.allow_unsafe_ec { - "test_unsafe_placement" - } else { - "production" - }; - let placement_mode = if req.allow_unsafe_ec { - "unsafe_colocated" - } else { - "protected" - }; - format!( - "service_ownership = \"{}\"\n\n[deployment]\nmode = \"{}\"\n\n[server]\nrpc_workers = {}\nhttp_listen_addr = \"{}:{}\"\nrpc_listen_addr = \"{}:{}\"\ninstance_id = \"{}\"\nkv_server_mgmt_seeds = [{}]\nkeepalive_interval_secs = 1\nkv_pool_size = {}\nkv_rpc_workers = {}\ndiskdb_pool_size = {}\ndiskdb_rpc_workers = {}\n\n[topology]\nrefresh_interval_secs = 1\n\n[range_guard]\nallow_all_when_empty = false\n\n[lifecycle]\ncache_capacity = 10000\nsweep_chunk_lock_interval_secs = 60\nlock_hold_warn_threshold_ms = 1000\n\n[placement]\nmode = \"{}\"\nallow_unsafe_ec = {}\nallow_degraded_failure_domains = {}\n", - if req.dynamic_ownership { "dynamic" } else { "fixed" }, - deployment_mode, - req.rpc_workers.unwrap_or(2), - node.host, - req.http_port, - node.host, - req.rpc_port, - req.instance_id, - seeds, - req.kv_connections.unwrap_or(1), - req.kv_client_rpc_workers.unwrap_or(2), - req.diskdb_connections.unwrap_or(1), - req.diskdb_client_rpc_workers.unwrap_or(2), - placement_mode, - req.allow_unsafe_ec, - req.allow_unsafe_ec, - ) -} - fn chunkdb_launch_args(req: &ChunkdbDeployRequest, config_path: &Path, log_dir: &Path) -> Vec { let mut args = vec![ "--config".into(), @@ -1425,7 +1356,7 @@ pub async fn deploy_diskio_local( ), }); } - std::mem::forget(child); + tokio::spawn(reap_child(child, pid, "crowdb-diskio")); Ok(DeployedDiskdb { server_id: req.server_id.clone(), endpoint, diff --git a/lib/crowdb-console-shared/src/lifecycle/chunkdb.rs b/lib/crowdb-console-shared/src/lifecycle/chunkdb.rs new file mode 100644 index 000000000..41fd2799e --- /dev/null +++ b/lib/crowdb-console-shared/src/lifecycle/chunkdb.rs @@ -0,0 +1,57 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::ChunkdbDeployRequest; +use crate::config::NodeEntry; + +/// Render the node-specific `ChunkDB` configuration after deployment input validation. +#[must_use] +pub fn chunkdb_config(req: &ChunkdbDeployRequest, node: &NodeEntry) -> String { + let seeds = req + .kv_server_mgmt_seeds + .iter() + .map(|seed| format!("{seed:?}")) + .collect::>() + .join(", "); + let deployment_mode = if req.local_deployment { + "local" + } else if req.allow_unsafe_ec { + "test_unsafe_placement" + } else { + "production" + }; + let placement_mode = if req.local_deployment || req.allow_unsafe_ec { + "unsafe_colocated" + } else { + "protected" + }; + let deployment_budget = if req.local_deployment { + "max_node_failures = 0\n" + } else { + "" + }; + let mut configuration = format!( + "service_ownership = \"{}\"\n\n[deployment]\nmode = \"{}\"\n{}\n[server]\nrpc_workers = {}\nhttp_listen_addr = \"{}:{}\"\nrpc_listen_addr = \"{}:{}\"\ninstance_id = \"{}\"\nkv_server_mgmt_seeds = [{}]\nkeepalive_interval_secs = 1\nkv_pool_size = {}\nkv_rpc_workers = {}\ndiskdb_pool_size = {}\ndiskdb_rpc_workers = {}\n\n[topology]\nrefresh_interval_secs = 1\n\n[range_guard]\nallow_all_when_empty = false\n\n[lifecycle]\ncache_capacity = 10000\nsweep_chunk_lock_interval_secs = 60\nlock_hold_warn_threshold_ms = 1000\n\n[placement]\nmode = \"{}\"\nallow_unsafe_ec = {}\nallow_degraded_failure_domains = {}\n", + if req.dynamic_ownership { "dynamic" } else { "fixed" }, + deployment_mode, + deployment_budget, + req.rpc_workers.unwrap_or(2), + node.host, + req.http_port, + node.host, + req.rpc_port, + req.instance_id, + seeds, + req.kv_connections.unwrap_or(1), + req.kv_client_rpc_workers.unwrap_or(2), + req.diskdb_connections.unwrap_or(1), + req.diskdb_client_rpc_workers.unwrap_or(2), + placement_mode, + req.allow_unsafe_ec && !req.local_deployment, + req.allow_unsafe_ec || req.local_deployment, + ); + if req.local_deployment { + configuration.push_str("\n[conversion]\nenabled = false\n"); + } + configuration +} diff --git a/lib/crowdb-console-shared/src/lifecycle/restart.rs b/lib/crowdb-console-shared/src/lifecycle/restart.rs index b94a353ee..6612c6f00 100644 --- a/lib/crowdb-console-shared/src/lifecycle/restart.rs +++ b/lib/crowdb-console-shared/src/lifecycle/restart.rs @@ -71,7 +71,7 @@ pub async fn restart_local_service(server_id: &str, pid: u32, spec: &LocalLaunch }); } } - std::mem::forget(child); + tokio::spawn(super::reap_child(child, new_pid, "managed auxiliary service")); Ok(new_pid) } diff --git a/lib/crowdb-console-shared/src/mgmt.rs b/lib/crowdb-console-shared/src/mgmt.rs index 15c9c6fcc..219f2f32f 100644 --- a/lib/crowdb-console-shared/src/mgmt.rs +++ b/lib/crowdb-console-shared/src/mgmt.rs @@ -70,6 +70,19 @@ impl ServerClient { self.post_empty(&format!("/stores/{sid}/groups"), req).await } + /// Read durable creation ownership independently of current runtime health. + /// + /// # Errors + /// Rejects unavailable, absent or malformed ownership observations. + pub async fn group_creation( + &self, + sid: u64, + gid: u64, + ) -> Result { + self.get_json(&format!("/stores/{sid}/groups/{gid}/creation")) + .await + } + /// Import a new replica's snapshot before it participates in voting. /// /// # Errors diff --git a/lib/crowdb-console-shared/src/ops/cluster.rs b/lib/crowdb-console-shared/src/ops/cluster.rs index 87f9cfe29..57ebefe80 100644 --- a/lib/crowdb-console-shared/src/ops/cluster.rs +++ b/lib/crowdb-console-shared/src/ops/cluster.rs @@ -28,6 +28,7 @@ use crate::ops::hardware::{self, AddDiskInput}; use crate::ops::OpContext; mod bootstrap; +pub mod readiness; mod storage_readiness; #[cfg(feature = "test-util")] pub use bootstrap::publish_topology_for_tests; @@ -625,7 +626,7 @@ async fn local_deploy_diskio( .insert(server_id.clone(), deployed.launch.clone()); expected_owners.insert(30_000 + node.id, (deployed.endpoint.clone(), node.id * 100 + 1)); ctx.config_mut().add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: server_id, url: deployed.endpoint.clone(), node_id: Some(node.id), @@ -723,6 +724,7 @@ pub async fn local_deploy_chunkdb( .join(format!("node{}", node.id)) .join(&server_id); let request = ChunkdbDeployRequest { + local_deployment: false, dynamic_ownership: config.dynamic_ownership, server_id: server_id.clone(), instance_id, @@ -750,7 +752,7 @@ pub async fn local_deploy_chunkdb( .local_launches .insert(server_id.clone(), deployed.launch.clone()); ctx.config_mut().add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: server_id, url: deployed.endpoint.clone(), node_id: Some(node.id), @@ -1095,7 +1097,7 @@ async fn deploy_diskdb_instances( .local_launches .insert(server_id.clone(), deployed.launch.clone()); ctx.config_mut().add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: server_id, url: deployed.endpoint.clone(), node_id: Some(node.id), @@ -1407,7 +1409,7 @@ async fn deploy_servers( let mut cfg = ctx.config_mut(); cfg.add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: nid.to_string(), url: deployed.mgmt_url.clone(), node_id: Some(*nid), @@ -1425,163 +1427,3 @@ async fn deploy_servers( } Ok(()) } - -// ── RPC fb-server deploy ────────────────────────────────────────── - -/// Summary of a completed RPC fb-server deploy. -#[derive(Debug, Clone, serde::Serialize)] -pub struct RpcDeploySummary { - pub port: u16, - pub pid: u32, - pub io_engines: u32, - pub io_workers: u32, - pub nagle: bool, -} - -/// Configuration for [`local_deploy_rpc`]. -#[derive(Debug, Clone)] -pub struct RpcDeployConfig { - /// Listen port. 0 = auto-allocate. - pub port: u16, - pub io_engines: u32, - pub io_workers: u32, - pub enable_nagle: bool, - pub metrics_interval: u32, -} - -impl Default for RpcDeployConfig { - fn default() -> Self { - Self { - port: 0, - io_engines: 1, - io_workers: 1, - enable_nagle: false, - metrics_interval: 2, - } - } -} - -/// Deploy a standalone `crowdb-rpc-fb-server` (C++ echo server) on -/// `127.0.0.1`. Spawns the process, waits for the `listening port=` -/// readiness line on stdout, and records the PID + port in the config -/// so `cluster destroy` can stop it. -/// -/// Unlike [`local_deploy`], this does not create rack/node entries or -/// bootstrap group 0 — the fb-server is a raw RPC echo endpoint with -/// no KV layer. -/// -/// # Errors -/// Returns [`Error::NotFound`] if the binary cannot be located. -/// Returns [`Error::Io`] on spawn or readiness failures. -pub async fn local_deploy_rpc( - ctx: &OpContext, - cfg: &RpcDeployConfig, - workspace_dir: Option<&std::path::Path>, -) -> Result { - let bin = lifecycle::crowdb_rpc_fb_server_bin().ok_or_else(|| Error::NotFound { - kind: "binary".into(), - id: "crowdb-rpc-fb-server".into(), - })?; - if !bin.exists() { - return Err(Error::NotFound { - kind: "binary".into(), - id: bin.display().to_string(), - }); - } - - let workspace = workspace_dir.map_or_else(default_workspace, std::path::PathBuf::from); - - let port = if cfg.port == 0 { - alloc_workspace_ports(&workspace, ServicePort::KvServerListen, 0, 1)?[0] - } else { - cfg.port - }; - let log_dir = workspace.join("log"); - std::fs::create_dir_all(&log_dir)?; - - let mut cmd = std::process::Command::new(&bin); - cmd.arg(format!("--port={port}")) - .arg(format!("--io_engines={}", cfg.io_engines)) - .arg(format!("--io_workers={}", cfg.io_workers)) - .arg(format!("--logdir={}", log_dir.display())) - .arg(format!("--metrics_interval={}", cfg.metrics_interval)); - if cfg.enable_nagle { - cmd.arg("--enable_nagle"); - } - // Redirect stdout/stderr to a log file so the child doesn't get - // SIGPIPE when the CLI exits. The fb-server also writes structured - // logs to --logdir, but stdout has the "listening port=" line. - let stdout_path = log_dir.join("fb-server.stdout.log"); - let stdout = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&stdout_path) - .map_err(Error::Io)?; - let stderr = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(stdout_path) - .map_err(Error::Io)?; - cmd.stdout(std::process::Stdio::from(stdout)) - .stderr(std::process::Stdio::from(stderr)); - - let child = cmd.spawn()?; - - let pid = child.id(); - let ready = wait_for_rpc_ready(port, pid, 10).await; - - if !ready { - let _ = lifecycle::stop_pid(pid); - return Err(Error::Io(std::io::Error::other(format!( - "crowdb-rpc-fb-server did not bind on port {port} within 10s" - )))); - } - - // Record in config so reset() can stop it. - let server_id = format!("rpc-fb-{pid}"); - let mut config = ctx.config_mut(); - config.add_server(ServerEntry { - dataset_origin: None, - id: server_id.clone(), - url: format!("http://127.0.0.1:{port}"), - node_id: None, - rpc_url: Some(format!("http://127.0.0.1:{port}")), - rest_port: None, - rpc_port: Some(port), - auto_start: false, - binary: Some(bin.display().to_string()), - election_profile: None, - pid: Some(pid), - service_type: ServiceType::Rpc, - rpc_workers: None, - no_fsync: false, - })?; - - Ok(RpcDeploySummary { - port, - pid, - io_engines: cfg.io_engines, - io_workers: cfg.io_workers, - nagle: cfg.enable_nagle, - }) -} - -/// Poll-connect to `127.0.0.1:port` until success or timeout. Drops -/// the stdout pipe (the fb-server writes logs to `--logdir`, not stdout -/// — stdout is only used for the `listening port=` line which we don't -/// need since we know the port). -async fn wait_for_rpc_ready(port: u16, pid: u32, timeout_secs: u64) -> bool { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(timeout_secs); - loop { - if std::time::Instant::now() >= deadline { - return false; - } - if !lifecycle::process_is_alive(pid) { - return false; - } - if std::net::TcpStream::connect(format!("127.0.0.1:{port}")).is_ok() { - return true; - } - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - } -} diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness.rs new file mode 100644 index 000000000..767ccd3de --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness.rs @@ -0,0 +1,243 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Reduce current dependency evidence without performing repair or deployment. + +use crowdb_protocol::mgmt::cluster::{ + ClusterObservation, ClusterPhase, ClusterStatus, DependencyKind, DependencyObservation, DependencyState, + ReadinessReason, +}; + +mod collect; +pub use collect::collect; +mod clock; +pub use clock::reduce_current; +mod member; +pub use member::member_evidence; + +const REQUIRED: [DependencyKind; 6] = [ + DependencyKind::Authority, + DependencyKind::Metadata, + DependencyKind::Storage, + DependencyKind::Member, + DependencyKind::Service, + DependencyKind::Protocol, +]; + +#[derive(Default)] +struct Evidence { + states: std::collections::BTreeMap>, + reasons: Vec, +} + +/// Determine whole-cluster readiness from a bounded observational snapshot. +#[must_use] +pub fn reduce(mut observation: ClusterObservation, now_unix_ms: u64) -> ClusterStatus { + observation.dependencies.sort(); + let mut evidence = collect_evidence(&observation, now_unix_ms); + for kind in REQUIRED.into_iter().filter(|kind| required(&observation, *kind)) { + if !evidence.states.contains_key(&kind) { + evidence.reasons.push(reason( + &format!("{kind:?}"), + "observation-missing", + "Required application evidence is missing", + )); + } + } + if invalid(&observation) { + evidence + .states + .entry(DependencyKind::Identity) + .or_default() + .push(DependencyState::Failed); + evidence.reasons.push(reason( + "cluster", + "invalid-observation", + "Unsupported cluster observation version, identity or initial voter policy", + )); + } + let phase = classify(&observation, &evidence.states); + evidence.reasons.sort(); + evidence.reasons.dedup(); + ClusterStatus { + phase, + reasons: evidence.reasons, + observation, + } +} + +fn collect_evidence(observation: &ClusterObservation, now_unix_ms: u64) -> Evidence { + let mut evidence = Evidence::default(); + let mut identities = std::collections::BTreeSet::new(); + for fact in &observation.dependencies { + if fact.operation_id != fact.current_operation_id { + let (code, explanation) = match (&fact.operation_id, &fact.current_operation_id) { + (None, Some(_)) => ( + "operation-observation-missing", + "No executed-operation observation is available", + ), + (Some(_), None) => ( + "current-operation-unavailable", + "Current committed operation is unavailable", + ), + _ => ( + "operation-superseded", + "Historical operation does not describe current readiness", + ), + }; + evidence.reasons.push(reason( + &fact.id, + code, + &format!( + "{explanation}; observed operation {}; current operation {}; {}", + fact.operation_id.as_deref().unwrap_or("none"), + fact.current_operation_id.as_deref().unwrap_or("none"), + fact.detail + ), + )); + let replacement = observation.dependencies.iter().any(|current| { + current.id == fact.id + && current.kind == fact.kind + && current.operation_id == fact.current_operation_id + && current.operation_id == current.current_operation_id + }); + if !replacement && fact.required && fact.kind != DependencyKind::Runtime { + evidence + .states + .entry(fact.kind) + .or_default() + .push(DependencyState::Unknown); + } + continue; + } + if !identities.insert(&fact.id) { + evidence + .states + .entry(DependencyKind::Identity) + .or_default() + .push(DependencyState::Failed); + evidence.reasons.push(reason( + &fact.id, + "conflicting-observations", + "Duplicate dependency identity", + )); + } + let state = current_state(fact, now_unix_ms); + if state != DependencyState::Ready { + evidence.reasons.push(observation_reason(fact, state)); + } + // Runtime facts never gate application readiness, including remote unknowns. + if fact.kind != DependencyKind::Runtime + && (fact.required || (fact.kind == DependencyKind::Identity && state == DependencyState::Failed)) + { + evidence.states.entry(fact.kind).or_default().push(state); + } + } + evidence +} + +fn classify( + observation: &ClusterObservation, + states: &std::collections::BTreeMap>, +) -> ClusterPhase { + let contains = |kind, state| states.get(&kind).is_some_and(|states| states.contains(&state)); + if states + .values() + .flatten() + .any(|state| *state == DependencyState::Failed) + { + ClusterPhase::Failed + } else if observation.bound && !contains(DependencyKind::Authority, DependencyState::Ready) { + ClusterPhase::Unavailable + } else if observation.bound + && REQUIRED + .iter() + .filter(|kind| required(observation, **kind)) + .all(|kind| states.contains_key(kind)) + && states + .values() + .flatten() + .all(|state| *state == DependencyState::Ready) + { + ClusterPhase::Ready + } else if observation.setup_completed + || states + .values() + .flatten() + .any(|state| *state == DependencyState::Stopped) + || (observation.bound + && states + .get(&DependencyKind::Member) + .is_some_and(|states| states.iter().any(|state| *state != DependencyState::Ready))) + { + ClusterPhase::Degraded + } else if observation.work_accepted { + ClusterPhase::Initializing + } else { + ClusterPhase::Waiting + } +} + +fn required(observation: &ClusterObservation, kind: DependencyKind) -> bool { + use crowdb_protocol::mgmt::node::setup::{SetupServiceKind, SetupStorage}; + + let Some(policy) = &observation.policy else { + return true; + }; + match kind { + DependencyKind::Storage => !matches!(policy.setup.storage, SetupStorage::Diskless), + DependencyKind::Protocol => policy + .setup + .services + .iter() + .any(|service| service.enabled && service.kind == SetupServiceKind::AccessServer), + _ => true, + } +} + +fn invalid(observation: &ClusterObservation) -> bool { + observation.version != 1 + || !matches!(observation.initial_voters, 1 | 3) + || (observation.bound && observation.cluster_id.is_none()) + || observation.cluster_id.as_ref().is_some_and(|identity| { + uuid::Uuid::parse_str(identity).map_or(true, |identity| identity.is_nil()) + }) + || observation.policy.as_ref().is_some_and(|policy| { + policy.validate().is_err() + || policy.initial_voters != observation.initial_voters + || policy.cluster != observation.cluster + }) +} + +fn observation_reason(fact: &DependencyObservation, state: DependencyState) -> ReadinessReason { + if state == fact.state { + fact.reason + .clone() + .unwrap_or_else(|| reason(&fact.id, "not-ready", "Dependency is not ready")) + } else { + reason( + &fact.id, + "observation-not-current", + "Dependency observation is stale, superseded or has a future timestamp", + ) + } +} + +fn current_state(fact: &DependencyObservation, now_unix_ms: u64) -> DependencyState { + let fresh = now_unix_ms + .checked_sub(fact.observed_at_unix_ms) + .is_some_and(|age| age <= fact.max_age_ms); + if fresh && fact.operation_id == fact.current_operation_id { + fact.state + } else { + DependencyState::Unknown + } +} + +fn reason(dependency: &str, code: &str, detail: &str) -> ReadinessReason { + ReadinessReason { + dependency: dependency.into(), + code: code.into(), + detail: detail.into(), + } +} diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/clock.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/clock.rs new file mode 100644 index 000000000..f30e25c2a --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/clock.rs @@ -0,0 +1,42 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Never admit evidence before its timestamp, including small peer clock offsets. + +use crowdb_protocol::mgmt::cluster::{ClusterObservation, ClusterStatus, DependencyKind}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::time::Instant; + +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |value| u64::try_from(value.as_millis()).unwrap_or(u64::MAX)) +} + +/// Reduce evidence after a bounded wait for a slightly faster peer clock. +/// +/// Timestamps and freshness limits are unchanged. Waiting consumes the caller's +/// existing deadline and cannot exceed any future fact's freshness horizon. +/// Larger offsets remain unknown immediately; elapsed evidence stays stale. +pub async fn reduce_current(observation: ClusterObservation, deadline: Instant) -> ClusterStatus { + let now = now_ms(); + let mut wait_ms = 0; + for fact in &observation.dependencies { + if fact.kind == DependencyKind::Runtime || fact.operation_id != fact.current_operation_id { + continue; + } + let ahead = fact.observed_at_unix_ms.saturating_sub(now); + if ahead > fact.max_age_ms.min(3000) { + return super::reduce(observation, now); + } + wait_ms = wait_ms.max(ahead); + } + if wait_ms > 0 { + if let Some(until) = Instant::now().checked_add(Duration::from_millis(wait_ms)) { + if until < deadline { + tokio::time::sleep_until(until).await; + } + } + } + super::reduce(observation, now_ms()) +} diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs new file mode 100644 index 000000000..3105a2587 --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect.rs @@ -0,0 +1,222 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded, read-only application evidence; no runtime access or repair. + +mod members; +mod storage; +use crate::{ + deployment::{self, registry, services, PreparedBootstrap}, + error::{Error, Result}, + ops::OpContext, +}; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_protocol::mgmt::{ + cluster::{ + ClusterObservation, ClusterStatus, DependencyKind, DependencyObservation, DependencyState, + ReadinessReason, + }, + node::observation::NodeRuntimeObservation, +}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::time::Instant; + +/// Observe one selected cluster without reserving or repairing anything. +/// +/// # Errors +/// Rejects malformed local policy/identity. Dependency failures produce a valid +/// unhealthy report, with unknown runtime facts when no runtime was supplied. +pub async fn collect(ctx: &OpContext, local: NodeRuntimeObservation) -> Result { + local.policy.validate().map_err(|_| invalid())?; + if local.version != 1 || uuid::Uuid::parse_str(&local.discovery_id).is_err() { + return Err(invalid()); + } + let deadline = Instant::now() + Duration::from_secs(6); + let mut observation = base(&local); + let authority = tokio::time::timeout_at( + deadline.min(Instant::now() + Duration::from_secs(2)), + authority(ctx), + ) + .await; + match authority { + Ok(Ok(bootstrap)) => { + if local + .binding + .as_ref() + .is_some_and(|binding| binding.bootstrap != bootstrap.identity) + { + observation.dependencies.push(fact( + "cluster/identity", + DependencyKind::Identity, + DependencyState::Failed, + "cluster-identity-conflict", + )); + } else { + observation.cluster_id = Some(bootstrap.identity.cluster_id.clone()); + observation.operation_id = Some(bootstrap.identity.operation_id.clone()); + observation.dependencies.push(fact( + "cluster/authority", + DependencyKind::Authority, + DependencyState::Ready, + "authority-observed", + )); + Box::pin(gather(ctx, &bootstrap, &local, &mut observation, deadline)).await; + } + } + _ => observation.dependencies.push(fact( + "cluster/authority", + DependencyKind::Authority, + DependencyState::Unknown, + "authority-unavailable", + )), + } + observation.observed_at_unix_ms = now_ms(); + Ok(super::reduce_current(observation, deadline).await) +} + +fn base(local: &NodeRuntimeObservation) -> ClusterObservation { + ClusterObservation { + version: 1, + cluster: local.policy.cluster.clone(), + cluster_id: local + .binding + .as_ref() + .map(|binding| binding.bootstrap.cluster_id.clone()), + bound: local.binding.is_some(), + initial_voters: local.policy.initial_voters, + policy: Some(local.policy.clone()), + observed_at_unix_ms: now_ms(), + members: Vec::new(), + operation_id: local.setup.as_ref().map(|setup| setup.operation_id.clone()), + work_accepted: local.binding.is_some() || local.setup.is_some(), + setup_completed: local + .setup + .as_ref() + .is_some_and(|setup| setup.phase == crowdb_protocol::mgmt::node::setup::NodeSetupPhase::Ready), + dependencies: vec![fact( + "runtime", + DependencyKind::Runtime, + DependencyState::Unknown, + "runtime-not-observed", + )], + runtime: None, + } +} + +async fn authority(ctx: &OpContext) -> Result { + let GetOutcome::Found { value, .. } = ctx + .kv() + .get(0, 0, deployment::CLUSTER_KEY, ReadMode::Linearizable, None) + .await? + else { + return Err(invalid()); + }; + if value.len() > 1024 * 1024 { + return Err(invalid()); + } + let bootstrap: PreparedBootstrap = serde_json::from_slice(&value).map_err(|_| invalid())?; + bootstrap.validate()?; + Ok(bootstrap) +} + +async fn gather( + ctx: &OpContext, + bootstrap: &PreparedBootstrap, + local: &NodeRuntimeObservation, + observation: &mut ClusterObservation, + deadline: Instant, +) { + let (nodes, intents, metadata, storage) = tokio::join!( + tokio::time::timeout_at(deadline, registry::read(ctx.kv())), + tokio::time::timeout_at(deadline, services::list(ctx.kv())), + tokio::time::timeout_at(deadline, storage::metadata(ctx, bootstrap)), + tokio::time::timeout_at(deadline, storage::read(ctx, bootstrap)), + ); + observation + .dependencies + .push(metadata.ok().and_then(Result::ok).unwrap_or_else(|| { + fact( + "cluster/metadata", + DependencyKind::Metadata, + DependencyState::Unknown, + "metadata-unavailable", + ) + })); + let storage = storage.ok().and_then(Result::ok); + let Ok(Ok(Some((registry, _)))) = nodes else { + append_storage(observation, storage.as_ref(), &std::collections::BTreeMap::new()); + return; + }; + if registry.cluster_id != bootstrap.identity.cluster_id || registry.nodes.len() > 4096 { + observation.dependencies.push(fact( + "cluster/registry", + DependencyKind::Identity, + DependencyState::Failed, + "registry-identity-conflict", + )); + return; + } + let member_nodes = members::read( + bootstrap, + local, + observation, + registry, + intents.ok().and_then(Result::ok), + deadline, + ) + .await; + append_storage(observation, storage.as_ref(), &member_nodes); +} + +fn append_storage( + observation: &mut ClusterObservation, + storage: Option<&storage::StorageSnapshot>, + nodes: &std::collections::BTreeMap, +) { + observation.dependencies.extend(storage.map_or_else( + || { + vec![fact( + "cluster/storage", + DependencyKind::Storage, + DependencyState::Unknown, + "storage-unavailable", + )] + }, + |snapshot| snapshot.observe(nodes), + )); +} + +pub(super) fn fact( + id: &str, + kind: DependencyKind, + state: DependencyState, + code: &str, +) -> DependencyObservation { + DependencyObservation { + id: id.into(), + kind, + required: true, + state, + detail: format!("{id}: {code}"), + observed_at_unix_ms: now_ms(), + max_age_ms: 3000, + revision: None, + operation_id: None, + current_operation_id: None, + reason: (state != DependencyState::Ready).then(|| ReadinessReason { + dependency: id.into(), + code: code.into(), + detail: "Required current application evidence is unavailable or conflicts with accepted state" + .into(), + }), + } +} + +pub(super) fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |value| u64::try_from(value.as_millis()).unwrap_or(u64::MAX)) +} +fn invalid() -> Error { + Error::Config("invalid cluster readiness identity or policy".into()) +} diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs new file mode 100644 index 000000000..e94cc2721 --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/members.rs @@ -0,0 +1,126 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Each confirmed member supplies bounded current application evidence. + +use super::{fact, now_ms}; +use crate::deployment::{registry, PreparedBootstrap}; +use crowdb_protocol::mgmt::{ + cluster::{ClusterMemberObservation, ClusterObservation, DependencyKind, DependencyState}, + node::{observation::NodeRuntimeObservation, NodeServiceIntent}, +}; +use futures::StreamExt; +use std::collections::BTreeMap; +use tokio::time::Instant; + +pub(super) async fn read( + bootstrap: &PreparedBootstrap, + local: &NodeRuntimeObservation, + observation: &mut ClusterObservation, + registry: registry::NodeRegistry, + intents: Option>, + deadline: Instant, +) -> BTreeMap { + let mut verified = BTreeMap::new(); + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .build(); + let Ok(client) = client else { + return verified; + }; + let expected: Vec<_> = registry + .nodes + .into_iter() + .filter(|node| node.confirmed && !node.cancelled) + .collect(); + for member in bootstrap.intent.members() { + if !expected.iter().any(|node| node.node_id == *member) { + observation.dependencies.push(fact( + &format!("node/{member}"), + DependencyKind::Member, + DependencyState::Unknown, + "confirmed-member-missing", + )); + } + } + let replies: Vec<_> = futures::stream::iter(expected.into_iter().map(|node| { + let client = &client; + async move { + let observed = if node.discovery_id == local.discovery_id { + Some(local.clone()) + } else { + tokio::time::timeout_at(deadline, member(client, &node)) + .await + .ok() + .flatten() + }; + (node, observed) + } + })) + .buffer_unordered(8) + .collect() + .await; + for (node, observed) in replies { + let host = node.host.trim_matches(['[', ']']); + observation.members.push(ClusterMemberObservation { + discovery_id: node.discovery_id.clone(), + node_id: Some(node.node_id), + management_endpoints: vec![format!( + "http://{}:9090", + if host.contains(':') { + format!("[{host}]") + } else { + host.into() + } + )], + }); + let now = now_ms(); + let facts = super::super::member_evidence( + bootstrap, + &node, + &local.policy, + intents.as_deref(), + observed.as_ref(), + now, + ); + if facts + .first() + .is_some_and(|fact| fact.kind == DependencyKind::Member && fact.state == DependencyState::Ready) + && observed.as_ref().is_some_and(|value| { + value.observed_at_unix_ms <= now && now - value.observed_at_unix_ms <= 3000 + }) + { + if let Some(value) = observed { + verified.insert(node.node_id, value); + } + } + observation.dependencies.extend(facts); + } + observation.members.sort_by_key(|member| member.node_id); + verified +} + +async fn member(client: &reqwest::Client, node: ®istry::NodeRecord) -> Option { + let host = node.host.trim_matches(['[', ']']); + let host = if host.contains(':') { + format!("[{host}]") + } else { + host.into() + }; + let mut reply = client + .get(format!("http://{host}:9090/api/cluster/node-observation")) + .send() + .await + .ok()? + .error_for_status() + .ok()?; + let mut bytes = Vec::new(); + while let Some(chunk) = reply.chunk().await.ok()? { + if bytes.len() + chunk.len() > 256 * 1024 { + return None; + } + bytes.extend_from_slice(&chunk); + } + serde_json::from_slice(&bytes).ok() +} diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage.rs new file mode 100644 index 000000000..cdad0557a --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage.rs @@ -0,0 +1,180 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Verify accepted preparation, physical placement, ownership and capacity. + +mod checks; +mod supplied; + +use super::{fact, now_ms}; +use crate::{ + deployment::{preparation, PreparedBootstrap}, + error::{Error, Result}, + ops::OpContext, +}; +use crowdb_kv_client::{DiskRecord, HardwareClient, ReadMode, ServiceRegistryClient}; +use crowdb_protocol::mgmt::cluster::{DependencyKind, DependencyObservation, DependencyState}; +use crowdb_protocol::{ + common::InstanceValue, + mgmt::node::observation::NodeRuntimeObservation, + sysdata::{DiskGroupEntry, DiskdbOwnerEntry, KVGroupBindEntry}, +}; +use std::collections::BTreeMap; +use std::sync::Arc; + +pub(super) async fn metadata( + ctx: &OpContext, + bootstrap: &PreparedBootstrap, +) -> Result { + let accepted = preparation::metadata_binding(ctx, &bootstrap.identity).await?; + let replicas = ctx + .sysmd() + .list_replicas_in_group(accepted.store_id, accepted.group_id) + .await?; + let exact = replicas.len() == accepted.replicas.len() + && accepted.replicas.iter().all(|expected| { + replicas.iter().any(|replica| { + replica.node_id == expected.node_id + && replica.replica_id == expected.replica_id + && replica.voting + }) + }); + if !exact { + return Ok(fact( + "cluster/metadata", + DependencyKind::Metadata, + DependencyState::Unknown, + "metadata-membership-differs", + )); + } + // A linearizable ordinary-group read proves current authority beyond Group 0. + let _ = ctx + .kv() + .get( + accepted.store_id, + accepted.group_id, + b"/deployment/readiness-probe", + ReadMode::Linearizable, + None, + ) + .await?; + Ok(fact( + "cluster/metadata", + DependencyKind::Metadata, + DependencyState::Ready, + "metadata-ready", + )) +} + +pub(super) async fn read(ctx: &OpContext, bootstrap: &PreparedBootstrap) -> Result { + let (accepted, revision) = preparation::read(ctx, &bootstrap.identity) + .await? + .ok_or_else(|| Error::Config("default preparation is absent".into()))?; + let hardware = HardwareClient::from_shared(Arc::clone(ctx.kv_arc())); + let services = ServiceRegistryClient::from_shared(Arc::clone(ctx.kv_arc())); + let (groups, disks, owners, binds, instances) = tokio::join!( + hardware.list_disk_groups(), + hardware.list_all_disks(), + hardware.list_owners(), + hardware.list_binds(), + services.read_all_diskdb_instances() + ); + Ok(StorageSnapshot { + accepted, + revision, + groups, + disks, + owners, + binds, + instances, + }) +} + +pub(super) struct StorageSnapshot { + accepted: preparation::DefaultPreparation, + revision: u64, + groups: crowdb_kv_client::Result>, + disks: crowdb_kv_client::Result>, + owners: crowdb_kv_client::Result>, + binds: crowdb_kv_client::Result>, + instances: crowdb_kv_client::Result>, +} + +impl StorageSnapshot { + pub(super) fn observe( + &self, + nodes: &BTreeMap, + ) -> Vec { + if self.accepted.retired { + return vec![fact( + "cluster/storage", + DependencyKind::Storage, + DependencyState::Stopped, + "preparation-retired", + )]; + } + let mut evidence = Vec::new(); + let now = now_ms(); + for expected in &self.accepted.storage { + if nodes.get(&expected.node_id).is_some_and(|node| { + matches!( + node.policy.setup.storage, + crowdb_protocol::mgmt::node::setup::SetupStorage::Diskless + ) + }) { + continue; + } + let bindings = if let Some(node) = nodes.get(&expected.node_id).filter(|node| { + matches!( + node.policy.setup.storage, + crowdb_protocol::mgmt::node::setup::SetupStorage::Supplied { .. } + ) + }) { + match supplied::bindings(expected, node, self.disks.as_deref(), self.groups.as_deref()) { + Ok(bindings) => bindings, + Err((code, detail)) => { + evidence.push(checks::observation( + &format!("storage/{}/{}", expected.rack_id, expected.node_id), + Err((code, detail)), + )); + continue; + } + } + } else { + vec![expected.clone()] + }; + for expected in &bindings { + let id = format!( + "storage/{}/{}/{}", + expected.rack_id, expected.node_id, expected.disk_group_id + ); + let disk = checks::disks( + expected, + self.groups.as_deref(), + self.disks.as_deref(), + self.binds.as_deref(), + ); + let owner = if disk.is_ok() { + checks::owner(expected, self.owners.as_deref(), self.instances.as_deref(), now) + } else { + Err(("storage-waiting-for-disks", format!("Waiting for {id}"))) + }; + let space = if owner.is_ok() { + checks::space(expected, self.owners.as_deref(), self.instances.as_deref(), now) + } else { + Err(("storage-waiting-for-owner", format!("Waiting for {id}/owner"))) + }; + for (suffix, result) in [("", disk), ("/owner", owner), ("/space", space)] { + let mut observation = checks::observation(&format!("{id}{suffix}"), result); + observation.revision = Some(self.revision); + observation.operation_id = Some(self.accepted.operation_id.clone()); + observation + .current_operation_id + .clone_from(&observation.operation_id); + evidence.push(observation); + } + } + } + evidence + } +} diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/checks.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/checks.rs new file mode 100644 index 000000000..21f3cd08e --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/checks.rs @@ -0,0 +1,252 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Ordered storage diagnostics from authoritative records; never repairs state. + +use super::super::fact; +use crate::deployment::preparation::DefaultStorageBinding; +use crowdb_kv_client::DiskRecord; +use crowdb_protocol::{ + common::{HwStatus, InstanceValue}, + mgmt::cluster::{DependencyKind, DependencyObservation, DependencyState}, + sysdata::{DiskGroupEntry, DiskdbOwnerEntry, KVGroupBindEntry}, +}; + +type Read<'a, T> = Result<&'a [T], &'a crowdb_kv_client::Error>; +type Check = Result; + +pub(super) fn observation(id: &str, result: Check) -> DependencyObservation { + let mut observation = fact( + id, + DependencyKind::Storage, + if result.is_ok() { + DependencyState::Ready + } else if result + .as_ref() + .is_err_and(|(code, _)| code.starts_with("storage-waiting-for-")) + { + DependencyState::Waiting + } else { + DependencyState::Unknown + }, + result.as_ref().err().map_or("storage-ready", |(code, _)| *code), + ); + observation.detail = match result { + Ok(detail) | Err((_, detail)) => detail, + }; + if let Some(reason) = &mut observation.reason { + reason.detail.clone_from(&observation.detail); + } + observation +} + +pub(super) fn disks( + expected: &DefaultStorageBinding, + groups: Read<'_, DiskGroupEntry>, + disks: Read<'_, DiskRecord>, + binds: Read<'_, KVGroupBindEntry>, +) -> Check { + let groups = groups.map_err(|_| { + ( + "storage-groups-unavailable", + "Group-0 disk-group read failed".into(), + ) + })?; + let group = groups + .iter() + .find(|group| (group.rack_id, group.node_id, group.dg_id) == location(expected)) + .ok_or_else(|| { + ( + "storage-group-missing", + "Accepted disk group is not registered in Group 0".into(), + ) + })?; + if group.value.status != HwStatus::Up as i32 { + return Err(( + "storage-group-not-up", + format!("Disk-group status is {}; expected Up", group.value.status), + )); + } + if group.value.disk_ids.is_empty() { + return Err(("storage-group-empty", "Disk group has no registered disks".into())); + } + let disks = disks.map_err(|_| ("storage-disks-unavailable", "Group-0 disk read failed".into()))?; + for id in &group.value.disk_ids { + let disk = disks + .iter() + .find(|disk| { + (disk.rack_id, disk.node_id, disk.disk_group_id) == location(expected) && disk.disk_id == *id + }) + .ok_or_else(|| { + ( + "storage-disk-missing", + format!("Disk {id:?} is absent from its accepted disk group"), + ) + })?; + if disk.value.status != HwStatus::Up as i32 { + return Err(( + "storage-disk-not-up", + format!("Disk {id:?} status is {}; expected Up", disk.value.status), + )); + } + if disk.value.capacity_units == 0 { + return Err(( + "storage-disk-capacity-invalid", + format!("Disk {id:?} has zero capacity units"), + )); + } + } + let binds = binds.map_err(|_| { + ( + "storage-bindings-unavailable", + "Group-0 metadata-binding read failed".into(), + ) + })?; + if !binds.iter().any(|bind| { + (bind.rack_id, bind.node_id, bind.dg_id) == location(expected) + && (bind.store_id, bind.group_id) == (expected.store_id, expected.group_id) + }) { + return Err(( + "storage-metadata-binding-missing", + format!( + "Disk group is not bound to accepted store {}/group {}", + expected.store_id, expected.group_id + ), + )); + } + Ok(format!("Group 0: disk group Up; all {} member disks Up with nonzero capacity; metadata binding store {}/group {} matches accepted placement", group.value.disk_ids.len(), expected.store_id, expected.group_id)) +} + +pub(super) fn owner( + expected: &DefaultStorageBinding, + owners: Read<'_, DiskdbOwnerEntry>, + instances: Read<'_, (u64, InstanceValue)>, + now: u64, +) -> Check { + let owners = owners.map_err(|_| ("storage-owners-unavailable", "Group-0 owner read failed".into()))?; + let owner = owners + .iter() + .find(|owner| (owner.rack_id, owner.node_id, owner.dg_id) == location(expected)) + .ok_or_else(|| ("storage-owner-missing", "Disk group has no assigned owner".into()))?; + if owner.lease_expiry_ms <= now { + return Err(( + "storage-owner-lease-expired", + format!( + "Owner {} lease expired at {}", + owner.instance_id, owner.lease_expiry_ms + ), + )); + } + let instances = instances.map_err(|_| { + ( + "storage-instances-unavailable", + "Group-0 live DiskDB registry read failed".into(), + ) + })?; + let instance = instances + .iter() + .find(|(id, _)| *id == owner.instance_id) + .ok_or_else(|| { + ( + "storage-owner-not-live", + format!("Owner {} has no live DiskDB registration", owner.instance_id), + ) + })?; + if !instance + .1 + .extra + .as_ref() + .and_then(|extra| extra.diskdb.as_ref()) + .is_some_and(|extra| extra.owned_dg_ids.contains(&expected.disk_group_id)) + { + return Err(( + "storage-owner-group-not-loaded", + format!( + "Owner {} has not reported disk group {} loaded", + owner.instance_id, expected.disk_group_id + ), + )); + } + Ok(format!("DiskDB instance {} is live and reports disk group {} loaded; owner lease expires at {} ({} ms remaining)", owner.instance_id, expected.disk_group_id, owner.lease_expiry_ms, owner.lease_expiry_ms - now)) +} + +pub(super) fn space( + expected: &DefaultStorageBinding, + owners: Read<'_, DiskdbOwnerEntry>, + instances: Read<'_, (u64, InstanceValue)>, + now: u64, +) -> Check { + let owners = owners.map_err(|_| ("storage-owners-unavailable", "Group-0 owner read failed".into()))?; + let owner = owners + .iter() + .find(|owner| (owner.rack_id, owner.node_id, owner.dg_id) == location(expected)) + .ok_or_else(|| ("storage-owner-missing", "Disk group has no assigned owner".into()))?; + let instances = instances.map_err(|_| { + ( + "storage-space-unavailable", + "Group-0 current-owner space summary read failed".into(), + ) + })?; + let usage = instances + .iter() + .find(|(id, _)| *id == owner.instance_id) + .and_then(|(_, instance)| instance.extra.as_ref()) + .and_then(|extra| extra.diskdb.as_ref()) + .and_then(|extra| { + extra + .group_usages + .iter() + .find(|usage| usage.disk_group_id == expected.disk_group_id) + }) + .ok_or_else(|| { + ( + "storage-space-missing", + format!( + "Current owner {} has not reported space usage for disk group {}", + owner.instance_id, expected.disk_group_id + ), + ) + })?; + if !usage.usage_known { + return Err(( + "storage-space-unknown", + "Disk-group space usage is not yet known".into(), + )); + } + if !now + .checked_sub(usage.sampled_at_ms) + .is_some_and(|age| age <= 15_000) + { + return Err(( + "storage-space-not-current", + format!( + "Space sample timestamp {} is stale or in the future relative to {now}", + usage.sampled_at_ms + ), + )); + } + if usage.allocatable_disk_count == 0 { + return Err(( + "storage-no-allocatable-disks", + "Disk group has no allocatable disks".into(), + )); + } + if usage.allocatable_free_bytes == 0 { + return Err(( + "storage-space-exhausted", + "Disk group has zero allocatable free bytes".into(), + )); + } + Ok(format!( + "Space sample at {} ({} ms old): {} allocatable disks, {} free bytes, {} capacity bytes", + usage.sampled_at_ms, + now - usage.sampled_at_ms, + usage.allocatable_disk_count, + usage.allocatable_free_bytes, + usage.allocatable_capacity_bytes + )) +} + +fn location(expected: &DefaultStorageBinding) -> (u64, u64, u64) { + (expected.rack_id, expected.node_id, expected.disk_group_id) +} diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/supplied.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/supplied.rs new file mode 100644 index 000000000..0d327d230 --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/collect/storage/supplied.rs @@ -0,0 +1,116 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Resolve every sealed supplied medium through its actual Group-0 placement. + +use crate::deployment::preparation::DefaultStorageBinding; +use crowdb_kv_client::DiskRecord; +use crowdb_protocol::{ + common::DiskId, + mgmt::node::{ + observation::NodeRuntimeObservation, + setup::{SetupStepPhase, SetupStorage, SetupStorageMode}, + }, + sysdata::DiskGroupEntry, + DiskIdExt, +}; +use std::collections::{BTreeMap, HashSet}; + +type Failure = (&'static str, String); + +pub(super) fn bindings( + expected: &DefaultStorageBinding, + node: &NodeRuntimeObservation, + disks: Result<&[DiskRecord], &crowdb_kv_client::Error>, + groups: Result<&[DiskGroupEntry], &crowdb_kv_client::Error>, +) -> Result, Failure> { + let SetupStorage::Supplied { disks: media } = &node.policy.setup.storage else { + unreachable!("caller selected supplied storage") + }; + let setup = node + .setup + .as_ref() + .filter(|setup| { + setup.discovery_id == node.discovery_id + && setup.node_id == Some(expected.node_id) + && setup.cluster_id.as_deref() + == node + .binding + .as_ref() + .map(|binding| binding.bootstrap.cluster_id.as_str()) + && setup.storage_mode == Some(SetupStorageMode::Supplied) + && uuid::Uuid::parse_str(&setup.operation_id).is_ok() + && setup.disks.len() == media.len() + && !media.is_empty() + }) + .ok_or_else(|| { + failure( + "storage-supplied-intent-missing", + "Current sealed supplied-media setup is unavailable", + ) + })?; + let disks = disks.map_err(|_| failure("storage-disks-unavailable", "Group-0 disk read failed"))?; + let groups = + groups.map_err(|_| failure("storage-groups-unavailable", "Group-0 disk-group read failed"))?; + let mut seen = HashSet::new(); + let mut selected = BTreeMap::new(); + for medium in media { + let status = setup + .disks + .iter() + .find(|disk| disk.path == medium.path && disk.phase == SetupStepPhase::Ready) + .ok_or_else(|| { + failure( + "storage-supplied-medium-pending", + "Accepted supplied medium is not registered", + ) + })?; + let id = status + .disk_id + .as_deref() + .and_then(|value| DiskId::from_display_string(value).ok()) + .filter(|id| seen.insert(*id)) + .ok_or_else(|| { + failure( + "storage-supplied-identity-invalid", + "Sealed supplied disk identity is missing or duplicated", + ) + })?; + let locations: Vec<_> = disks.iter().filter(|disk| disk.disk_id == id).collect(); + let [disk] = locations.as_slice() else { + return Err(failure( + "storage-supplied-disk-missing", + "Accepted supplied disk has no unique Group-0 placement", + )); + }; + if disk.rack_id != expected.rack_id || disk.node_id != expected.node_id { + return Err(failure( + "storage-supplied-placement-conflict", + "Accepted supplied disk belongs to a different node", + )); + } + if !groups.iter().any(|group| { + group.rack_id == expected.rack_id + && group.node_id == expected.node_id + && group.dg_id == disk.disk_group_id + && group.value.disk_ids.contains(&id) + }) { + return Err(failure( + "storage-supplied-membership-conflict", + "Accepted supplied disk is absent from its registered group membership", + )); + } + selected.insert( + disk.disk_group_id, + DefaultStorageBinding { + disk_group_id: disk.disk_group_id, + ..expected.clone() + }, + ); + } + Ok(selected.into_values().collect()) +} + +fn failure(code: &'static str, detail: &str) -> Failure { + (code, detail.into()) +} diff --git a/lib/crowdb-console-shared/src/ops/cluster/readiness/member.rs b/lib/crowdb-console-shared/src/ops/cluster/readiness/member.rs new file mode 100644 index 000000000..f311bbc7f --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/cluster/readiness/member.rs @@ -0,0 +1,280 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Verify each member's local evidence against committed identity and intent. + +use crate::deployment::{registry::NodeRecord, PreparedBootstrap}; +use crowdb_protocol::mgmt::{ + cluster::{DependencyKind, DependencyObservation, DependencyState, ReadinessReason}, + node::{ + deployment::NodeDeploymentPolicy, observation::NodeRuntimeObservation, setup::SetupServiceKind, + NodeServiceAction, NodeServiceIntent, + }, +}; + +/// Project a bounded node response; missing or conflicting members cannot be +/// hidden by another member's healthy service of the same kind. +#[must_use] +pub fn member_evidence( + bootstrap: &PreparedBootstrap, + expected: &NodeRecord, + policy: &NodeDeploymentPolicy, + intents: Option<&[NodeServiceIntent]>, + observed: Option<&NodeRuntimeObservation>, + now: u64, +) -> Vec { + let id = format!("node/{}", expected.node_id); + let Some(node) = observed else { + return vec![fact( + id, + DependencyKind::Member, + DependencyState::Unknown, + now, + "member-unreachable", + )]; + }; + if duplicate_evidence(node) + || node.version != 1 + || node.discovery_id != expected.discovery_id + || node.policy.validate().is_err() + || node.policy.formation() != policy.formation() + || node.binding.as_ref().is_some_and(|binding| { + binding.node_id != expected.node_id || binding.bootstrap != bootstrap.identity + }) + { + return vec![fact( + id, + DependencyKind::Identity, + DependencyState::Failed, + now, + "member-identity-conflict", + )]; + } + if node.binding.is_none() { + return vec![fact( + id, + DependencyKind::Member, + DependencyState::Initializing, + node.observed_at_unix_ms, + "member-binding-pending", + )]; + } + let mut facts = vec![fact( + id, + DependencyKind::Member, + DependencyState::Ready, + node.observed_at_unix_ms, + "member-observed", + )]; + for selected in node + .policy + .setup + .services + .iter() + .filter(|service| service.enabled) + { + let kind = service_name(selected.kind); + let current: Vec<_> = intents + .into_iter() + .flatten() + .filter(|intent| { + intent.node_id == expected.node_id + && intent.cluster_id == bootstrap.identity.cluster_id + && intent.kind == kind + }) + .collect(); + if current.is_empty() { + facts.push(service_fact(node, kind, None)); + } else { + for intent in current { + facts.push(service_fact(node, kind, Some(intent))); + } + } + if selected.kind == SetupServiceKind::AccessServer { + for protocol in ["s3", "iceberg", "dataset"] { + facts.push(protocol_fact( + node, + protocol, + intents, + &bootstrap.identity.cluster_id, + )); + } + } + } + facts +} + +fn service_fact( + node: &NodeRuntimeObservation, + kind: &str, + current: Option<&NodeServiceIntent>, +) -> DependencyObservation { + let node_id = node.binding.as_ref().map_or(0, |binding| binding.node_id); + let service_id = current.map_or_else(|| format!("{kind}-{node_id}"), |intent| intent.service_id.clone()); + let observed = node.services.iter().find(|service| { + service.kind == kind + && (service.service_id == service_id + || (kind == "kv" && current.is_none() && service.service_id == "kv")) + }); + let mut result = fact( + format!("node/{node_id}/service/{service_id}"), + DependencyKind::Service, + DependencyState::Unknown, + node.observed_at_unix_ms, + "service-observation-missing", + ); + result.current_operation_id = current.map(|intent| intent.operation_id.clone()); + if let Some(observed) = observed { + result.operation_id.clone_from(&observed.operation_id); + result.observed_at_unix_ms = observed.observed_at_unix_ms; + result.max_age_ms = observed.max_age_ms.min(3000); + result.state = observed.state; + result.detail = format!( + "Service {}: {:?}; observed operation {}; current operation {}; reason {}; {}", + observed.service_id, + observed.state, + observed.operation_id.as_deref().unwrap_or("none"), + result.current_operation_id.as_deref().unwrap_or("none"), + observed.reason_code.as_deref().unwrap_or("none"), + observed.detail, + ); + } + if current.is_none() && kind != "kv" { + result.state = DependencyState::Unknown; + result.detail = format!("Service {service_id}: current committed {kind} intent is unavailable"); + } + if current + .is_some_and(|intent| matches!(intent.action, NodeServiceAction::Stop | NodeServiceAction::Delete)) + { + result.state = DependencyState::Stopped; + result.operation_id = result.current_operation_id.clone(); + result.observed_at_unix_ms = node.observed_at_unix_ms; + result.detail = format!("Service {service_id}: current committed action is stop or delete"); + } + result.reason = (result.state != DependencyState::Ready).then(|| ReadinessReason { + dependency: result.id.clone(), + code: "required-service-not-ready".into(), + detail: result.detail.clone(), + }); + result +} + +fn protocol_fact( + node: &NodeRuntimeObservation, + protocol: &str, + intents: Option<&[NodeServiceIntent]>, + cluster: &str, +) -> DependencyObservation { + let node_id = node.binding.as_ref().map_or(0, |binding| binding.node_id); + let id = format!("node/{node_id}/protocol/{protocol}"); + let current = intents.into_iter().flatten().find(|intent| { + intent.node_id == node_id + && intent.cluster_id == cluster + && intent.service_id == format!("access-{node_id}") + && intent.kind == "access" + }); + let mut result = fact( + id.clone(), + DependencyKind::Protocol, + DependencyState::Unknown, + node.observed_at_unix_ms, + "protocol-observation-missing", + ); + result.current_operation_id = current.map(|intent| intent.operation_id.clone()); + if let Some(observed) = node + .protocols + .iter() + .find(|observed| observed.id == id && observed.kind == DependencyKind::Protocol) + { + result.operation_id.clone_from(&observed.operation_id); + result.observed_at_unix_ms = observed.observed_at_unix_ms; + result.max_age_ms = observed.max_age_ms.min(3000); + result.state = observed.state; + result.detail.clone_from(&observed.detail); + result.reason.clone_from(&observed.reason); + } + if current.is_none() { + result.state = DependencyState::Unknown; + result.reason = None; + result.detail = format!("{protocol}: current committed Access service intent is unavailable"); + } else if current + .is_some_and(|intent| matches!(intent.action, NodeServiceAction::Stop | NodeServiceAction::Delete)) + { + result.state = DependencyState::Stopped; + result.operation_id = result.current_operation_id.clone(); + result.observed_at_unix_ms = node.observed_at_unix_ms; + result.detail = format!("{protocol}: current committed Access service action is stop or delete"); + } + if result.state == DependencyState::Ready { + result.reason = None; + } else if result.reason.is_none() || result.state == DependencyState::Stopped { + result.reason = Some(ReadinessReason { + dependency: id, + code: if result.state == DependencyState::Stopped { + "operator-stopped-service" + } else { + "required-protocol-not-ready" + } + .into(), + detail: result.detail.clone(), + }); + } + result +} + +fn service_name(kind: SetupServiceKind) -> &'static str { + match kind { + SetupServiceKind::PaxosKv => "kv", + SetupServiceKind::Diskdb => "diskdb", + SetupServiceKind::Diskio => "diskio", + SetupServiceKind::Chunkdb => "chunkdb", + SetupServiceKind::ChunkKv => "chunk-kv", + SetupServiceKind::AccessServer => "access", + } +} + +fn fact( + id: String, + kind: DependencyKind, + state: DependencyState, + now: u64, + code: &str, +) -> DependencyObservation { + let reason = (state != DependencyState::Ready).then(|| reason(&id, code)); + DependencyObservation { + detail: format!("{id}: {code}"), + id, + kind, + required: true, + state, + observed_at_unix_ms: now, + max_age_ms: 3000, + revision: None, + operation_id: None, + current_operation_id: None, + reason, + } +} + +fn reason(id: &str, code: &str) -> ReadinessReason { + ReadinessReason { + dependency: id.into(), + code: code.into(), + detail: "Required member evidence is unavailable or does not match current authority".into(), + } +} + +fn duplicate_evidence(node: &NodeRuntimeObservation) -> bool { + let mut services = std::collections::BTreeSet::new(); + let mut protocols = std::collections::BTreeSet::new(); + node.services.len() > 64 + || node.protocols.len() > 3 + || node + .services + .iter() + .any(|service| !services.insert(&service.service_id)) + || node + .protocols + .iter() + .any(|protocol| !protocols.insert(&protocol.id)) +} diff --git a/lib/crowdb-console-shared/src/ops/kv_logical.rs b/lib/crowdb-console-shared/src/ops/kv_logical.rs index b357259b8..57088a747 100644 --- a/lib/crowdb-console-shared/src/ops/kv_logical.rs +++ b/lib/crowdb-console-shared/src/ops/kv_logical.rs @@ -21,7 +21,9 @@ use crate::clients::http::ServerClient; use crate::error::{Error, Result}; use crate::ops::OpContext; +mod default_group; mod group_wiring; +pub use default_group::ensure as ensure_default_group; mod publication; mod replica_join; mod replica_rollback; @@ -283,6 +285,9 @@ pub async fn add_group( message: "nodes list must not be empty".into(), }); } + if !ctx.is_test_scenario() { + crate::deployment::preparation::check_group_creation(ctx, store_id, group_id).await?; + } if !ctx.is_test_scenario() && (ctx.sysmd().get_group(store_id, group_id).await?.is_some() || !ctx @@ -314,6 +319,7 @@ pub async fn add_group( AddGroupInitialRole::Follower }), start_election: Some(single), + deployment: None, }; group_wiring::create_local(&client, store_id, &req) .await @@ -356,6 +362,9 @@ pub async fn add_group( // Record in group-0 sysdata. The sysdata write must // succeed — add_replica reads sysdata to find existing replicas, and // a missing group record causes a spurious 404. + if !ctx.is_test_scenario() { + crate::deployment::preparation::check_group_creation(ctx, store_id, group_id).await?; + } publication::group(ctx, store_id, group_id, &succeeded).await?; Ok(()) } @@ -379,6 +388,7 @@ pub async fn remove_group(ctx: &OpContext, store_id: u64, group_id: u64) -> Resu }); } let replicas = ctx.sysmd().list_replicas_in_group(store_id, group_id).await?; + crate::deployment::preparation::retire_group(ctx, store_id, group_id).await?; let node_ids: Vec = replicas.iter().map(|r| r.node_id).collect(); for nid in &node_ids { let client = server_client(ctx, *nid).await?; diff --git a/lib/crowdb-console-shared/src/ops/kv_logical/default_group.rs b/lib/crowdb-console-shared/src/ops/kv_logical/default_group.rs new file mode 100644 index 000000000..d8201442b --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/kv_logical/default_group.rs @@ -0,0 +1,168 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Resume the accepted default group without rollback or replacement identities. + +use crowdb_protocol::{ + kv_membership::{GroupMember, GroupMembershipState}, + mgmt::{AddGroupInitialRole, AddGroupRequest, GroupDeploymentIdentity}, +}; + +use super::{group_wiring, publication}; +use crate::{ + deployment::preparation::{self, DefaultPreparation}, + error::{Error, Result}, + ops::OpContext, +}; + +/// Create/wire/publish exactly the accepted ordinary group, then observe a read. +/// +/// # Errors +/// Rejects changed/retired intent and foreign native or published ownership. +/// Incomplete creation remains owned by the accepted operation for later replay. +pub async fn ensure(ctx: &OpContext, accepted: &DefaultPreparation) -> Result { + current(ctx, accepted).await?; + let group = &accepted.metadata; + let members: Vec<_> = group + .replicas + .iter() + .map(|replica| (replica.node_id, replica.replica_id)) + .collect(); + let resolved = group_wiring::resolve(ctx, group.store_id, &members).await?; + let complete = published( + ctx, + accepted, + &resolved + .iter() + .map(|(_, endpoint)| endpoint.clone()) + .collect::>(), + ) + .await?; + let deployment = GroupDeploymentIdentity { + cluster_id: accepted.bootstrap.cluster_id.clone(), + operation_id: accepted.operation_id.clone(), + }; + let mut requests = Vec::new(); + for (index, ((client, _), (_, replica))) in resolved.iter().zip(&members).enumerate() { + let request = AddGroupRequest { + group_id: group.group_id, + replica_id: *replica, + initial_role: Some(if index == 0 { + AddGroupInitialRole::Leader + } else { + AddGroupInitialRole::Follower + }), + start_election: Some(members.len() == 1), + deployment: Some(deployment.clone()), + }; + if complete { + let local = client.list_groups(group.store_id).await?; + if !local + .iter() + .any(|local| local.group_id == group.group_id && local.local_replica_id == *replica) + { + return Err(Error::Config( + "published default replica is missing; recover retained data explicitly".into(), + )); + } + } + requests.push((client, request)); + } + for (client, request) in &requests { + current(ctx, accepted).await?; + group_wiring::create_local(client, group.store_id, request).await?; + } + if !complete { + current(ctx, accepted).await?; + group_wiring::wire(ctx, group.store_id, group.group_id, &members).await?; + current(ctx, accepted).await?; + publication::group(ctx, group.store_id, group.group_id, &members).await?; + } + ctx.kv() + .seed_leader(group.store_id, group.group_id, resolved[0].1.clone()); + let readable = tokio::time::timeout( + std::time::Duration::from_secs(3), + ctx.kv().get( + group.store_id, + group.group_id, + b"/deployment/default-readiness", + crowdb_kv_client::ReadMode::Linearizable, + None, + ), + ) + .await; + Ok(matches!(readable, Ok(Ok(_)))) +} + +async fn current(ctx: &OpContext, accepted: &DefaultPreparation) -> Result<()> { + let actual = preparation::read(ctx, &accepted.bootstrap).await?; + if !actual.is_some_and(|(actual, _)| actual == *accepted && !actual.retired) { + return Err(Error::Config( + "default preparation changed or was explicitly retired".into(), + )); + } + Ok(()) +} + +async fn published(ctx: &OpContext, accepted: &DefaultPreparation, endpoints: &[String]) -> Result { + let group = &accepted.metadata; + let expected: Vec<_> = group + .replicas + .iter() + .zip(endpoints) + .map(|(replica, endpoint)| GroupMember { + replica_id: replica.replica_id, + node_id: replica.node_id, + endpoint: endpoint.clone(), + voting: true, + }) + .collect(); + let membership = ctx.membership().read(group.store_id, group.group_id).await?; + if let Some(membership) = &membership { + let initial = match &membership.record().installation { + GroupMembershipState::Ready => true, + GroupMembershipState::Installing { previous_members } => previous_members.is_empty(), + }; + if membership.record().epoch != 1 || membership.record().members != expected || !initial { + return Err(Error::Config("default metadata membership has changed".into())); + } + } + let actual = ctx.sysmd().get_group(group.store_id, group.group_id).await?; + if actual + .as_ref() + .is_some_and(|actual| actual.store_id != group.store_id || actual.group_id != group.group_id) + { + return Err(Error::Config( + "default metadata publication has another identity".into(), + )); + } + let mut replicas = ctx + .sysmd() + .list_replicas_in_group(group.store_id, group.group_id) + .await?; + let mut expected_replicas: Vec<_> = expected + .iter() + .map(|replica| { + let mut value = publication::replica_value( + group.store_id, + group.group_id, + replica.replica_id, + replica.node_id, + ); + value.endpoint.clone_from(&replica.endpoint); + value + }) + .collect(); + replicas.sort_unstable_by_key(|replica| replica.replica_id); + expected_replicas.sort_unstable_by_key(|replica| replica.replica_id); + if actual.is_some() && (replicas != expected_replicas || membership.is_none()) + || actual.is_none() && !replicas.is_empty() + { + return Err(Error::Config( + "default metadata publication differs from accepted replicas".into(), + )); + } + Ok(actual.is_some() + && membership + .is_some_and(|membership| membership.record().installation == GroupMembershipState::Ready)) +} diff --git a/lib/crowdb-console-shared/src/ops/kv_logical/group_wiring.rs b/lib/crowdb-console-shared/src/ops/kv_logical/group_wiring.rs index c02d5e920..3d628be18 100644 --- a/lib/crowdb-console-shared/src/ops/kv_logical/group_wiring.rs +++ b/lib/crowdb-console-shared/src/ops/kv_logical/group_wiring.rs @@ -19,6 +19,7 @@ pub(super) async fn create_local( let groups = client.list_groups(store_id).await?; if let Some(group) = groups.iter().find(|group| group.group_id == request.group_id) { return if group.local_replica_id == request.replica_id { + verify_creation(client, store_id, request).await?; Ok(false) } else { Err(Error::Conflict { @@ -30,11 +31,14 @@ pub(super) async fn create_local( match client.add_group(store_id, request).await { Ok(()) => Ok(true), Err(error) => { - if matches!(&error, Error::UpstreamRpc { status, .. } if status.contains("HTTP 409")) { + if request.deployment.is_some() + || matches!(&error, Error::UpstreamRpc { status, .. } if status.contains("HTTP 409")) + { let groups = client.list_groups(store_id).await?; if groups.iter().any(|group| { group.group_id == request.group_id && group.local_replica_id == request.replica_id }) { + verify_creation(client, store_id, request).await?; return Ok(false); } } @@ -43,6 +47,30 @@ pub(super) async fn create_local( } } +async fn verify_creation(client: &ServerClient, store_id: u64, request: &AddGroupRequest) -> Result<()> { + let actual = match client.group_creation(store_id, request.group_id).await { + Ok(actual) => actual, + Err(Error::UpstreamRpc { status, .. }) + if request.deployment.is_none() && status.contains("HTTP 404") => + { + return Ok(()) + } + Err(error) => return Err(error), + }; + if actual.version != 1 + || actual.store_id != store_id + || actual.group_id != request.group_id + || actual.replica_id != request.replica_id + || actual.deployment != request.deployment + || actual.retired + { + return Err(Error::Config( + "local group creation ownership differs or is retired".into(), + )); + } + Ok(()) +} + pub(super) async fn resolve( ctx: &OpContext, store_id: u64, diff --git a/lib/crowdb-console-shared/src/ops/kv_logical/publication.rs b/lib/crowdb-console-shared/src/ops/kv_logical/publication.rs index 1ca21d640..caf02720e 100644 --- a/lib/crowdb-console-shared/src/ops/kv_logical/publication.rs +++ b/lib/crowdb-console-shared/src/ops/kv_logical/publication.rs @@ -34,29 +34,14 @@ pub(super) async fn group( let authority = ctx.membership(); let mut complete_members = Vec::with_capacity(members.len()); for (node_id, replica_id) in members { - let endpoint = ctx - .sysmd() - .read_all_kv_server_instances() - .await? - .into_iter() - .find_map(|(_, instance)| { - (instance - .extra - .as_ref() - .and_then(|extra| extra.kv_server.as_ref()) - .and_then(|server| server.node_id) - == Some(*node_id)) - .then_some(instance.rpc_endpoint) - }); - let endpoint = match endpoint { - Some(endpoint) => endpoint, - None => super::rpc_endpoint_for_store(ctx, *node_id, store_id) - .await - .ok_or_else(|| Error::Validation { - field: "members.endpoint".into(), - message: format!("node {node_id} has no KV endpoint for store {store_id}"), - })?, - }; + // KV registrations advertise HTTP management origins. Resolve the + // store listener before publishing a native membership endpoint. + let endpoint = super::rpc_endpoint_for_store(ctx, *node_id, store_id) + .await + .ok_or_else(|| Error::Validation { + field: "members.endpoint".into(), + message: format!("node {node_id} has no KV endpoint for store {store_id}"), + })?; complete_members.push(GroupMember { replica_id: *replica_id, node_id: *node_id, @@ -72,7 +57,18 @@ pub(super) async fn group( let value = GroupValue { store_id, group_id }; let mut replicas: Vec<_> = members .iter() - .map(|(node_id, replica_id)| replica_value(store_id, group_id, *replica_id, *node_id)) + .map(|(node_id, replica_id)| { + let mut value = replica_value(store_id, group_id, *replica_id, *node_id); + if let Some(member) = installing + .record() + .members + .iter() + .find(|member| member.replica_id == *replica_id) + { + value.endpoint.clone_from(&member.endpoint); + } + value + }) .collect(); replicas.sort_unstable_by_key(|replica| replica.replica_id); let mut batch = vec![put(&key, &value)?]; diff --git a/lib/crowdb-console-shared/src/ops/kv_server.rs b/lib/crowdb-console-shared/src/ops/kv_server.rs index 4e93f7892..933e6f403 100644 --- a/lib/crowdb-console-shared/src/ops/kv_server.rs +++ b/lib/crowdb-console-shared/src/ops/kv_server.rs @@ -131,7 +131,7 @@ pub async fn deploy( }; let entry = ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: node_id.to_string(), url: deployed.mgmt_url.clone(), node_id: Some(node_id), @@ -276,7 +276,7 @@ pub async fn restart( let mut cfg = ctx.config_mut(); let _ = cfg.remove_server_for_node(node_id); let new_entry = ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: node_id.to_string(), url: deployed.mgmt_url.clone(), node_id: Some(node_id), diff --git a/lib/crowdb-console-shared/src/ops/s3.rs b/lib/crowdb-console-shared/src/ops/s3.rs index a6106956f..1f1ce1e4e 100644 --- a/lib/crowdb-console-shared/src/ops/s3.rs +++ b/lib/crowdb-console-shared/src/ops/s3.rs @@ -1,1338 +1,12 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Persistent, local S3 mini-cluster lifecycle and thin HTTP operations. +//! Native S3 transport and signing; deployment belongs to the node lifecycle. -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; -use std::process::Stdio; -use std::time::{Duration, Instant}; +mod client; +mod exchange; +mod signing; -use crowdb_protocol::port::namespace::{assign_process_ports, RuntimeNamespace}; -use crowdb_protocol::ServicePort; -use serde::Serialize; - -use crate::config::{ConsoleConfig, LocalLaunchSpec, NodeEntry, RackEntry, ServerEntry, ServiceType}; -use crate::error::{Error, Result}; -use crate::lifecycle; -use crate::ops::cluster::{self, KvDeployTunables, LocalChunkdbDeployConfig, LocalDiskdbDeployConfig}; -use crate::ops::OpContext; - -mod local_state; - -const MARKER_FILE: &str = "s3-mini-cluster.json"; -const INITIALIZING_FILE: &str = "s3-mini-cluster.initializing.json"; -const BOOTSTRAP_INTENT_FILE: &str = "bootstrap-intent.toml"; -const MASTER_KEY: &str = "1111111111111111111111111111111111111111111111111111111111111111"; -const NAMESPACE_ID: &str = "s3-mini-cluster"; -const BODY_PREVIEW_LIMIT: usize = 64 * 1024; - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, serde::Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum StorageProfile { - #[default] - Persistent, - Memory, -} - -#[derive(Debug, Clone, Serialize, serde::Deserialize)] -pub struct MiniClusterRecord { - pub version: u32, - #[serde(default)] - pub protected_test: bool, - pub endpoint: String, - #[serde(default)] - pub web_endpoint: String, - #[serde(default)] - pub web_pid: Option, - pub tenant: String, - #[serde(default)] - pub storage_profile: StorageProfile, -} - -#[derive(Debug, Clone, Serialize)] -pub struct MiniClusterStatus { - pub created: bool, - pub endpoint: String, - pub web_endpoint: String, - pub data_dir: PathBuf, - pub running_services: usize, - pub total_services: usize, -} - -#[must_use] -pub fn config_path(data_dir: &Path) -> PathBuf { - local_state::path(data_dir) -} - -/// Load a persisted mini-cluster record and local process state. -/// -/// # Errors -/// Returns an error for an unrecognized directory or invalid persisted data. -pub fn load(data_dir: &Path) -> Result<(ConsoleConfig, MiniClusterRecord)> { - let marker = std::fs::read(data_dir.join(MARKER_FILE)).map_err(|error| Error::Validation { - field: "root".into(), - message: format!("{} is not a CROWDB S3 mini-cluster: {error}", data_dir.display()), - })?; - let record = serde_json::from_slice(&marker).map_err(|error| Error::Config(error.to_string()))?; - let (config, _) = local_state::load(data_dir)?; - Ok((config, record)) -} - -/// Create or restart a persistent local S3 mini-cluster. -/// -/// # Errors -/// Returns an error for an unsafe directory, missing binary, failed service, -/// or failed readiness condition. -pub async fn start(data_dir: &Path) -> Result { - start_with_profile( - data_dir, - StorageProfile::Persistent, - 16 * 1024 * 1024 * 1024, - false, - ) - .await -} - -/// Start a simulated three-rack cluster for protected-placement tests. -/// -/// # Errors -/// Returns an error for an unsafe directory, missing binary, failed service, -/// or failed readiness condition. -pub async fn start_protected_test_cluster(data_dir: &Path) -> Result { - start_with_profile( - data_dir, - StorageProfile::Persistent, - 16 * 1024 * 1024 * 1024, - true, - ) - .await -} - -/// Create a fresh memory-backed cluster for an S3 benchmark. -/// -/// # Errors -/// Returns an error unless the location is missing or empty, or when a service -/// cannot start. Memory clusters are intentionally not restartable. -pub async fn start_memory(data_dir: &Path, memory_budget_bytes: u64) -> Result { - if memory_budget_bytes < 64 * 1024 * 1024 { - return Err(Error::Validation { - field: "memory_budget_bytes".into(), - message: "must be at least 64 MiB".into(), - }); - } - start_with_profile(data_dir, StorageProfile::Memory, memory_budget_bytes, false).await -} - -async fn start_with_profile( - data_dir: &Path, - storage_profile: StorageProfile, - capacity_bytes: u64, - protected_test: bool, -) -> Result { - archive_incomplete_attempt(data_dir)?; - validate_location(data_dir)?; - std::fs::create_dir_all(data_dir)?; - let canonical_root = std::fs::canonicalize(data_dir)?; - let data_dir = canonical_root.as_path(); - let marker_path = data_dir.join(MARKER_FILE); - if marker_path.exists() { - let (_, record) = load(data_dir)?; - if record.storage_profile != storage_profile { - return Err(Error::Validation { - field: "storage_profile".into(), - message: format!( - "existing cluster uses {:?}, requested {:?}", - record.storage_profile, storage_profile - ), - }); - } - if record.protected_test != protected_test { - return Err(Error::Validation { - field: "protected_test".into(), - message: "existing cluster uses a different rack layout".into(), - }); - } - if storage_profile == StorageProfile::Memory { - return Err(Error::Validation { - field: "root".into(), - message: "memory benchmark clusters cannot be restarted".into(), - }); - } - return restart(data_dir).await; - } - - if storage_profile == StorageProfile::Persistent { - RuntimeNamespace::persistent(data_dir, NAMESPACE_ID).map_err(namespace_error)?; - } - let config = ConsoleConfig::default(); - let ctx = OpContext::new( - "127.0.0.1:10000".into(), - vec!["http://127.0.0.1:10000".into()], - config, - ); - let (disk, chunk) = storage_configs(storage_profile, capacity_bytes, protected_test); - if let Some(status) = - resume_if_interrupted(data_dir, &disk, &chunk, storage_profile, protected_test).await? - { - return Ok(status); - } - let mut record = MiniClusterRecord { - version: 1, - protected_test, - endpoint: String::new(), - web_endpoint: String::new(), - web_pid: None, - tenant: "local".into(), - storage_profile, - }; - save_record(&data_dir.join(INITIALIZING_FILE), &record)?; - let initialized = initialize_new(&ctx, data_dir, &disk, &chunk, storage_profile, protected_test).await; - let endpoints = match initialized { - Ok(endpoints) => endpoints, - Err(error) => { - stop_config_processes(&mut ctx.config_mut()); - let _ = local_state::save(data_dir, &ctx.config()); - return Err(error); - } - }; - record.endpoint = endpoints.s3; - record.web_endpoint = endpoints.web; - record.web_pid = Some(endpoints.web_pid); - save_record(&marker_path, &record)?; - let _ = std::fs::remove_file(data_dir.join(INITIALIZING_FILE)); - let status = status_from(data_dir, true, &ctx.config(), &record); - Ok(status) -} - -fn storage_configs( - storage_profile: StorageProfile, - capacity_bytes: u64, - protected_test: bool, -) -> (LocalDiskdbDeployConfig, LocalChunkdbDeployConfig) { - let logical_capacity = if storage_profile == StorageProfile::Memory { - 16 * 1024 * 1024 * 1024 - } else { - capacity_bytes - }; - let per_node_capacity = logical_capacity - .checked_div(3) - .unwrap_or(capacity_bytes) - .max(64 * 1024 * 1024) - / (1024 * 1024) - * (1024 * 1024); - let disk = LocalDiskdbDeployConfig { - keepalive_interval_secs: protected_test.then_some(1), - disk_groups_per_node: 1, - disks_per_group: 1, - capacity_bytes: per_node_capacity, - zone_size_bytes: per_node_capacity, - unit_size_bytes: 1024 * 1024, - data_groups: vec![1], - rpc_workers: None, - kv_connections: None, - kv_client_rpc_workers: None, - free_batch_enabled: None, - free_flush_max_batch: None, - }; - let chunk = LocalChunkdbDeployConfig { - instance_count: 3, - storage_groups: vec![1, 2, 3], - dynamic_ownership: protected_test, - allow_unsafe_ec: !protected_test, - rpc_workers: None, - diskio_rpc_workers: None, - kv_connections: None, - kv_client_rpc_workers: None, - diskdb_connections: None, - diskdb_client_rpc_workers: None, - metrics_interval: None, - }; - (disk, chunk) -} - -async fn resume_if_interrupted( - data_dir: &Path, - disk: &LocalDiskdbDeployConfig, - chunk: &LocalChunkdbDeployConfig, - storage_profile: StorageProfile, - protected_test: bool, -) -> Result> { - if !data_dir.join(INITIALIZING_FILE).exists() || !local_state::path(data_dir).exists() { - return Ok(None); - } - let record: MiniClusterRecord = serde_json::from_slice(&std::fs::read(data_dir.join(INITIALIZING_FILE))?) - .map_err(|error| Error::Config(error.to_string()))?; - if record.version != 1 - || record.storage_profile != storage_profile - || record.protected_test != protected_test - { - return Err(Error::Conflict { - kind: "S3 bootstrap profile".into(), - id: data_dir.display().to_string(), - }); - } - resume_incomplete(data_dir, disk, chunk, record).await.map(Some) -} - -fn archive_incomplete_attempt(data_dir: &Path) -> Result<()> { - if !data_dir.join(INITIALIZING_FILE).exists() || data_dir.join(MARKER_FILE).exists() { - return Ok(()); - } - if local_state::path(data_dir).exists() || data_dir.join(BOOTSTRAP_INTENT_FILE).exists() { - return Ok(()); - } - let name = data_dir - .file_name() - .and_then(|value| value.to_str()) - .ok_or_else(|| Error::Validation { - field: "root".into(), - message: "incomplete cluster directory has no archiveable name".into(), - })?; - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |duration| duration.as_millis()); - let archived = data_dir.with_file_name(format!("{name}.failed-{timestamp}")); - if data_dir.join("namespace.json").is_file() { - RuntimeNamespace::persistent(data_dir, NAMESPACE_ID) - .map_err(namespace_error)? - .release() - .map_err(namespace_error)?; - } - std::fs::rename(data_dir, &archived)?; - Ok(()) -} - -async fn initialize_new( - ctx: &OpContext, - data_dir: &Path, - disk: &LocalDiskdbDeployConfig, - chunk: &LocalChunkdbDeployConfig, - storage_profile: StorageProfile, - protected_test: bool, -) -> Result { - let tunables = KvDeployTunables { - kv_backend: (storage_profile == StorageProfile::Memory).then(|| "mem-block".into()), - wal_backend: (storage_profile == StorageProfile::Memory).then(|| "mem-block".into()), - no_fsync: (storage_profile == StorageProfile::Memory).then_some(true), - ..KvDeployTunables::default() - }; - let (_, nodes) = if protected_test { - cluster::prepare_local_deploy_distinct_racks(ctx, 3, Some(data_dir), Some(&tunables)).await? - } else { - cluster::prepare_local_deploy(ctx, 3, Some(data_dir), Some(&tunables)).await? - }; - local_state::save(data_dir, &ctx.config())?; - cluster::init_with_intent(ctx, &nodes, &data_dir.join(BOOTSTRAP_INTENT_FILE)).await?; - initialize_after_kv(ctx, data_dir, disk, chunk, storage_profile, protected_test).await -} - -async fn initialize_after_kv( - ctx: &OpContext, - data_dir: &Path, - disk: &LocalDiskdbDeployConfig, - chunk: &LocalChunkdbDeployConfig, - storage_profile: StorageProfile, - protected_test: bool, -) -> Result { - let storage_services = ctx - .config() - .servers - .iter() - .filter(|server| { - matches!( - server.service_type, - ServiceType::Diskdb | ServiceType::Diskio | ServiceType::Chunkdb - ) - }) - .count(); - if storage_services == 6 + chunk.instance_count { - for group in &disk.data_groups { - if ctx.sysmd().get_group(0, *group).await?.is_none() { - return Err(Error::NotFound { - kind: "S3 data group".into(), - id: group.to_string(), - }); - } - } - cluster::restart_storage_services(ctx).await?; - } else if storage_services == 0 { - match storage_profile { - StorageProfile::Persistent => { - cluster::local_deploy_combined_file_backed_after_kv(ctx, data_dir, disk, chunk).await?; - } - StorageProfile::Memory => { - cluster::local_deploy_combined_after_kv(ctx, data_dir, disk, chunk, "mem").await?; - } - } - } else { - clear_partial_storage_launches(ctx, data_dir)?; - match storage_profile { - StorageProfile::Persistent => { - cluster::local_deploy_combined_file_backed_after_kv(ctx, data_dir, disk, chunk).await?; - } - StorageProfile::Memory => { - cluster::local_deploy_combined_after_kv(ctx, data_dir, disk, chunk, "mem").await?; - } - } - } - let seeds = management_seeds(&ctx.config()); - local_state::save(data_dir, &ctx.config())?; - let chunk_kv = spawn_chunk_kv(data_dir, &seeds, protected_test).await?; - add_service(ctx, chunk_kv)?; - local_state::save(data_dir, &ctx.config())?; - let access = spawn_access(data_dir, &seeds).await?; - let endpoint = access.entry.url.clone(); - add_service(ctx, access)?; - local_state::save(data_dir, &ctx.config())?; - let (web_endpoint, web_pid) = spawn_web(data_dir).await?; - Ok(StartedEndpoints { - s3: endpoint, - web: web_endpoint, - web_pid, - }) -} - -fn clear_partial_storage_launches(ctx: &OpContext, data_dir: &Path) -> Result<()> { - let partial = ctx - .config() - .servers - .iter() - .filter(|server| { - matches!( - server.service_type, - ServiceType::Diskdb | ServiceType::Diskio | ServiceType::Chunkdb - ) - }) - .cloned() - .collect::>(); - for server in &partial { - if let Some(pid) = server.pid.filter(|pid| lifecycle::process_is_alive(*pid)) { - let launch = ctx - .config() - .local_launches - .get(&server.id) - .cloned() - .ok_or_else(|| Error::Config(format!("{} has no launch specification", server.id)))?; - let actual_cwd = std::fs::read_link(format!("/proc/{pid}/cwd"))?; - if actual_cwd != Path::new(&launch.workdir) { - return Err(Error::Conflict { - kind: "S3 process identity".into(), - id: server.id.clone(), - }); - } - lifecycle::stop_pid_with_timeout(pid, Duration::from_secs(5))?; - if lifecycle::process_is_alive(pid) { - return Err(Error::Conflict { - kind: "S3 process still running".into(), - id: server.id.clone(), - }); - } - } - } - let ids = partial.into_iter().map(|server| server.id).collect::>(); - let mut config = ctx.config_mut(); - config.servers.retain(|server| !ids.contains(&server.id)); - for id in ids { - config.local_launches.remove(&id); - } - local_state::save(data_dir, &config) -} - -async fn resume_incomplete( - data_dir: &Path, - disk: &LocalDiskdbDeployConfig, - chunk: &LocalChunkdbDeployConfig, - mut record: MiniClusterRecord, -) -> Result { - let (mut config, seeds) = local_state::load(data_dir)?; - restore_launch_nodes(&mut config, record.protected_test)?; - let group0 = config - .servers - .iter() - .find(|server| server.service_type == ServiceType::PaxosKv) - .and_then(|server| server.rpc_url.as_deref()) - .ok_or_else(|| Error::Config("S3 bootstrap has no KV RPC seed".into()))? - .trim_start_matches("http://") - .to_owned(); - let ctx = OpContext::new(group0, seeds.clone(), config); - for node_id in 1..=3 { - let rack_id = if record.protected_test { node_id } else { 1 }; - let server_dir = data_dir - .join(format!("rack{rack_id}")) - .join(format!("node{node_id}")) - .join(format!("kv-server-{node_id}")); - crate::ops::kv_server::restart(&ctx, node_id, Some(&server_dir), None, &seeds).await?; - } - local_state::save(data_dir, &ctx.config())?; - cluster::init_with_intent(&ctx, &[1, 2, 3], &data_dir.join(BOOTSTRAP_INTENT_FILE)).await?; - let endpoints = initialize_after_kv( - &ctx, - data_dir, - disk, - chunk, - record.storage_profile, - record.protected_test, - ) - .await?; - record.endpoint = endpoints.s3; - record.web_endpoint = endpoints.web; - record.web_pid = Some(endpoints.web_pid); - save_record(&data_dir.join(MARKER_FILE), &record)?; - std::fs::remove_file(data_dir.join(INITIALIZING_FILE))?; - let status = status_from(data_dir, false, &ctx.config(), &record); - Ok(status) -} - -async fn restart(data_dir: &Path) -> Result { - let (mut config, mut record) = load(data_dir)?; - restore_launch_nodes(&mut config, record.protected_test)?; - if let Some(pid) = record.web_pid.take() { - let _ = lifecycle::stop_pid_with_timeout(pid, Duration::from_secs(5)); - save_record(&data_dir.join(MARKER_FILE), &record)?; - } - let seeds = management_seeds(&config); - let group0 = config - .servers - .iter() - .find(|server| server.service_type == ServiceType::PaxosKv) - .and_then(|server| server.rpc_url.as_deref()) - .unwrap_or("http://127.0.0.1:10000") - .trim_start_matches("http://") - .to_owned(); - let ctx = OpContext::new(group0, seeds.clone(), config); - let node_ids = ctx.config().nodes.iter().map(|node| node.id).collect::>(); - for node_id in node_ids { - let rack_id = if record.protected_test { node_id } else { 1 }; - let server_dir = data_dir - .join(format!("rack{rack_id}")) - .join(format!("node{node_id}")) - .join(format!("kv-server-{node_id}")); - crate::ops::kv_server::restart(&ctx, node_id, Some(&server_dir), None, &seeds).await?; - } - cluster::restart_storage_services(&ctx).await?; - local_state::save(data_dir, &ctx.config())?; - for kind in [ServiceType::ChunkKv, ServiceType::AccessServer] { - let server = ctx - .config() - .servers - .iter() - .find(|server| server.service_type == kind) - .cloned(); - let Some(server) = server else { - let spawned = match kind { - ServiceType::ChunkKv => spawn_chunk_kv(data_dir, &seeds, record.protected_test).await?, - ServiceType::AccessServer => spawn_access(data_dir, &seeds).await?, - _ => unreachable!(), - }; - if kind == ServiceType::AccessServer { - record.endpoint.clone_from(&spawned.entry.url); - } - add_service(&ctx, spawned)?; - local_state::save(data_dir, &ctx.config())?; - continue; - }; - let mut launch = ctx - .config() - .local_launches - .get(&server.id) - .cloned() - .ok_or_else(|| Error::Config(format!("{} has no launch specification", server.id)))?; - if kind == ServiceType::AccessServer { - launch - .env - .insert("CROWDB_S3_MASTER_KEY".into(), MASTER_KEY.into()); - } - let pid = lifecycle::restart_local_service(&server.id, server.pid.unwrap_or(0), &launch).await?; - if let Some(entry) = ctx - .config_mut() - .servers - .iter_mut() - .find(|entry| entry.id == server.id) - { - entry.pid = Some(pid); - } - local_state::save(data_dir, &ctx.config())?; - } - local_state::save(data_dir, &ctx.config())?; - let (web_endpoint, web_pid) = spawn_web(data_dir).await?; - record.web_endpoint = web_endpoint; - record.web_pid = Some(web_pid); - save_record(&data_dir.join(MARKER_FILE), &record)?; - let status = status_from(data_dir, false, &ctx.config(), &record); - Ok(status) -} - -fn save_record(path: &Path, record: &MiniClusterRecord) -> Result<()> { - std::fs::write( - path, - serde_json::to_vec_pretty(record).map_err(|error| Error::Config(error.to_string()))?, - )?; - Ok(()) -} - -/// Inspect persisted process liveness without mutating the cluster. -/// -/// # Errors -/// Returns an error when the directory is not a valid mini-cluster. -pub fn status(data_dir: &Path) -> Result { - let (config, record) = load(data_dir)?; - Ok(status_from(data_dir, false, &config, &record)) -} - -/// Stop all recorded processes while preserving data and configuration. -/// -/// # Errors -/// Returns an error when persisted state cannot be loaded or saved. -pub fn stop(data_dir: &Path) -> Result { - let (mut config, mut record) = load(data_dir)?; - if let Some(pid) = record.web_pid.take() { - let _ = lifecycle::stop_pid_with_timeout(pid, Duration::from_secs(5)); - } - stop_config_processes(&mut config); - local_state::save(data_dir, &config)?; - save_record(&data_dir.join(MARKER_FILE), &record)?; - Ok(status_from(data_dir, false, &config, &record)) -} - -/// Stop and permanently delete a persistent local S3 mini-cluster. -/// -/// # Errors -/// Returns an error when the location is not a recognized cluster, process -/// state cannot be persisted, claims cannot be released, or files cannot be -/// removed. -pub fn delete(data_dir: &Path) -> Result { - let status = stop(data_dir)?; - RuntimeNamespace::persistent(data_dir, NAMESPACE_ID) - .map_err(namespace_error)? - .delete() - .map_err(namespace_error)?; - Ok(status) -} - -fn validate_location(data_dir: &Path) -> Result<()> { - if !data_dir.exists() { - return Ok(()); - } - let mut entries = std::fs::read_dir(data_dir)?; - if entries.next().transpose()?.is_none() - || data_dir.join(MARKER_FILE).exists() - || (data_dir.join(INITIALIZING_FILE).exists() && local_state::path(data_dir).exists()) - { - return Ok(()); - } - Err(Error::Validation { - field: "root".into(), - message: format!( - "{} is non-empty and has no {MARKER_FILE}; refusing to overwrite it", - data_dir.display() - ), - }) -} - -fn stop_config_processes(config: &mut ConsoleConfig) { - for server in &mut config.servers { - if let Some(pid) = server.pid.take() { - let _ = lifecycle::stop_pid_with_timeout(pid, Duration::from_secs(5)); - } - } -} - -fn management_seeds(config: &ConsoleConfig) -> Vec { - config - .servers - .iter() - .filter(|server| server.service_type == ServiceType::PaxosKv) - .map(|server| server.url.clone()) - .collect() -} - -fn restore_launch_nodes(config: &mut ConsoleConfig, protected_test: bool) -> Result<()> { - let node_ids: Vec<_> = config - .servers - .iter() - .filter(|server| server.service_type == ServiceType::PaxosKv) - .map(|server| { - server - .node_id - .ok_or_else(|| Error::Config("S3 KV launch has no node id".into())) - }) - .collect::>()?; - for id in node_ids { - let rack_id = if protected_test { id } else { 1 }; - if config.racks.iter().all(|rack| rack.id != rack_id) { - config.add_rack(RackEntry { - id: rack_id, - name: format!("rack-{rack_id}"), - })?; - } - config.add_node(NodeEntry { - id, - rack_id, - host: "127.0.0.1".into(), - ssh_port: 22, - ssh_user: String::new(), - ssh_key: None, - ssh_password: None, - ssh_credential_ref: None, - })?; - } - Ok(()) -} - -struct SpawnedService { - entry: ServerEntry, - launch: LocalLaunchSpec, -} - -struct StartedEndpoints { - s3: String, - web: String, - web_pid: u32, -} - -fn add_service(ctx: &OpContext, service: SpawnedService) -> Result<()> { - ctx.config_mut() - .local_launches - .insert(service.entry.id.clone(), service.launch); - ctx.config_mut().add_server(service.entry) -} - -async fn spawn_chunk_kv(data_dir: &Path, seeds: &[String], protected_test: bool) -> Result { - let binary = find_binary("CROWDB_CHUNK_KV_SERVER_BIN", "crowdb-chunk-kv-server")?; - let rpc_port = assign_cluster_port(data_dir, ServicePort::ChunkKvRpc, "chunk-kv-1-rpc")?; - let http_port = assign_cluster_port(data_dir, ServicePort::ChunkKvHttp, "chunk-kv-1-http")?; - let workdir = data_dir.join("services/chunk-kv-1"); - let log_dir = workdir.join("log"); - std::fs::create_dir_all(&log_dir)?; - let seed_toml = seeds - .iter() - .map(|s| format!("{s:?}")) - .collect::>() - .join(", "); - let stream_mirror_copies = if protected_test { 2 } else { 1 }; - let config_path = workdir.join("chunk-kv.toml"); - std::fs::write( - &config_path, - format!( - "instance_id = 10000\nrpc_listen_addr = \"127.0.0.1:{rpc_port}\"\nrpc_advertise_addr = \"127.0.0.1:{rpc_port}\"\nhttp_listen_addr = \"127.0.0.1:{http_port}\"\ngroup0_mgmt_seeds = [{seed_toml}]\ncatalog_refresh_interval_ms = 200\n\n[balance]\nenabled = true\ntarget_partitions_per_owner = 1\ntarget_partition_bytes = 9223372036854775807\nminimum_weighted_improvement_percent = 100\ncooldown_ms = 9223372036854775807\nmax_owner_request_rate = 0\n\n[storage]\nmetadata_store_id = 0\nstream_mirror_copies = {stream_mirror_copies}\n\n[bootstrap_partition]\npartition_id = {{ high = 1, low = 1 }}\ntree_id = 1\nstream_name = {{ high = 2, low = 1 }}\nowner_epoch = 1\nmetadata_group_id = 1\n" - ), - )?; - let launch = LocalLaunchSpec { - env_file: None, - program: binary.to_string_lossy().into_owned(), - args: vec![ - "--config".into(), - config_path.to_string_lossy().into_owned(), - "--log-dir".into(), - log_dir.to_string_lossy().into_owned(), - ], - workdir: workdir.to_string_lossy().into_owned(), - env: BTreeMap::new(), - readiness_url: Some(format!("http://127.0.0.1:{http_port}/ready")), - }; - let pid = spawn(&launch, "chunk-kv-1").await?; - Ok(SpawnedService { - entry: server_entry("chunk-kv-1", ServiceType::ChunkKv, http_port, rpc_port, pid), - launch, - }) -} - -async fn spawn_access(data_dir: &Path, seeds: &[String]) -> Result { - let binary = find_binary("CROWDB_ACCESS_SERVER_BIN", "crowdb-access-server")?; - let port = assign_cluster_port(data_dir, ServicePort::AccessServerHttp, "access-server-1")?; - let health_port = assign_cluster_port(data_dir, ServicePort::AccessServerHealthHttp, "access-server-1")?; - let workdir = data_dir.join("services/access-server-1"); - std::fs::create_dir_all(workdir.join("log"))?; - let mut env = BTreeMap::new(); - env.insert("CROWDB_S3_LISTEN".into(), format!("127.0.0.1:{port}")); - env.insert( - "CROWDB_ACCESS_HEALTH_LISTEN".into(), - format!("127.0.0.1:{health_port}"), - ); - env.insert("CROWDB_MANAGEMENT_SEEDS".into(), seeds.join(",")); - env.insert("CROWDB_S3_TENANT".into(), "local".into()); - env.insert("CROWDB_S3_MASTER_KEY".into(), MASTER_KEY.into()); - env.insert("CROWDB_S3_REGION".into(), "us-east-1".into()); - env.insert("CROWDB_S3_TRUSTED_NETWORK".into(), "true".into()); - env.insert("CROWDB_S3_EC_DATA".into(), "2".into()); - env.insert("CROWDB_S3_EC_CODE".into(), "1".into()); - let runtime_launch = LocalLaunchSpec { - env_file: None, - program: binary.to_string_lossy().into_owned(), - args: vec!["s3".into()], - workdir: workdir.to_string_lossy().into_owned(), - env, - readiness_url: Some(format!("http://127.0.0.1:{health_port}/_crowdb/health/ready")), - }; - let pid = spawn(&runtime_launch, "access-server-1").await?; - let mut persisted_launch = runtime_launch; - persisted_launch.env.remove("CROWDB_S3_MASTER_KEY"); - Ok(SpawnedService { - entry: server_entry("access-server-1", ServiceType::AccessServer, port, port, pid), - launch: persisted_launch, - }) -} - -async fn spawn_web(data_dir: &Path) -> Result<(String, u32)> { - use crate::config::web::{WebMode, WebProcessConfig}; - - let binary = find_binary("CROWDB_WEB_BIN", "crowdb-web")?; - let port = assign_cluster_port(data_dir, ServicePort::Web, "web-1")?; - let root = std::fs::canonicalize(data_dir)?; - let workdir = root.join("services/web-1"); - let log_dir = workdir.join("log"); - std::fs::create_dir_all(&log_dir)?; - let ui_root = root.join("ui"); - std::fs::create_dir_all(&ui_root)?; - let endpoint = format!("http://127.0.0.1:{port}"); - let (_, seeds) = local_state::load(data_dir)?; - let config = WebProcessConfig { - version: 1, - mode: WebMode::BareMetal, - readonly: false, - bind: "127.0.0.1".into(), - port, - group0_management_seeds: seeds, - ui_root, - monitor_status: None, - log_dir, - log_max_file_mb: 30, - log_max_files: 5, - request_timeout_ms: Some(5_000), - }; - config.validate()?; - let config_path = root.join("s3-web.toml"); - std::fs::write( - &config_path, - toml::to_string_pretty(&config).map_err(|error| Error::Config(error.to_string()))?, - )?; - let mut env = BTreeMap::new(); - env.insert("CROWDB_ICEBERG_MANAGE_TOKEN".into(), web_token(&root)?); - let launch = LocalLaunchSpec { - env_file: None, - program: binary.to_string_lossy().into_owned(), - args: vec!["--config".into(), config_path.to_string_lossy().into_owned()], - workdir: workdir.to_string_lossy().into_owned(), - env, - readiness_url: Some(format!("{endpoint}/healthz")), - }; - let pid = spawn(&launch, "web-1").await?; - Ok((endpoint, pid)) -} - -fn web_token(root: &Path) -> Result { - use std::io::{Read, Write}; - use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; - - let path = root.join("s3-web-manage.token"); - if let Ok(metadata) = std::fs::symlink_metadata(&path) { - if !metadata.file_type().is_file() || metadata.permissions().mode() & 0o077 != 0 { - return Err(Error::Config( - "S3 Web management token file is not private".into(), - )); - } - let token = std::fs::read_to_string(path)?; - if token.len() != 64 || !token.bytes().all(|byte| byte.is_ascii_hexdigit()) { - return Err(Error::Config("S3 Web management token is invalid".into())); - } - return Ok(token); - } - let mut bytes = [0u8; 32]; - std::fs::File::open("/dev/urandom")?.read_exact(&mut bytes)?; - let mut token = String::with_capacity(64); - for byte in bytes { - const HEX: &[u8; 16] = b"0123456789abcdef"; - token.push(char::from(HEX[usize::from(byte >> 4)])); - token.push(char::from(HEX[usize::from(byte & 0x0f)])); - } - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(path)?; - file.write_all(token.as_bytes())?; - file.sync_all()?; - Ok(token) -} - -fn assign_cluster_port(data_dir: &Path, service: ServicePort, identity: &str) -> Result { - if data_dir.join("namespace.json").is_file() { - return RuntimeNamespace::persistent(data_dir, NAMESPACE_ID) - .map_err(namespace_error)? - .assign_named_port(service, identity) - .map_err(namespace_error); - } - assign_process_ports(service, 0, 1) - .map_err(namespace_error)? - .into_iter() - .next() - .ok_or_else(|| Error::Validation { - field: "port_alloc".into(), - message: "port allocator returned no assignment".into(), - }) -} - -fn namespace_error(error: impl std::fmt::Display) -> Error { - Error::Validation { - field: "runtime_namespace".into(), - message: error.to_string(), - } -} - -fn server_entry(id: &str, service_type: ServiceType, rest_port: u16, rpc_port: u16, pid: u32) -> ServerEntry { - ServerEntry { - dataset_origin: None, - id: id.into(), - url: format!("http://127.0.0.1:{rest_port}"), - node_id: None, - rpc_url: Some(format!("http://127.0.0.1:{rpc_port}")), - rest_port: Some(rest_port), - rpc_port: Some(rpc_port), - auto_start: true, - binary: None, - election_profile: None, - pid: Some(pid), - service_type, - rpc_workers: None, - no_fsync: false, - } -} - -async fn spawn(spec: &LocalLaunchSpec, id: &str) -> Result { - let log_path = Path::new(&spec.workdir) - .join("log") - .join(format!("{id}.stdout.log")); - let log = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&log_path)?; - let mut child = lifecycle::detached_command(&spec.program) - .args(&spec.args) - .envs(&spec.env) - .current_dir(&spec.workdir) - .stdout(Stdio::from(log.try_clone()?)) - .stderr(Stdio::from(log)) - .kill_on_drop(false) - .spawn()?; - let pid = child - .id() - .ok_or_else(|| Error::Config(format!("{id} has no pid")))?; - if let Some(url) = &spec.readiness_url { - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(1)) - .build() - .map_err(|error| http_error(&error))?; - let deadline = Instant::now() + Duration::from_secs(60); - loop { - if let Some(status) = child.try_wait()? { - return Err(Error::UpstreamRpc { - node_id: id.into(), - status: format!("exited before ready: {status}; log={}", log_path.display()), - }); - } - if client - .get(url) - .send() - .await - .is_ok_and(|response| response.status().is_success()) - { - break; - } - if Instant::now() >= deadline { - let _ = child.kill().await; - return Err(Error::UpstreamRpc { - node_id: id.into(), - status: format!("readiness timeout; log={}", log_path.display()), - }); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - } - std::mem::forget(child); - Ok(pid) -} - -fn find_binary(env: &str, name: &str) -> Result { - if let Some(path) = std::env::var_os(env) - .map(PathBuf::from) - .filter(|path| path.is_file()) - { - return Ok(path); - } - if let Ok(exe) = std::env::current_exe() { - for root in exe.ancestors().take(5) { - let candidate = root.join(name); - if candidate.is_file() { - return Ok(candidate); - } - } - } - Err(Error::NotFound { - kind: "binary".into(), - id: format!("{name} (set {env})"), - }) -} - -fn status_from( - data_dir: &Path, - created: bool, - config: &ConsoleConfig, - record: &MiniClusterRecord, -) -> MiniClusterStatus { - let web_is_running = record.web_pid.is_some_and(lifecycle::process_is_alive); - MiniClusterStatus { - created, - endpoint: record.endpoint.clone(), - web_endpoint: record.web_endpoint.clone(), - data_dir: data_dir.to_path_buf(), - running_services: config - .servers - .iter() - .filter(|server| server.pid.is_some_and(lifecycle::process_is_alive)) - .count() - + usize::from(web_is_running), - total_services: config.servers.len() + usize::from(!record.web_endpoint.is_empty()), - } -} - -fn http_error(error: &reqwest::Error) -> Error { - Error::UpstreamRpc { - node_id: "s3".into(), - status: error.to_string(), - } -} - -/// Send one thin S3 HTTP operation to the endpoint persisted in `data_dir`. -/// -/// # Errors -/// Returns an error for invalid cluster state, transport/protocol failure, or -/// any non-success S3 response. -pub async fn request( - data_dir: &Path, - method: reqwest::Method, - bucket: Option<&str>, - object: Option<&str>, - query: &[(&str, String)], - body: Option>, -) -> Result<(u16, Vec)> { - request_with_range(data_dir, method, bucket, object, query, body, None).await -} - -/// Send one thin S3 HTTP operation with an optional inclusive byte range. -/// -/// # Errors -/// Returns an error for a malformed range or any cluster, transport, protocol, -/// or S3 response failure. -pub async fn request_with_range( - data_dir: &Path, - method: reqwest::Method, - bucket: Option<&str>, - object: Option<&str>, - query: &[(&str, String)], - body: Option>, - range: Option<(u64, u64)>, -) -> Result<(u16, Vec)> { - let (_, record) = load(data_dir)?; - S3HttpClient::new(record.endpoint)? - .request(method, bucket, object, query, body, range) - .await -} - -/// Send one S3 operation and retain its HTTP request and response metadata. -/// -/// Unlike [`request_with_range`], an HTTP error status is returned as an -/// exchange so an interactive caller can show the response headers and body. -/// Transport and local configuration failures still return [`Error`]. -/// -/// # Errors -/// Returns an error for malformed input, invalid cluster state, or transport -/// failure. -pub async fn request_exchange_with_range( - data_dir: &Path, - method: reqwest::Method, - bucket: Option<&str>, - object: Option<&str>, - query: &[(&str, String)], - body: Option>, - range: Option<(u64, u64)>, -) -> Result { - request_exchange_with_headers( - data_dir, - method, - bucket, - object, - query, - body, - range, - reqwest::header::HeaderMap::new(), - ) - .await -} - -/// Send one S3 operation with caller-supplied request headers and retain the -/// complete HTTP exchange metadata. -/// -/// # Errors -/// Returns an error for malformed input, invalid cluster state, or transport -/// failure. -#[allow(clippy::too_many_arguments)] -pub async fn request_exchange_with_headers( - data_dir: &Path, - method: reqwest::Method, - bucket: Option<&str>, - object: Option<&str>, - query: &[(&str, String)], - body: Option>, - range: Option<(u64, u64)>, - headers: reqwest::header::HeaderMap, -) -> Result { - let (_, record) = load(data_dir)?; - S3HttpClient::new(record.endpoint)? - .request_exchange_with_headers(method, bucket, object, query, body, range, headers) - .await -} - -/// Captured metadata for an outbound S3 HTTP request. -#[derive(Debug)] -pub struct S3HttpRequest { - pub method: reqwest::Method, - pub url: reqwest::Url, - pub headers: reqwest::header::HeaderMap, - /// `None` means no body was supplied; `Some(0)` is an explicit empty body. - pub body_bytes: Option, - /// Bounded copy of a textual request body for interactive display. - pub body_preview: Option>, - pub body_preview_truncated: bool, -} - -/// One completed S3 HTTP request/response exchange. -#[derive(Debug)] -pub struct S3HttpExchange { - pub request: S3HttpRequest, - pub status: reqwest::StatusCode, - pub response_headers: reqwest::header::HeaderMap, - pub response_body: Vec, -} - -impl S3HttpExchange { - /// Convert the captured exchange to the compact result used by non-CLI - /// callers, preserving the existing non-success error behavior. - /// - /// # Errors - /// Returns an upstream error for a non-2xx HTTP response. - pub fn into_result(self) -> Result<(u16, Vec)> { - if !self.status.is_success() { - return Err(Error::UpstreamRpc { - node_id: "s3".into(), - status: format!( - "HTTP {}: {}", - self.status.as_u16(), - String::from_utf8_lossy(&self.response_body) - ), - }); - } - Ok((self.status.as_u16(), self.response_body)) - } -} - -/// Reusable thin client for one local S3 endpoint. -#[derive(Clone)] -pub struct S3HttpClient { - endpoint: String, - client: reqwest::Client, -} - -impl S3HttpClient { - /// Build a client with the standard per-request timeout. - /// - /// # Errors - /// Returns an error when the HTTP client cannot be constructed. - pub fn new(endpoint: String) -> Result { - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(30)) - .build() - .map_err(|error| http_error(&error))?; - Ok(Self { endpoint, client }) - } - - /// Load the endpoint persisted for a mini-cluster. - /// - /// # Errors - /// Returns an error when the cluster record or HTTP client is invalid. - pub fn from_data_dir(data_dir: &Path) -> Result { - let (_, record) = load(data_dir)?; - Self::new(record.endpoint) - } - - /// Send one S3 request, optionally with an inclusive byte range. - /// - /// # Errors - /// Returns an error for an invalid range or transport/protocol/S3 failure. - pub async fn request( - &self, - method: reqwest::Method, - bucket: Option<&str>, - object: Option<&str>, - query: &[(&str, String)], - body: Option>, - range: Option<(u64, u64)>, - ) -> Result<(u16, Vec)> { - self.request_exchange(method, bucket, object, query, body, range) - .await? - .into_result() - } - - /// Send one S3 request and retain HTTP metadata even for non-2xx status. - /// - /// # Errors - /// Returns an error for an invalid range, request construction failure, or - /// transport/protocol failure. - pub async fn request_exchange( - &self, - method: reqwest::Method, - bucket: Option<&str>, - object: Option<&str>, - query: &[(&str, String)], - body: Option>, - range: Option<(u64, u64)>, - ) -> Result { - self.request_exchange_with_headers( - method, - bucket, - object, - query, - body, - range, - reqwest::header::HeaderMap::new(), - ) - .await - } - - /// Send one S3 request with caller-supplied headers and retain HTTP - /// metadata even for a non-2xx status. - /// - /// # Errors - /// Returns an error for an invalid range, request construction failure, or - /// transport/protocol failure. - #[allow(clippy::too_many_arguments)] - pub async fn request_exchange_with_headers( - &self, - method: reqwest::Method, - bucket: Option<&str>, - object: Option<&str>, - query: &[(&str, String)], - body: Option>, - range: Option<(u64, u64)>, - headers: reqwest::header::HeaderMap, - ) -> Result { - let mut url = self.endpoint.clone(); - if let Some(bucket) = bucket { - url.push('/'); - url.push_str(&encode_path(bucket)); - } - if let Some(object) = object { - url.push('/'); - url.push_str(&object.split('/').map(encode_path).collect::>().join("/")); - } - if !query.is_empty() { - url.push('?'); - url.push_str( - &form_urlencoded::Serializer::new(String::new()) - .extend_pairs(query.iter().map(|(k, v)| (*k, v.as_str()))) - .finish(), - ); - } - let mut request = self.client.request(method, url).headers(headers); - if let Some((start, end)) = range { - if start > end { - return Err(Error::Validation { - field: "range".into(), - message: "range start must not exceed end".into(), - }); - } - request = request.header(reqwest::header::RANGE, format!("bytes={start}-{end}")); - } - let body_bytes = body.as_ref().map(Vec::len); - let textual_body = request - .try_clone() - .and_then(|builder| builder.build().ok()) - .and_then(|request| request.headers().get(reqwest::header::CONTENT_TYPE).cloned()) - .and_then(|value| value.to_str().ok().map(str::to_ascii_lowercase)) - .is_some_and(|value| { - value.starts_with("text/") || value.contains("json") || value.contains("xml") - }); - let body_preview = body - .as_ref() - .filter(|_| textual_body) - .map(|body| body.iter().copied().take(BODY_PREVIEW_LIMIT).collect::>()); - let body_preview_truncated = - body_bytes.is_some_and(|bytes| textual_body && bytes > BODY_PREVIEW_LIMIT); - if let Some(body) = body { - request = request - .header(reqwest::header::CONTENT_LENGTH, body.len()) - .body(body); - } - let mut request = request.build().map_err(|error| http_error(&error))?; - if !request.headers().contains_key(reqwest::header::HOST) { - let host = request.url().host_str().ok_or_else(|| Error::Validation { - field: "endpoint".into(), - message: "S3 endpoint has no host".into(), - })?; - let host = if host.contains(':') { - format!("[{host}]") - } else { - host.to_string() - }; - let authority = request - .url() - .port() - .map_or(host.clone(), |port| format!("{host}:{port}")); - let value = - reqwest::header::HeaderValue::from_str(&authority).map_err(|error| Error::Validation { - field: "endpoint".into(), - message: format!("invalid S3 endpoint authority: {error}"), - })?; - request.headers_mut().insert(reqwest::header::HOST, value); - } - let request_summary = S3HttpRequest { - method: request.method().clone(), - url: request.url().clone(), - headers: request.headers().clone(), - body_bytes, - body_preview, - body_preview_truncated, - }; - let response = self - .client - .execute(request) - .await - .map_err(|error| http_error(&error))?; - let status = response.status(); - let response_headers = response.headers().clone(); - let response_body = response - .bytes() - .await - .map_err(|error| http_error(&error))? - .to_vec(); - Ok(S3HttpExchange { - request: request_summary, - status, - response_headers, - response_body, - }) - } -} - -fn encode_path(value: &str) -> String { - value - .bytes() - .map(|byte| { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { - char::from(byte).to_string() - } else { - format!("%{byte:02X}") - } - }) - .collect() -} +pub use client::{S3Credentials, S3HttpClient}; +pub use exchange::{S3HttpExchange, S3HttpRequest}; +pub use signing::sign_request; diff --git a/lib/crowdb-console-shared/src/ops/s3/client.rs b/lib/crowdb-console-shared/src/ops/s3/client.rs new file mode 100644 index 000000000..b6b064714 --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/s3/client.rs @@ -0,0 +1,269 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Selected S3 endpoint transport, independent of runtime/process lifecycle. + +use super::{sign_request, S3HttpExchange, S3HttpRequest}; +use crate::error::{Error, Result}; +use std::time::Duration; + +const BODY_PREVIEW_LIMIT: usize = 64 * 1024; + +/// Credentials selected from the attached cluster, never formatted in diagnostics. +#[derive(Clone)] +pub struct S3Credentials { + pub access: String, + pub secret: String, + pub region: String, + pub session: Option, +} + +/// Reusable thin client for one local S3 endpoint. +#[derive(Clone)] +pub struct S3HttpClient { + endpoint: String, + client: reqwest::Client, + credentials: Option, +} + +impl S3HttpClient { + /// Build a client with the standard per-request timeout. + /// + /// # Errors + /// Returns an error when the HTTP client cannot be constructed. + pub fn new(endpoint: String) -> Result { + let client = reqwest::Client::builder() + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(Duration::from_secs(30)) + .build() + .map_err(|error| http_error(&error))?; + Ok(Self { + endpoint, + client, + credentials: None, + }) + } + + /// Build an authenticated client for a selected cluster's published S3 origin. + /// + /// # Errors + /// Rejects malformed origins or empty credentials before issuing requests. + pub fn signed(endpoint: String, credentials: S3Credentials) -> Result { + let origins = crate::deployment::target::management_origins(&[endpoint]) + .map_err(|_| Error::Config("invalid selected S3 origin".into()))?; + if [&credentials.access, &credentials.secret, &credentials.region] + .iter() + .any(|value| value.is_empty() || value.contains(['\n', '\r'])) + { + return Err(Error::Config("invalid selected S3 credentials".into())); + } + let mut client = Self::new(origins[0].clone())?; + client.credentials = Some(credentials); + Ok(client) + } + + /// Send one S3 request, optionally with an inclusive byte range. + /// + /// # Errors + /// Returns an error for an invalid range or transport/protocol/S3 failure. + pub async fn request( + &self, + method: reqwest::Method, + bucket: Option<&str>, + object: Option<&str>, + query: &[(&str, String)], + body: Option>, + range: Option<(u64, u64)>, + ) -> Result<(u16, Vec)> { + self.request_exchange(method, bucket, object, query, body, range) + .await? + .into_result() + } + + /// Send one S3 request and retain HTTP metadata even for non-2xx status. + /// + /// # Errors + /// Returns an error for an invalid range, request construction failure, or + /// transport/protocol failure. + pub async fn request_exchange( + &self, + method: reqwest::Method, + bucket: Option<&str>, + object: Option<&str>, + query: &[(&str, String)], + body: Option>, + range: Option<(u64, u64)>, + ) -> Result { + self.request_exchange_with_headers( + method, + bucket, + object, + query, + body, + range, + reqwest::header::HeaderMap::new(), + ) + .await + } + + /// Send one S3 request with caller-supplied headers and retain HTTP + /// metadata even for a non-2xx status. + /// + /// # Errors + /// Returns an error for an invalid range, request construction failure, or + /// transport/protocol failure. + #[allow(clippy::too_many_arguments)] + pub async fn request_exchange_with_headers( + &self, + method: reqwest::Method, + bucket: Option<&str>, + object: Option<&str>, + query: &[(&str, String)], + body: Option>, + range: Option<(u64, u64)>, + headers: reqwest::header::HeaderMap, + ) -> Result { + let url = request_url(&self.endpoint, bucket, object, query)?; + let mut headers = headers; + if let Some(credentials) = &self.credentials { + sign_request( + credentials, + &method, + &url, + body.as_deref().unwrap_or_default(), + &mut headers, + ) + .map_err(Error::Config)?; + } + let mut request = self.client.request(method, url).headers(headers); + if let Some((start, end)) = range { + if start > end { + return Err(Error::Validation { + field: "range".into(), + message: "range start must not exceed end".into(), + }); + } + request = request.header(reqwest::header::RANGE, format!("bytes={start}-{end}")); + } + let body_bytes = body.as_ref().map(Vec::len); + let textual_body = request + .try_clone() + .and_then(|builder| builder.build().ok()) + .and_then(|request| request.headers().get(reqwest::header::CONTENT_TYPE).cloned()) + .and_then(|value| value.to_str().ok().map(str::to_ascii_lowercase)) + .is_some_and(|value| { + value.starts_with("text/") || value.contains("json") || value.contains("xml") + }); + let body_preview = body + .as_ref() + .filter(|_| textual_body) + .map(|body| body.iter().copied().take(BODY_PREVIEW_LIMIT).collect::>()); + let body_preview_truncated = + body_bytes.is_some_and(|bytes| textual_body && bytes > BODY_PREVIEW_LIMIT); + if let Some(body) = body { + request = request + .header(reqwest::header::CONTENT_LENGTH, body.len()) + .body(body); + } + let mut request = request.build().map_err(|error| http_error(&error))?; + require_host(&mut request)?; + let request_summary = S3HttpRequest { + method: request.method().clone(), + url: request.url().clone(), + headers: request.headers().clone(), + body_bytes, + body_preview, + body_preview_truncated, + }; + let response = self + .client + .execute(request) + .await + .map_err(|error| http_error(&error))?; + let status = response.status(); + let response_headers = response.headers().clone(); + let response_body = response + .bytes() + .await + .map_err(|error| http_error(&error))? + .to_vec(); + Ok(S3HttpExchange { + request: request_summary, + status, + response_headers, + response_body, + }) + } +} + +fn encode_path(value: &str) -> String { + value + .bytes() + .map(|byte| { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'~') { + char::from(byte).to_string() + } else { + format!("%{byte:02X}") + } + }) + .collect() +} + +fn request_url( + endpoint: &str, + bucket: Option<&str>, + object: Option<&str>, + query: &[(&str, String)], +) -> Result { + let mut url = endpoint.to_owned(); + if let Some(bucket) = bucket { + url.push('/'); + url.push_str(&encode_path(bucket)); + } + if let Some(object) = object { + url.push('/'); + url.push_str(&object.split('/').map(encode_path).collect::>().join("/")); + } + if !query.is_empty() { + url.push('?'); + url.push_str( + &form_urlencoded::Serializer::new(String::new()) + .extend_pairs(query.iter().map(|(k, v)| (*k, v.as_str()))) + .finish(), + ); + } + reqwest::Url::parse(&url).map_err(|_| Error::Config("invalid S3 request URL".into())) +} + +fn require_host(request: &mut reqwest::Request) -> Result<()> { + if !request.headers().contains_key(reqwest::header::HOST) { + let host = request.url().host_str().ok_or_else(|| Error::Validation { + field: "endpoint".into(), + message: "S3 endpoint has no host".into(), + })?; + let host = if host.contains(':') { + format!("[{host}]") + } else { + host.to_string() + }; + let authority = request + .url() + .port() + .map_or(host.clone(), |port| format!("{host}:{port}")); + let value = + reqwest::header::HeaderValue::from_str(&authority).map_err(|error| Error::Validation { + field: "endpoint".into(), + message: format!("invalid S3 endpoint authority: {error}"), + })?; + request.headers_mut().insert(reqwest::header::HOST, value); + } + Ok(()) +} + +fn http_error(error: &reqwest::Error) -> Error { + Error::UpstreamRpc { + node_id: "s3".into(), + status: error.to_string(), + } +} diff --git a/lib/crowdb-console-shared/src/ops/s3/exchange.rs b/lib/crowdb-console-shared/src/ops/s3/exchange.rs new file mode 100644 index 000000000..52e54018f --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/s3/exchange.rs @@ -0,0 +1,49 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Request/response metadata for workload accounting and CLI diagnostics. + +use crate::error::{Error, Result}; + +/// Captured metadata for an outbound S3 HTTP request. +#[derive(Debug)] +pub struct S3HttpRequest { + pub method: reqwest::Method, + pub url: reqwest::Url, + pub headers: reqwest::header::HeaderMap, + /// `None` means no body was supplied; `Some(0)` is an explicit empty body. + pub body_bytes: Option, + /// Bounded copy of a textual request body for interactive display. + pub body_preview: Option>, + pub body_preview_truncated: bool, +} + +/// One completed S3 HTTP request/response exchange. +#[derive(Debug)] +pub struct S3HttpExchange { + pub request: S3HttpRequest, + pub status: reqwest::StatusCode, + pub response_headers: reqwest::header::HeaderMap, + pub response_body: Vec, +} + +impl S3HttpExchange { + /// Convert the captured exchange to the compact result used by non-CLI + /// callers, preserving the existing non-success error behavior. + /// + /// # Errors + /// Returns an upstream error for a non-2xx HTTP response. + pub fn into_result(self) -> Result<(u16, Vec)> { + if !self.status.is_success() { + return Err(Error::UpstreamRpc { + node_id: "s3".into(), + status: format!( + "HTTP {}: {}", + self.status.as_u16(), + String::from_utf8_lossy(&self.response_body) + ), + }); + } + Ok((self.status.as_u16(), self.response_body)) + } +} diff --git a/lib/crowdb-console-shared/src/ops/s3/local_state.rs b/lib/crowdb-console-shared/src/ops/s3/local_state.rs deleted file mode 100644 index 35483e699..000000000 --- a/lib/crowdb-console-shared/src/ops/s3/local_state.rs +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Local S3 mini-cluster process inputs and runtime identity, without topology. - -use std::collections::{BTreeMap, HashSet}; -use std::fs::{self, OpenOptions}; -use std::io::Write; -use std::os::unix::fs::OpenOptionsExt; -use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; - -use serde::{Deserialize, Serialize}; - -use crate::config::{ConsoleConfig, LocalLaunchSpec, ServerEntry, ServiceType}; -use crate::error::{Error, Result}; - -const FILE: &str = "s3-local-state.toml"; -const VERSION: u32 = 1; -static NEXT_TEMP: AtomicU64 = AtomicU64::new(0); - -#[derive(Debug, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct LocalState { - version: u32, - group0_seeds: Vec, - #[serde(rename = "service")] - services: Vec, - #[serde(default)] - local_launches: BTreeMap, -} - -pub(super) fn path(data_dir: &Path) -> PathBuf { - data_dir.join(FILE) -} - -pub(super) fn load(data_dir: &Path) -> Result<(ConsoleConfig, Vec)> { - let body = fs::read_to_string(path(data_dir))?; - let state: LocalState = toml::from_str(&body).map_err(|error| Error::Config(error.to_string()))?; - state.validate()?; - Ok(( - ConsoleConfig { - servers: state.services, - local_launches: state.local_launches, - ..ConsoleConfig::default() - }, - state.group0_seeds, - )) -} - -pub(super) fn save(data_dir: &Path, config: &ConsoleConfig) -> Result<()> { - let group0_seeds: Vec<_> = config - .servers - .iter() - .filter(|server| server.service_type == ServiceType::PaxosKv) - .map(|server| server.url.clone()) - .collect(); - let state = LocalState { - version: VERSION, - group0_seeds, - services: config.servers.clone(), - local_launches: config.local_launches.clone(), - }; - state.validate()?; - let body = toml::to_string_pretty(&state).map_err(|error| Error::Config(error.to_string()))?; - let destination = path(data_dir); - let temporary = destination.with_extension(format!( - "tmp.{}.{}", - std::process::id(), - NEXT_TEMP.fetch_add(1, Ordering::Relaxed) - )); - let result = (|| { - let mut file = OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&temporary)?; - file.write_all(body.as_bytes())?; - file.sync_all()?; - fs::rename(&temporary, &destination)?; - fs::File::open(data_dir)?.sync_all()?; - Ok(()) - })(); - if result.is_err() { - let _ = fs::remove_file(temporary); - } - result -} - -impl LocalState { - fn validate(&self) -> Result<()> { - let expected_seeds: Vec<_> = self - .services - .iter() - .filter(|service| service.service_type == ServiceType::PaxosKv) - .map(|service| service.url.clone()) - .collect(); - if self.version != VERSION - || expected_seeds.is_empty() - || expected_seeds != self.group0_seeds - || self.group0_seeds.iter().any(String::is_empty) - { - return Err(Error::Config( - "S3 local state version or Group 0 seeds are invalid".into(), - )); - } - let mut identities = HashSet::new(); - if self - .services - .iter() - .any(|service| !identities.insert(&service.id)) - || self - .local_launches - .values() - .any(|launch| launch.env.contains_key("CROWDB_S3_MASTER_KEY")) - { - return Err(Error::Config( - "S3 local state has duplicate services or inline secrets".into(), - )); - } - Ok(()) - } -} diff --git a/lib/crowdb-console-shared/src/ops/s3/signing.rs b/lib/crowdb-console-shared/src/ops/s3/signing.rs new file mode 100644 index 000000000..d110ac3f9 --- /dev/null +++ b/lib/crowdb-console-shared/src/ops/s3/signing.rs @@ -0,0 +1,107 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::S3Credentials; +use hmac::{Hmac, Mac}; +use reqwest::{header::HeaderMap, Method}; +use sha2::{Digest, Sha256}; +use std::fmt::Write as _; + +fn encode(value: &str, slash: bool) -> String { + let mut result = String::new(); + for byte in percent_encoding::percent_decode_str(value) { + if byte.is_ascii_alphanumeric() || b"-._~".contains(&byte) || (slash && byte == b'/') { + result.push(char::from(byte)); + } else { + write!(result, "%{byte:02X}").expect("String writing cannot fail"); + } + } + result +} + +fn mac(key: &[u8], value: &str) -> Vec { + let mut mac = Hmac::::new_from_slice(key).expect("HMAC accepts arbitrary key lengths"); + mac.update(value.as_bytes()); + mac.finalize().into_bytes().to_vec() +} + +/// Sign a path-style S3 request using cluster-selected credentials. +/// +/// # Errors +/// Rejects credentials that cannot be represented as HTTP headers. +pub fn sign_request( + credentials: &S3Credentials, + method: &Method, + url: &reqwest::Url, + body: &[u8], + headers: &mut HeaderMap, +) -> Result<(), String> { + let timestamp = chrono::DateTime::::from(std::time::SystemTime::now()) + .format("%Y%m%dT%H%M%SZ") + .to_string(); + let date = ×tamp[..8]; + let payload = hex::encode(Sha256::digest(body)); + let origin = url.origin().ascii_serialization(); + let host = origin.split_once("://").map_or(origin.as_str(), |(_, host)| host); + let mut signed = std::collections::BTreeMap::from([ + ("host", host.to_owned()), + ("x-amz-content-sha256", payload.clone()), + ("x-amz-date", timestamp.clone()), + ]); + if let Some(token) = &credentials.session { + signed.insert("x-amz-security-token", token.clone()); + } + let names = signed.keys().copied().collect::>().join(";"); + let mut canonical_headers = String::new(); + for (name, value) in &signed { + writeln!(canonical_headers, "{name}:{}", value.trim()).expect("String writing cannot fail"); + } + let mut query: Vec<_> = url + .query() + .unwrap_or("") + .split('&') + .filter(|part| !part.is_empty()) + .map(|part| { + let (key, value) = part.split_once('=').unwrap_or((part, "")); + (encode(key, false), encode(value, false)) + }) + .collect(); + query.sort(); + let query = query + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join("&"); + let canonical = format!( + "{method}\n{}\n{query}\n{canonical_headers}\n{names}\n{payload}", + encode(url.path(), true) + ); + let scope = format!("{date}/{}/s3/aws4_request", credentials.region); + let mut key = format!("AWS4{}", credentials.secret).into_bytes(); + for part in [date, &credentials.region, "s3", "aws4_request"] { + key = mac(&key, part); + } + let signature = hex::encode(mac( + &key, + &format!( + "AWS4-HMAC-SHA256\n{timestamp}\n{scope}\n{}", + hex::encode(Sha256::digest(canonical.as_bytes())) + ), + )); + signed.insert( + "authorization", + format!( + "AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={names}, Signature={signature}", + credentials.access + ), + ); + for (name, value) in signed { + let mut value: reqwest::header::HeaderValue = + value.parse().map_err(|_| "Invalid cluster S3 credential")?; + if matches!(name, "authorization" | "x-amz-security-token") { + value.set_sensitive(true); + } + headers.insert(name, value); + } + Ok(()) +} diff --git a/lib/crowdb-console-shared/src/ops/s3_bench.rs b/lib/crowdb-console-shared/src/ops/s3_bench.rs index d7a9392ed..48b24a3bd 100644 --- a/lib/crowdb-console-shared/src/ops/s3_bench.rs +++ b/lib/crowdb-console-shared/src/ops/s3_bench.rs @@ -1,10 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Bounded S3 workloads over an invocation-owned memory-backed mini-cluster. +//! Bounded S3 workloads attached to an existing selected cluster. -use std::path::PathBuf; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -13,9 +12,7 @@ use reqwest::Method; use serde::Serialize; use crate::error::{Error, Result}; -use crate::ops::s3::{self, S3HttpClient}; - -const BENCH_BUCKET: &str = "crowdb-memory-bench"; +use crate::ops::s3::S3HttpClient; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum S3BenchWorkload { @@ -28,7 +25,6 @@ pub enum S3BenchWorkload { #[derive(Debug, Clone)] pub struct S3BenchConfig { - pub work_dir: PathBuf, pub workload: S3BenchWorkload, pub object_size: usize, pub dataset_objects: usize, @@ -37,7 +33,7 @@ pub struct S3BenchConfig { pub duration: Duration, pub warmup_operations: u64, pub seed: u64, - pub memory_budget_bytes: u64, + pub max_dataset_bytes: u64, pub list_limit: usize, pub mix: MixWeights, } @@ -145,23 +141,14 @@ pub struct S3BenchResult { pub warmup_operations: u64, pub object_size: usize, pub dataset_objects: usize, - pub memory_budget_bytes: u64, + pub max_dataset_bytes: u64, pub estimated_dataset_bytes: u64, - pub peak_resident_bytes: u64, - pub backing: S3BenchBacking, + pub bucket: String, pub failures: S3BenchFailures, pub by_operation: S3BenchOperations, pub object_bytes_verified: bool, } -#[derive(Debug, Serialize)] -pub struct S3BenchBacking { - pub kv: &'static str, - pub wal: &'static str, - pub diskio: &'static str, - pub chunk_kv: &'static str, -} - #[derive(Debug, Default, Serialize)] pub struct S3BenchFailures { pub metadata: u64, @@ -203,18 +190,30 @@ struct OperationAccumulator { latencies_us: Vec, } -/// Start an invocation-owned memory cluster, run one workload, and stop it. +#[derive(Clone)] +struct Invocation { + config: S3BenchConfig, + bucket: String, +} + +impl std::ops::Deref for Invocation { + type Target = S3BenchConfig; + fn deref(&self) -> &Self::Target { + &self.config + } +} + +/// Run against an already selected S3 endpoint, using one private workload bucket. /// /// # Errors -/// Returns validation, cluster startup, request-path, or cleanup errors. -pub async fn run(config: S3BenchConfig) -> Result { +/// Returns validation or request-path errors; never starts or stops services. +pub async fn run(config: S3BenchConfig, client: S3HttpClient) -> Result { let estimated_dataset_bytes = validate_workload(&config)?; - s3::start_memory(&config.work_dir, config.memory_budget_bytes).await?; - let outcome = run_attached(&config, estimated_dataset_bytes).await; - let stop_result = s3::stop(&config.work_dir); - let result = outcome?; - stop_result?; - Ok(result) + let invocation = Invocation { + config, + bucket: format!("crowdb-bench-{}", uuid::Uuid::new_v4().simple()), + }; + run_requests(&invocation, estimated_dataset_bytes, client).await } fn validate_workload(config: &S3BenchConfig) -> Result { @@ -223,48 +222,26 @@ fn validate_workload(config: &S3BenchConfig) -> Result { .ok() .and_then(|size| size.checked_mul(u64::try_from(config.dataset_objects).ok()?)) .ok_or_else(|| validation("dataset", "dataset size overflow"))?; - if estimated_dataset_bytes > config.memory_budget_bytes / 2 { + if estimated_dataset_bytes > config.max_dataset_bytes { return Err(validation( - "memory_budget_bytes", - "prepared dataset exceeds half of the memory budget", + "max_dataset_bytes", + "prepared dataset exceeds its declared byte bound", )); } Ok(estimated_dataset_bytes) } -async fn run_attached(config: &S3BenchConfig, estimated_dataset_bytes: u64) -> Result { - let client = S3HttpClient::from_data_dir(&config.work_dir)?; - run_requests(config, estimated_dataset_bytes, client).await -} - -/// Exercise the workload driver against a deterministic HTTP protocol peer. -/// -/// # Errors -/// Returns configuration, HTTP or workload validation errors. -#[cfg(feature = "test-util")] -pub async fn run_workload_for_tests(config: S3BenchConfig, endpoint: String) -> Result { - let estimated_dataset_bytes = validate_workload(&config)?; - run_requests(&config, estimated_dataset_bytes, S3HttpClient::new(endpoint)?).await -} - async fn run_requests( - config: &S3BenchConfig, + config: &Invocation, estimated_dataset_bytes: u64, client: S3HttpClient, ) -> Result { client - .request(Method::PUT, Some(BENCH_BUCKET), None, &[], None, None) + .request(Method::PUT, Some(&config.bucket), None, &[], None, None) .await?; prepare_dataset(&client, config).await?; let warmup_writes = run_warmup(&client, config).await?; - let sampling = Arc::new(AtomicBool::new(true)); - let peak_resident = Arc::new(AtomicU64::new(cluster_resident_bytes(&config.work_dir))); - let sampler = tokio::spawn(sample_cluster_resident( - config.work_dir.clone(), - Arc::clone(&sampling), - Arc::clone(&peak_resident), - )); let admitted = Arc::new(AtomicU64::new(0)); let deadline = Instant::now() + config.duration; let start = Instant::now(); @@ -285,25 +262,12 @@ async fn run_requests( ); } let duration_ms = elapsed_ms(start); - sampling.store(false, Ordering::Release); - sampler.await.map_err(|error| Error::Config(error.to_string()))?; - let peak_resident_bytes = peak_resident.load(Ordering::Acquire); cleanup(&client, &combined.written_operations, &warmup_writes, config).await?; - if peak_resident_bytes > config.memory_budget_bytes { - return Err(validation( - "memory_budget_bytes", - &format!( - "cluster peak resident bytes {peak_resident_bytes} exceeded budget {}", - config.memory_budget_bytes - ), - )); - } Ok(build_result( config, combined, duration_ms, estimated_dataset_bytes, - peak_resident_bytes, )) } @@ -323,7 +287,7 @@ fn validate_config(config: &S3BenchConfig) -> Result<()> { Ok(()) } -async fn prepare_dataset(client: &S3HttpClient, config: &S3BenchConfig) -> Result<()> { +async fn prepare_dataset(client: &S3HttpClient, config: &Invocation) -> Result<()> { if config.workload == S3BenchWorkload::Write { return Ok(()); } @@ -332,7 +296,7 @@ async fn prepare_dataset(client: &S3HttpClient, config: &S3BenchConfig) -> Resul client .request( Method::PUT, - Some(BENCH_BUCKET), + Some(&config.bucket), Some(&dataset_key(index)), &[], Some(body.clone()), @@ -343,7 +307,7 @@ async fn prepare_dataset(client: &S3HttpClient, config: &S3BenchConfig) -> Resul Ok(()) } -async fn run_warmup(client: &S3HttpClient, config: &S3BenchConfig) -> Result> { +async fn run_warmup(client: &S3HttpClient, config: &Invocation) -> Result> { let mut written_operations = Vec::new(); for index in 0..config.warmup_operations { let workload = selected_workload(config.workload, config.mix, config.seed ^ index); @@ -357,7 +321,7 @@ async fn run_warmup(client: &S3HttpClient, config: &S3BenchConfig) -> Result, deadline: Instant, worker: usize, @@ -392,7 +356,7 @@ async fn run_worker( async fn execute( client: &S3HttpClient, - config: &S3BenchConfig, + config: &Invocation, workload: S3BenchWorkload, operation: u64, warmup: bool, @@ -407,7 +371,7 @@ async fn execute( client .request( Method::PUT, - Some(BENCH_BUCKET), + Some(&config.bucket), Some(&key), &[], Some(deterministic_body(config.object_size, config.seed ^ operation)), @@ -418,7 +382,7 @@ async fn execute( S3BenchWorkload::Read => { let key = dataset_key(dataset_index(operation, config.dataset_objects)); let (_, body) = client - .request(Method::GET, Some(BENCH_BUCKET), Some(&key), &[], None, None) + .request(Method::GET, Some(&config.bucket), Some(&key), &[], None, None) .await?; if body.len() != config.object_size { return Err(validation("read", "response length differs from prepared object")); @@ -432,7 +396,7 @@ async fn execute( let (_, body) = client .request( Method::GET, - Some(BENCH_BUCKET), + Some(&config.bucket), Some(&key), &[], None, @@ -453,7 +417,7 @@ async fn execute( ("max-keys", config.list_limit.to_string()), ]; let (_, body) = client - .request(Method::GET, Some(BENCH_BUCKET), None, &query, None, None) + .request(Method::GET, Some(&config.bucket), None, &query, None, None) .await?; validate_list(&body)?; } @@ -466,7 +430,7 @@ async fn cleanup( client: &S3HttpClient, written_operations: &[u64], warmup_writes: &[u64], - config: &S3BenchConfig, + config: &Invocation, ) -> Result<()> { let mut keys = Vec::with_capacity(config.dataset_objects + written_operations.len() + warmup_writes.len()); @@ -483,7 +447,8 @@ async fn cleanup( ); let results = stream::iter(keys.into_iter().map(|key| { let client = client.clone(); - async move { delete_key(&client, &key).await } + let bucket = &config.bucket; + async move { delete_key(&client, bucket, &key).await } })) .buffer_unordered(config.concurrency) .collect::>() @@ -492,14 +457,14 @@ async fn cleanup( result?; } client - .request(Method::DELETE, Some(BENCH_BUCKET), None, &[], None, None) + .request(Method::DELETE, Some(&config.bucket), None, &[], None, None) .await?; Ok(()) } -async fn delete_key(client: &S3HttpClient, key: &str) -> Result<()> { +async fn delete_key(client: &S3HttpClient, bucket: &str, key: &str) -> Result<()> { client - .request(Method::DELETE, Some(BENCH_BUCKET), Some(key), &[], None, None) + .request(Method::DELETE, Some(bucket), Some(key), &[], None, None) .await?; Ok(()) } @@ -532,11 +497,10 @@ fn xml_values<'a>(xml: &'a str, tag: &str) -> Vec<&'a str> { } fn build_result( - config: &S3BenchConfig, + config: &Invocation, mut stats: WorkerStats, duration_ms: u64, estimated_dataset_bytes: u64, - peak_resident_bytes: u64, ) -> S3BenchResult { let total_operations = stats.operations.iter().map(|op| op.attempts).sum(); let total_errors = stats.operations.iter().map(|op| op.attempts - op.successes).sum(); @@ -563,15 +527,9 @@ fn build_result( warmup_operations: config.warmup_operations, object_size: config.object_size, dataset_objects: config.dataset_objects, - memory_budget_bytes: config.memory_budget_bytes, + max_dataset_bytes: config.max_dataset_bytes, estimated_dataset_bytes, - peak_resident_bytes, - backing: S3BenchBacking { - kv: "mem-block", - wal: "mem-block", - diskio: "mem", - chunk_kv: "mem", - }, + bucket: config.bucket.clone(), failures: stats.failures, by_operation, object_bytes_verified: false, @@ -683,32 +641,6 @@ fn elapsed_ms(start: Instant) -> u64 { u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX) } -fn cluster_resident_bytes(data_dir: &std::path::Path) -> u64 { - s3::load(data_dir).map_or(0, |(config, _)| { - config - .servers - .iter() - .filter_map(|server| server.pid) - .filter_map(process_resident_bytes) - .sum() - }) -} - -async fn sample_cluster_resident(data_dir: PathBuf, sampling: Arc, peak: Arc) { - while sampling.load(Ordering::Acquire) { - peak.fetch_max(cluster_resident_bytes(&data_dir), Ordering::AcqRel); - tokio::time::sleep(Duration::from_millis(100)).await; - } - peak.fetch_max(cluster_resident_bytes(&data_dir), Ordering::AcqRel); -} - -fn process_resident_bytes(pid: u32) -> Option { - let status = std::fs::read_to_string(format!("/proc/{pid}/status")).ok()?; - let line = status.lines().find(|line| line.starts_with("VmRSS:"))?; - let kib = line.split_ascii_whitespace().nth(1)?.parse::().ok()?; - kib.checked_mul(1024) -} - fn validation(field: &str, message: &str) -> Error { Error::Validation { field: field.into(), diff --git a/lib/crowdb-console-shared/src/ssh.rs b/lib/crowdb-console-shared/src/ssh.rs index 274709c38..faa9816ab 100644 --- a/lib/crowdb-console-shared/src/ssh.rs +++ b/lib/crowdb-console-shared/src/ssh.rs @@ -106,6 +106,28 @@ impl Session { /// Returns `Error::UpstreamRpc` for any russh / auth failure, with the /// `host:port` as the synthetic server id. pub(crate) async fn connect(node: &NodeEntry, creds: &SshCreds) -> Result { + Self::connect_verified( + node, + creds, + HostVerification::Address(known_hosts_store_for_session()?), + ) + .await + } + + /// Connect using an accepted node's key without consulting or changing TOFU. + pub(crate) async fn connect_pinned( + node: &NodeEntry, + creds: &SshCreds, + expected: PublicKey, + ) -> Result { + Self::connect_verified(node, creds, HostVerification::Member(expected)).await + } + + async fn connect_verified( + node: &NodeEntry, + creds: &SshCreds, + verification: HostVerification, + ) -> Result { if node.ssh_user.is_empty() { return Err(Error::Validation { field: "ssh_user".into(), @@ -120,14 +142,9 @@ impl Session { ..Default::default() }); - // Resolve a process-shared known_hosts store; if no path can - // be determined (no $HOME and no $CROWDB_KV_KNOWN_HOSTS) we fall - // back to a strict-reject ephemeral store so we never silently - // accept arbitrary keys. - let store = known_hosts_store_for_session()?; let handler = ClientHandler { host_id: id.clone(), - store, + verification, }; let mut handle = client::connect(config, addr, handler) @@ -166,6 +183,10 @@ impl Session { /// # Errors /// `Error::UpstreamRpc` for channel / exec / IO failures. pub(crate) async fn exec(&mut self, command: &str) -> Result { + self.exec_bounded(command, usize::MAX).await + } + + pub(crate) async fn exec_bounded(&mut self, command: &str, max_bytes: usize) -> Result { let id = "".to_string(); let mut channel = self .handle @@ -179,6 +200,13 @@ impl Session { let mut exit = None; while let Some(msg) = channel.wait().await { + let incoming = match &msg { + ChannelMsg::Data { data } | ChannelMsg::ExtendedData { data, .. } => data.len(), + _ => 0, + }; + if stdout.len().saturating_add(stderr.len()).saturating_add(incoming) > max_bytes { + return Err(Error::Config("SSH command output exceeds its bound".into())); + } match msg { ChannelMsg::Data { ref data } => stdout.extend_from_slice(data), ChannelMsg::ExtendedData { ref data, ext } => { @@ -241,7 +269,12 @@ fn ssh_err(id: &str, e: &impl std::fmt::Display) -> Error { struct ClientHandler { host_id: String, - store: Arc, + verification: HostVerification, +} + +enum HostVerification { + Address(Arc), + Member(PublicKey), } #[async_trait::async_trait] @@ -252,8 +285,13 @@ impl client::Handler for ClientHandler { &mut self, server_public_key: &PublicKey, ) -> std::result::Result { + let HostVerification::Address(store) = &self.verification else { + return Ok( + matches!(&self.verification, HostVerification::Member(expected) if expected == server_public_key), + ); + }; let presented = KeyRecord::from_public_key(server_public_key); - match self.store.check_or_insert(&self.host_id, &presented) { + match store.check_or_insert(&self.host_id, &presented) { Ok(KnownHostsOutcome::Known | KnownHostsOutcome::InsertedTofu) => Ok(true), Ok(KnownHostsOutcome::Mismatch { expected, got }) => { warn!( diff --git a/lib/crowdb-console-shared/tests/admission_control_failure_test.rs b/lib/crowdb-console-shared/tests/admission_control_failure_test.rs new file mode 100644 index 000000000..ad1c33c19 --- /dev/null +++ b/lib/crowdb-console-shared/tests/admission_control_failure_test.rs @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{config::NodeEntry, deployment::admission::remote_control, error::Error}; +use crowdb_protocol::mgmt::node::NodeControl; + +#[path = "common/ssh_server.rs"] +mod ssh_server; + +#[tokio::test] +async fn dependency_wait_survives_the_remote_monitor_control_boundary() { + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("monitor-dependency-wait"); + std::env::set_var("CROWDB_KV_KNOWN_HOSTS", root.path().join("known-hosts")); + let key = russh::keys::key::KeyPair::generate_ed25519().unwrap(); + let path = root.path().join("operator-key"); + russh::keys::encode_pkcs8_pem(&key, std::fs::File::create(&path).unwrap()).unwrap(); + for (message, waiting) in [ + ("Error: Waiting: DiskDB has not loaded registered disks\n", true), + ("Error: service operation was superseded\n", false), + ] { + let server = ssh_server::TestSshServer::start_with_result( + key.clone_public_key().unwrap(), + ssh_server::TestReply::Failure(message.into()), + ) + .await; + let node = NodeEntry { + id: 1, + rack_id: 1, + host: "127.0.0.1".into(), + ssh_port: server.port, + ssh_user: "operator".into(), + ssh_key: Some(path.to_string_lossy().into_owned()), + ssh_password: None, + ssh_credential_ref: None, + }; + let error = remote_control(&node, &NodeControl::Identity).await.unwrap_err(); + assert_eq!( + matches!(error, Error::DependenciesNotReady(_)), + waiting, + "{error}" + ); + } +} diff --git a/lib/crowdb-console-shared/tests/bootstrap_intent_test.rs b/lib/crowdb-console-shared/tests/bootstrap_intent_test.rs index a30e6131d..f9703b2a5 100644 --- a/lib/crowdb-console-shared/tests/bootstrap_intent_test.rs +++ b/lib/crowdb-console-shared/tests/bootstrap_intent_test.rs @@ -25,7 +25,7 @@ fn config() -> ConsoleConfig { ssh_credential_ref: Some("ops-key".into()), }); config.servers.push(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "kv-1".into(), url: "http://127.0.0.1:10000".into(), node_id: Some(1), diff --git a/lib/crowdb-console-shared/tests/cluster_observation_test.rs b/lib/crowdb-console-shared/tests/cluster_observation_test.rs new file mode 100644 index 000000000..9d288584f --- /dev/null +++ b/lib/crowdb-console-shared/tests/cluster_observation_test.rs @@ -0,0 +1,109 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/default_preparation.rs"] +mod common; +use common::TestPreparationAuthority; +use crowdb_console_shared::{deployment::preparation, ops::cluster::readiness::collect}; +use crowdb_protocol::mgmt::{ + cluster::{ClusterPhase, DependencyKind, DependencyState}, + node::{deployment::NodeDeploymentPolicy, observation::NodeRuntimeObservation, NodeBinding}, +}; + +fn local(authority: &TestPreparationAuthority) -> NodeRuntimeObservation { + NodeRuntimeObservation { + version: 1, + discovery_id: uuid::Uuid::new_v4().to_string(), + policy: NodeDeploymentPolicy::default(), + binding: Some(NodeBinding { + node_id: 2, + bootstrap: authority.bootstrap.identity.clone(), + management_seeds: Vec::new(), + }), + setup: None, + observed_at_unix_ms: 0, + services: Vec::new(), + protocols: Vec::new(), + } +} + +#[tokio::test] +async fn live_group0_does_not_imply_prepared_storage_or_protocol_readiness_and_never_repairs() { + let authority = TestPreparationAuthority::start(&[2]).await; + let before = authority + .ctx + .kv() + .scan_bounded(0, 0, b"", b"", b"", 4096, false, None) + .await + .unwrap() + .items; + let report = collect(&authority.ctx, local(&authority)).await.unwrap(); + assert_ne!(report.phase, ClusterPhase::Ready); + assert!(report + .observation + .dependencies + .iter() + .any(|fact| fact.kind == DependencyKind::Authority && fact.state == DependencyState::Ready)); + assert!(report + .observation + .dependencies + .iter() + .any(|fact| fact.kind == DependencyKind::Storage && fact.state == DependencyState::Unknown)); + assert!(report.observation.runtime.is_none()); + assert_eq!( + authority + .ctx + .kv() + .scan_bounded(0, 0, b"", b"", b"", 4096, false, None) + .await + .unwrap() + .items, + before + ); + assert!(preparation::read(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap() + .is_none()); +} + +#[tokio::test] +async fn reserved_or_retired_preparation_cannot_be_reported_as_ready() { + let authority = TestPreparationAuthority::start(&[2]).await; + let accepted = preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + let report = collect(&authority.ctx, local(&authority)).await.unwrap(); + assert_ne!(report.phase, ClusterPhase::Ready); + assert!(report + .observation + .dependencies + .iter() + .any(|fact| fact.id == "storage/4/2/1" && fact.state == DependencyState::Unknown)); + preparation::retire( + &authority.ctx, + &authority.bootstrap.identity, + &accepted.operation_id, + ) + .await + .unwrap(); + let report = collect(&authority.ctx, local(&authority)).await.unwrap(); + assert_ne!(report.phase, ClusterPhase::Ready); + assert!(report + .observation + .dependencies + .iter() + .any(|fact| fact.kind == DependencyKind::Storage && fact.state == DependencyState::Stopped)); +} + +#[tokio::test] +async fn a_different_bound_cluster_is_an_identity_failure() { + let authority = TestPreparationAuthority::start(&[2]).await; + let mut local = local(&authority); + local.binding.as_mut().unwrap().bootstrap.cluster_id = uuid::Uuid::new_v4().to_string(); + let report = collect(&authority.ctx, local).await.unwrap(); + assert_eq!(report.phase, ClusterPhase::Failed); + assert!(report + .reasons + .iter() + .any(|reason| reason.code == "cluster-identity-conflict")); +} diff --git a/lib/crowdb-console-shared/tests/cluster_readiness_test.rs b/lib/crowdb-console-shared/tests/cluster_readiness_test.rs new file mode 100644 index 000000000..a29b9fc53 --- /dev/null +++ b/lib/crowdb-console-shared/tests/cluster_readiness_test.rs @@ -0,0 +1,280 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::ops::cluster::readiness::reduce; +use crowdb_protocol::mgmt::cluster::{ + ClusterObservation, ClusterPhase, DependencyKind, DependencyObservation, DependencyState, +}; + +fn fact(kind: DependencyKind) -> DependencyObservation { + DependencyObservation { + id: format!("{kind:?}"), + kind, + detail: "Current test evidence".into(), + required: true, + state: DependencyState::Ready, + observed_at_unix_ms: 1000, + max_age_ms: 100, + revision: Some(1), + operation_id: None, + current_operation_id: None, + reason: None, + } +} + +fn observation() -> ClusterObservation { + ClusterObservation { + version: 1, + cluster: "dev".into(), + cluster_id: Some("27a1f711-7ea4-4ddc-8d4e-3146017fd8cb".into()), + bound: true, + initial_voters: 3, + policy: None, + observed_at_unix_ms: 1000, + members: vec![], + operation_id: None, + work_accepted: true, + setup_completed: true, + runtime: None, + dependencies: [ + DependencyKind::Authority, + DependencyKind::Metadata, + DependencyKind::Storage, + DependencyKind::Member, + DependencyKind::Service, + DependencyKind::Protocol, + ] + .into_iter() + .map(fact) + .collect(), + } +} + +#[test] +fn complete_current_application_evidence_is_ready_without_local_runtime_facts() { + let mut input = observation(); + let mut runtime = fact(DependencyKind::Runtime); + runtime.state = DependencyState::Unknown; + input.dependencies.push(runtime); + let report = reduce(input, 1000); + assert_eq!(report.phase, ClusterPhase::Ready); + assert!(report.reasons.iter().any(|reason| reason.dependency == "Runtime")); + let bytes = serde_json::to_vec(&report).unwrap(); + assert_eq!( + serde_json::from_slice::(&bytes).unwrap(), + report + ); +} + +#[test] +fn terminal_failure_precedes_quorum_loss_and_retains_every_dependency_reason() { + let mut input = observation(); + input.dependencies[0].state = DependencyState::Unknown; + input.dependencies[2].state = DependencyState::Failed; + input.dependencies[4].state = DependencyState::Stopped; + let report = reduce(input.clone(), 1000); + assert_eq!(report.phase, ClusterPhase::Failed); + for name in ["Authority", "Storage", "Service"] { + assert!(report.reasons.iter().any(|reason| reason.dependency == name)); + } + input.dependencies.reverse(); + assert_eq!(reduce(input, 1000), report); +} + +#[test] +fn superseded_failure_cannot_poison_a_recovered_dependency() { + let mut input = observation(); + let mut old = fact(DependencyKind::Storage); + old.state = DependencyState::Failed; + old.operation_id = Some("old-operation".into()); + old.current_operation_id = Some("new-operation".into()); + input.dependencies.push(old); + input.dependencies[2].operation_id = Some("new-operation".into()); + input.dependencies[2].current_operation_id = Some("new-operation".into()); + let report = reduce(input.clone(), 1000); + assert_eq!(report.phase, ClusterPhase::Ready); + assert!(report + .reasons + .iter() + .any(|reason| reason.code == "operation-superseded")); + input.dependencies.reverse(); + assert_eq!(reduce(input, 1000), report); +} + +#[test] +fn another_ready_instance_cannot_replace_missing_current_evidence() { + let mut input = observation(); + let mut pending = fact(DependencyKind::Service); + pending.id = "service/another-node".into(); + pending.operation_id = Some("old".into()); + pending.current_operation_id = Some("new".into()); + input.dependencies.push(pending); + let report = reduce(input, 1000); + assert_eq!(report.phase, ClusterPhase::Degraded); + assert!(report + .reasons + .iter() + .any(|reason| reason.dependency == "service/another-node")); +} + +#[test] +fn bound_quorum_loss_is_unavailable_but_prebootstrap_missing_voters_are_waiting() { + let mut input = observation(); + input.dependencies[0].state = DependencyState::Unknown; + assert_eq!(reduce(input.clone(), 1000).phase, ClusterPhase::Unavailable); + input.bound = false; + input.cluster_id = None; + input.setup_completed = false; + input.work_accepted = false; + input.dependencies[3].state = DependencyState::Waiting; + assert_eq!(reduce(input.clone(), 1000).phase, ClusterPhase::Waiting); + input.work_accepted = true; + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Initializing); +} + +#[test] +fn intentionally_stopped_required_services_and_missing_members_are_degraded() { + for kind in [DependencyKind::Service, DependencyKind::Member] { + let mut input = observation(); + input + .dependencies + .iter_mut() + .find(|fact| fact.kind == kind) + .unwrap() + .state = DependencyState::Stopped; + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Degraded); + } +} + +#[test] +fn stale_and_future_success_are_unknown_and_cannot_establish_readiness() { + for timestamp in [0, 1001] { + let mut input = observation(); + input.dependencies[4].observed_at_unix_ms = timestamp; + let report = reduce(input, 1000); + assert_eq!(report.phase, ClusterPhase::Degraded); + assert!(report + .reasons + .iter() + .any(|reason| reason.code == "observation-not-current")); + } +} + +#[test] +fn missing_required_categories_and_duplicate_dependency_identity_cannot_pass() { + let mut input = observation(); + input.dependencies.pop(); + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Degraded); + let mut input = observation(); + input.dependencies.push(input.dependencies[0].clone()); + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Failed); +} + +#[test] +fn validated_identity_conflict_is_terminal_even_before_first_bootstrap() { + let mut input = observation(); + input.bound = false; + input.cluster_id = None; + let mut conflict = fact(DependencyKind::Identity); + conflict.required = false; + conflict.state = DependencyState::Failed; + input.dependencies.push(conflict); + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Failed); +} + +#[test] +fn missing_execution_is_not_reported_as_a_superseded_operation() { + let mut input = observation(); + input.dependencies[4].operation_id = None; + input.dependencies[4].current_operation_id = Some("current-start".into()); + let report = reduce(input, 1000); + assert_ne!(report.phase, ClusterPhase::Ready); + let reason = report + .reasons + .iter() + .find(|reason| reason.code == "operation-observation-missing") + .unwrap(); + assert!(reason.detail.contains("current-start")); + assert!(!report + .reasons + .iter() + .any(|reason| reason.code == "operation-superseded")); +} + +fn live_observation(ahead_ms: u64) -> ClusterObservation { + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + let mut input = observation(); + input.observed_at_unix_ms = now; + for fact in &mut input.dependencies { + fact.observed_at_unix_ms = now + ahead_ms; + fact.max_age_ms = 3000; + } + input +} + +#[tokio::test] +async fn slightly_future_evidence_waits_for_the_clock_without_rewriting_timestamps() { + use crowdb_console_shared::ops::cluster::readiness::reduce_current; + use tokio::time::{Duration, Instant}; + let input = live_observation(40); + let mut original = input.dependencies.clone(); + original.sort(); + assert_ne!( + reduce(input.clone(), input.observed_at_unix_ms).phase, + ClusterPhase::Ready + ); + let report = reduce_current(input, Instant::now() + Duration::from_secs(1)).await; + assert_eq!(report.phase, ClusterPhase::Ready); + assert_eq!(report.observation.dependencies, original); +} + +#[tokio::test] +async fn future_wait_cannot_extend_the_deadline_or_refresh_old_evidence() { + use crowdb_console_shared::ops::cluster::readiness::reduce_current; + use tokio::time::{Duration, Instant}; + for (ahead, budget) in [(5000, 1000), (1000, 10)] { + let input = live_observation(ahead); + let before = Instant::now(); + let report = reduce_current(input, before + Duration::from_millis(budget)).await; + assert_eq!(report.phase, ClusterPhase::Unavailable); + assert!(before.elapsed() < Duration::from_millis(500)); + } + let mut input = live_observation(40); + input.dependencies[0].observed_at_unix_ms = input.observed_at_unix_ms - 10_000; + let old = input.dependencies[0].clone(); + let report = reduce_current(input, Instant::now() + Duration::from_secs(1)).await; + assert_eq!(report.phase, ClusterPhase::Unavailable); + assert_eq!(report.observation.dependencies[0], old); +} + +#[test] +fn diskless_kv_only_policy_requires_its_selected_services_without_inventing_protocols_or_disks() { + use crowdb_protocol::mgmt::node::{ + deployment::NodeDeploymentPolicy, + setup::{SetupServiceKind, SetupStorage}, + }; + let mut input = observation(); + let mut policy = NodeDeploymentPolicy::default(); + policy.cluster.clone_from(&input.cluster); + policy.initial_voters = input.initial_voters; + policy.setup.storage = SetupStorage::Diskless; + for service in &mut policy.setup.services { + service.enabled = service.kind == SetupServiceKind::PaxosKv; + } + policy.validate().unwrap(); + input.policy = Some(policy); + input + .dependencies + .retain(|fact| !matches!(fact.kind, DependencyKind::Storage | DependencyKind::Protocol)); + assert_eq!(reduce(input.clone(), 1000).phase, ClusterPhase::Ready); + let mut remote_protocol = fact(DependencyKind::Protocol); + remote_protocol.state = DependencyState::Unknown; + input.dependencies.push(remote_protocol); + assert_eq!(reduce(input, 1000).phase, ClusterPhase::Degraded); +} diff --git a/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs b/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs new file mode 100644 index 000000000..5159309ff --- /dev/null +++ b/lib/crowdb-console-shared/tests/cluster_storage_observation_test.rs @@ -0,0 +1,314 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/default_preparation.rs"] +mod common; +use common::TestPreparationAuthority; +use crowdb_console_shared::{deployment::preparation, ops::cluster::readiness::collect}; +use crowdb_protocol::mgmt::{ + cluster::DependencyState, + node::{deployment::NodeDeploymentPolicy, observation::NodeRuntimeObservation, NodeBinding}, +}; + +fn local(authority: &TestPreparationAuthority) -> NodeRuntimeObservation { + NodeRuntimeObservation { + version: 1, + discovery_id: uuid::Uuid::new_v4().to_string(), + policy: NodeDeploymentPolicy::default(), + binding: Some(NodeBinding { + node_id: 2, + bootstrap: authority.bootstrap.identity.clone(), + management_seeds: Vec::new(), + }), + setup: None, + observed_at_unix_ms: 0, + services: Vec::new(), + protocols: Vec::new(), + } +} + +async fn put(authority: &TestPreparationAuthority, path: &str, value: &impl serde::Serialize) { + authority + .ctx + .kv() + .put(0, 0, path.as_bytes(), &serde_json::to_vec(value).unwrap(), None) + .await + .unwrap(); +} + +async fn storage_layer(authority: &TestPreparationAuthority, suffix: &str, code: Option<&str>) -> String { + let report = collect(&authority.ctx, local(authority)).await.unwrap(); + let id = format!("storage/4/2/1{suffix}"); + let fact = report + .observation + .dependencies + .iter() + .find(|fact| fact.id == id) + .unwrap(); + assert!( + !fact.detail.is_empty(), + "Every storage result needs concrete evidence" + ); + assert_eq!( + fact.reason.as_ref().map(|reason| reason.code.as_str()), + code, + "{fact:?}" + ); + assert_eq!( + fact.state, + if code.is_none() { + DependencyState::Ready + } else if code.is_some_and(|code| code.starts_with("storage-waiting-for-")) { + DependencyState::Waiting + } else { + DependencyState::Unknown + } + ); + if let Some(reason) = &fact.reason { + assert_eq!(reason.detail, fact.detail); + } + fact.detail.clone() +} + +async fn ready_disks( + authority: &TestPreparationAuthority, +) -> (crowdb_protocol::diskdb::rpc::DiskValue, String) { + use crowdb_protocol::{ + common::{BindMapValue, DiskId, HwStatus}, + diskdb::rpc::{DiskGroupValue, DiskValue}, + key::{DiskGroupKey, DiskKey, TextKey}, + }; + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + storage_layer(authority, "", Some("storage-group-missing")).await; + storage_layer(authority, "/owner", Some("storage-waiting-for-disks")).await; + storage_layer(authority, "/space", Some("storage-waiting-for-owner")).await; + let disk_id = DiskId { high: 17, low: 23 }; + let group_path = DiskGroupKey { + rack_id: 4, + node_id: 2, + disk_group_id: 1, + } + .to_path(); + let disk_path = DiskKey { + rack_id: 4, + node_id: 2, + disk_group_id: 1, + disk_id, + } + .to_path(); + let mut group = DiskGroupValue { + status: HwStatus::Init as i32, + disk_ids: vec![disk_id], + ..Default::default() + }; + put(authority, &group_path, &group).await; + storage_layer(authority, "", Some("storage-group-not-up")).await; + group.status = HwStatus::Up as i32; + put(authority, &group_path, &group).await; + storage_layer(authority, "", Some("storage-disk-missing")).await; + let mut disk = DiskValue { + status: HwStatus::Init as i32, + capacity_units: 1024, + ..Default::default() + }; + put(authority, &disk_path, &disk).await; + storage_layer(authority, "", Some("storage-disk-not-up")).await; + disk.status = HwStatus::Up as i32; + put(authority, &disk_path, &disk).await; + storage_layer(authority, "", Some("storage-metadata-binding-missing")).await; + put( + authority, + "/hw/dg_bind/4/2/1", + &BindMapValue { + store_id: 0, + group_id: 1, + }, + ) + .await; + assert!(storage_layer(authority, "", None) + .await + .contains("all 1 member disks Up")); + (disk, disk_path) +} + +#[tokio::test] +async fn storage_diagnostics_follow_disk_owner_space_dependencies() { + use crowdb_protocol::common::{DiskGroupUsageSummary, HwStatus, OwnerMapValue}; + let authority = TestPreparationAuthority::start(&[2]).await; + let (mut disk, disk_path) = ready_disks(&authority).await; + storage_layer(&authority, "/owner", Some("storage-owner-missing")).await; + let mut owner = OwnerMapValue { + instance_id: 42, + lease_expiry_ms: 1, + ..Default::default() + }; + put(&authority, "/hw/dg_owner/4/2/1", &owner).await; + storage_layer(&authority, "/owner", Some("storage-owner-lease-expired")).await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(); + owner.lease_expiry_ms = u64::try_from(now).unwrap() + 120_000; + put(&authority, "/hw/dg_owner/4/2/1", &owner).await; + storage_layer(&authority, "/owner", Some("storage-owner-not-live")).await; + let registry = crowdb_kv_client::ServiceRegistryClient::from_shared(authority.ctx.kv_arc().clone()); + registry + .register_diskdb(42, "127.0.0.1:7200", &[], &[]) + .await + .unwrap(); + storage_layer(&authority, "/owner", Some("storage-owner-group-not-loaded")).await; + registry + .register_diskdb(42, "127.0.0.1:7200", &[1], &[]) + .await + .unwrap(); + assert!(storage_layer(&authority, "/owner", None) + .await + .contains("instance 42")); + storage_layer(&authority, "/space", Some("storage-space-missing")).await; + let mut usage = DiskGroupUsageSummary { + disk_group_id: 1, + ..Default::default() + }; + registry + .heartbeat_diskdb(42, "127.0.0.1:7200", &[1], std::slice::from_ref(&usage)) + .await + .unwrap(); + storage_layer(&authority, "/space", Some("storage-space-unknown")).await; + usage.usage_known = true; + storage_layer(&authority, "/owner", None).await; + registry + .heartbeat_diskdb(42, "127.0.0.1:7200", &[1], std::slice::from_ref(&usage)) + .await + .unwrap(); + storage_layer(&authority, "/space", Some("storage-space-not-current")).await; + usage.sampled_at_ms = u64::try_from(now).unwrap(); + registry + .heartbeat_diskdb(42, "127.0.0.1:7200", &[1], std::slice::from_ref(&usage)) + .await + .unwrap(); + storage_layer(&authority, "/space", Some("storage-no-allocatable-disks")).await; + usage.allocatable_disk_count = 1; + registry + .heartbeat_diskdb(42, "127.0.0.1:7200", &[1], std::slice::from_ref(&usage)) + .await + .unwrap(); + storage_layer(&authority, "/space", Some("storage-space-exhausted")).await; + usage.allocatable_free_bytes = 4096; + usage.allocatable_capacity_bytes = 8192; + registry + .heartbeat_diskdb(42, "127.0.0.1:7200", &[1], std::slice::from_ref(&usage)) + .await + .unwrap(); + assert!(storage_layer(&authority, "/space", None) + .await + .contains("4096 free bytes")); + disk.status = HwStatus::Offline as i32; + put(&authority, &disk_path, &disk).await; + storage_layer(&authority, "/space", Some("storage-waiting-for-owner")).await; +} + +#[tokio::test] +async fn space_must_come_from_the_current_owner_instead_of_a_detached_summary() { + use crowdb_protocol::common::{DiskGroupUsageSummary, OwnerMapValue}; + let authority = TestPreparationAuthority::start(&[2]).await; + ready_disks(&authority).await; + let now = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + put( + &authority, + "/hw/dg_owner/4/2/1", + &OwnerMapValue { + instance_id: 42, + lease_expiry_ms: now + 120_000, + ..Default::default() + }, + ) + .await; + let registry = crowdb_kv_client::ServiceRegistryClient::from_shared(authority.ctx.kv_arc().clone()); + registry + .register_diskdb(42, "127.0.0.1:7200", &[1], &[]) + .await + .unwrap(); + let usage = DiskGroupUsageSummary { + disk_group_id: 1, + usage_known: true, + sampled_at_ms: now, + allocatable_disk_count: 1, + allocatable_free_bytes: 4096, + ..Default::default() + }; + registry + .register_diskdb(43, "127.0.0.1:7201", &[1], std::slice::from_ref(&usage)) + .await + .unwrap(); + put(&authority, "/hw/dg_usage/1", &usage).await; + storage_layer(&authority, "/owner", None).await; + storage_layer(&authority, "/space", Some("storage-space-missing")).await; + registry + .heartbeat_diskdb(42, "127.0.0.1:7200", &[1], &[usage]) + .await + .unwrap(); + storage_layer(&authority, "/space", None).await; +} + +#[tokio::test] +async fn diskless_member_does_not_require_an_unused_storage_reservation() { + use crowdb_console_shared::deployment::registry::{self, NodeRecord, NodeRegistry}; + use crowdb_protocol::mgmt::{cluster::DependencyKind, node::setup::SetupStorage}; + let authority = TestPreparationAuthority::start(&[2]).await; + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + let mut node = local(&authority); + node.policy.setup.storage = SetupStorage::Diskless; + node.observed_at_unix_ms = u64::try_from( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(), + ) + .unwrap(); + registry::publish( + authority.ctx.kv(), + &NodeRegistry { + cluster_id: authority.bootstrap.identity.cluster_id.clone(), + nodes: vec![NodeRecord { + discovery_id: node.discovery_id.clone(), + node_id: 2, + rack_id: 4, + physical_host_id: "owned".into(), + host: "node-2".into(), + ssh_port: 2222, + ssh_user: "crowdb".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + confirmed: true, + host_key: None, + cancelled: false, + }], + }, + ) + .await + .unwrap(); + let report = collect(&authority.ctx, node).await.unwrap(); + assert!(report + .observation + .dependencies + .iter() + .any(|fact| fact.id == "node/2" && fact.state == DependencyState::Ready)); + assert!( + !report + .observation + .dependencies + .iter() + .any(|fact| fact.kind == DependencyKind::Storage), + "Diskless is an accepted choice, not missing disk evidence: {:?}", + report.observation.dependencies + ); +} diff --git a/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs b/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs new file mode 100644 index 000000000..276b5ee80 --- /dev/null +++ b/lib/crowdb-console-shared/tests/cluster_supplied_storage_observation_test.rs @@ -0,0 +1,282 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/default_preparation.rs"] +mod common; +use common::TestPreparationAuthority; +use crowdb_console_shared::{ + deployment::{ + preparation, + registry::{self, NodeRecord, NodeRegistry}, + }, + ops::cluster::readiness::collect, +}; +use crowdb_protocol::{ + common::{BindMapValue, DiskGroupUsageSummary, DiskId, HwStatus, OwnerMapValue}, + diskdb::rpc::{DiskGroupValue, DiskValue}, + key::{DiskGroupKey, DiskKey, TextKey}, + mgmt::{ + cluster::{DependencyKind, DependencyState}, + node::{ + deployment::NodeDeploymentPolicy, + observation::NodeRuntimeObservation, + setup::{ + NodeSetupPhase, NodeSetupStatus, SetupDiskStatus, SetupMedium, SetupStepPhase, SetupStorage, + SetupStorageMode, + }, + storage::DiskBackend, + NodeBinding, + }, + }, + DiskIdExt, +}; + +fn now() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() + .try_into() + .unwrap() +} + +async fn put(authority: &TestPreparationAuthority, path: &str, value: &impl serde::Serialize) { + authority + .ctx + .kv() + .put(0, 0, path.as_bytes(), &serde_json::to_vec(value).unwrap(), None) + .await + .unwrap(); +} + +async fn setup() -> (TestPreparationAuthority, NodeRuntimeObservation) { + let authority = TestPreparationAuthority::start(&[2]).await; + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + let discovery_id = uuid::Uuid::new_v4().to_string(); + let operation_id = uuid::Uuid::new_v4().to_string(); + registry::publish( + authority.ctx.kv(), + &NodeRegistry { + cluster_id: authority.bootstrap.identity.cluster_id.clone(), + nodes: vec![NodeRecord { + discovery_id: discovery_id.clone(), + node_id: 2, + physical_host_id: "owned-host".into(), + rack_id: 4, + host: "node-2".into(), + ssh_port: 2222, + ssh_user: "crowdb".into(), + operation_id: operation_id.clone(), + confirmed: true, + host_key: None, + cancelled: false, + }], + }, + ) + .await + .unwrap(); + let mut policy = NodeDeploymentPolicy::default(); + let mut statuses = Vec::new(); + let mut media = Vec::new(); + let mut usage = Vec::new(); + for group in [7, 9] { + let (medium, status, summary) = test_medium(&authority, group).await; + media.push(medium); + statuses.push(status); + usage.push(summary); + } + crowdb_kv_client::ServiceRegistryClient::from_shared(authority.ctx.kv_arc().clone()) + .register_diskdb(42, "127.0.0.1:7200", &[7, 9], &usage) + .await + .unwrap(); + policy.setup.storage = SetupStorage::Supplied { disks: media }; + policy.validate().unwrap(); + let node = NodeRuntimeObservation { + version: 1, + discovery_id: discovery_id.clone(), + policy, + binding: Some(NodeBinding { + node_id: 2, + bootstrap: authority.bootstrap.identity.clone(), + management_seeds: vec![], + }), + setup: Some(NodeSetupStatus { + version: 1, + discovery_id, + operation_id, + phase: NodeSetupPhase::Ready, + storage_mode: Some(SetupStorageMode::Supplied), + node_id: Some(2), + cluster_id: Some(authority.bootstrap.identity.cluster_id.clone()), + disks: statuses, + services: vec![], + error: None, + }), + observed_at_unix_ms: now(), + services: vec![], + protocols: vec![], + }; + (authority, node) +} + +async fn test_medium( + authority: &TestPreparationAuthority, + group: u64, +) -> (SetupMedium, SetupDiskStatus, DiskGroupUsageSummary) { + let id = DiskId { high: 17, low: group }; + let path = format!("/data/disks/{group}"); + let medium = SetupMedium { + path: path.clone(), + backend: DiskBackend::File, + backing_identity: format!("file:{group}"), + resource_identity: Some(format!("file:1:{group}")), + capacity_bytes: 64 * 1024 * 1024, + media_type: None, + initialize: true, + }; + let status = SetupDiskStatus { + path: path.clone(), + phase: SetupStepPhase::Ready, + detail: None, + operation_id: uuid::Uuid::new_v4().to_string(), + disk_id: Some(id.to_display_string()), + }; + put( + authority, + &DiskGroupKey { + rack_id: 4, + node_id: 2, + disk_group_id: group, + } + .to_path(), + &DiskGroupValue { + status: HwStatus::Up as i32, + disk_ids: vec![id], + ..Default::default() + }, + ) + .await; + put( + authority, + &DiskKey { + rack_id: 4, + node_id: 2, + disk_group_id: group, + disk_id: id, + } + .to_path(), + &DiskValue { + status: HwStatus::Up as i32, + capacity_units: 64, + device_path: path, + ..Default::default() + }, + ) + .await; + put( + authority, + &format!("/hw/dg_bind/4/2/{group}"), + &BindMapValue { + store_id: 0, + group_id: 1, + }, + ) + .await; + put( + authority, + &format!("/hw/dg_owner/4/2/{group}"), + &OwnerMapValue { + instance_id: 42, + lease_expiry_ms: now() + 120_000, + ..Default::default() + }, + ) + .await; + let usage = DiskGroupUsageSummary { + disk_group_id: group, + usage_known: true, + sampled_at_ms: now(), + allocatable_disk_count: 1, + allocatable_free_bytes: 4096, + allocatable_capacity_bytes: 8192, + ..Default::default() + }; + (medium, status, usage) +} + +#[tokio::test] +async fn supplied_storage_checks_every_actual_group_and_never_requires_the_automatic_reservation() { + let (authority, node) = setup().await; + let _ = authority.ctx.kv().drain_window(); + let report = collect(&authority.ctx, node.clone()).await.unwrap(); + let facts: Vec<_> = report + .observation + .dependencies + .iter() + .filter(|fact| fact.kind == DependencyKind::Storage) + .collect(); + assert_eq!( + facts.len(), + 6, + "two accepted media classes require disk, owner and space evidence each" + ); + assert!( + facts.iter().all(|fact| fact.state == DependencyState::Ready), + "{facts:?}" + ); + assert!(!facts.iter().any(|fact| fact.id.starts_with("storage/4/2/1"))); + let work = authority.ctx.kv().drain_window(); + assert_eq!( + work.put.len() + work.delete.len() + work.batch_write.len(), + 0, + "status must never repair placement" + ); + let key = DiskKey { + rack_id: 4, + node_id: 2, + disk_group_id: 9, + disk_id: DiskId { high: 17, low: 9 }, + }; + put( + &authority, + &key.to_path(), + &DiskValue { + status: HwStatus::Offline as i32, + capacity_units: 64, + ..Default::default() + }, + ) + .await; + let report = collect(&authority.ctx, node).await.unwrap(); + assert!(report + .observation + .dependencies + .iter() + .any(|fact| fact.id == "storage/4/2/9" && fact.state != DependencyState::Ready)); +} + +#[tokio::test] +async fn missing_or_duplicate_supplied_identity_and_conflicting_setup_never_become_ready() { + let (authority, node) = setup().await; + for mutation in 0..4 { + let mut changed = node.clone(); + let setup = changed.setup.as_mut().unwrap(); + match mutation { + 0 => setup.disks[1].disk_id = None, + 1 => setup.disks[1].disk_id = setup.disks[0].disk_id.clone(), + 2 => setup.cluster_id = Some(uuid::Uuid::new_v4().to_string()), + _ => setup.disks[1].phase = SetupStepPhase::Failed, + } + let report = collect(&authority.ctx, changed).await.unwrap(); + assert!( + report + .observation + .dependencies + .iter() + .any(|fact| fact.kind == DependencyKind::Storage && fact.state != DependencyState::Ready), + "mutation {mutation}" + ); + } +} diff --git a/lib/crowdb-console-shared/tests/cluster_target_test.rs b/lib/crowdb-console-shared/tests/cluster_target_test.rs new file mode 100644 index 000000000..d414da519 --- /dev/null +++ b/lib/crowdb-console-shared/tests/cluster_target_test.rs @@ -0,0 +1,172 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/cluster_target.rs"] +mod common; + +use common::{identity, TestTargetServer}; +use crowdb_console_shared::deployment::target::{ + resolve, ClusterTargetBinding, RuntimeOwnership, RuntimeSelection, TargetError, +}; +use std::sync::atomic::Ordering; + +fn binding(server: &TestTargetServer, id: u128) -> ClusterTargetBinding { + ClusterTargetBinding { + version: 1, + name: "demo".into(), + cluster_id: uuid::Uuid::from_u128(id).to_string(), + management_endpoints: vec![server.origin.clone()], + runtime: None, + } +} + +#[tokio::test] +async fn endpoint_only_attach_observes_identity_without_runtime_ownership() { + let server = TestTargetServer::start(identity(1)).await; + let selected = resolve(None, &[server.origin.clone(), server.origin.clone()], None) + .await + .unwrap(); + assert_eq!(selected.identity, identity(1)); + assert_eq!(selected.management_endpoints, vec![server.origin.clone()]); + assert!(selected.runtime.is_none()); + assert_eq!(server.requests.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn named_binding_and_explicit_addresses_must_observe_the_same_uuid() { + let original = TestTargetServer::start(identity(1)).await; + let replacement = TestTargetServer::start(identity(2)).await; + let binding = binding(&original, 1); + assert!(resolve(Some(&binding), &[], None).await.is_ok()); + assert!(matches!( + resolve(Some(&binding), std::slice::from_ref(&replacement.origin), None).await, + Err(TargetError::IdentityConflict) + )); + // Explicit addresses replace the saved address list, but never its UUID. + assert_eq!(original.requests.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn mixed_uuid_endpoints_fail_even_without_a_named_binding() { + let left = TestTargetServer::start(identity(1)).await; + let right = TestTargetServer::start(identity(2)).await; + assert!(matches!( + resolve(None, &[left.origin.clone(), right.origin.clone()], None).await, + Err(TargetError::IdentityConflict) + )); + assert_eq!(left.requests.load(Ordering::Relaxed), 1); + assert_eq!(right.requests.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn runtime_endpoint_or_context_change_fails_before_contacting_application() { + let server = TestTargetServer::start(identity(1)).await; + let mut binding = binding(&server, 1); + let selection = RuntimeSelection { + runtime: "docker".into(), + endpoint: "unix:///var/run/docker.sock".into(), + context: "default".into(), + }; + binding.runtime = Some(RuntimeOwnership { + selection: selection.clone(), + owner: uuid::Uuid::new_v4().to_string(), + }); + for replacement in [ + Some(RuntimeSelection { + context: "remote".into(), + ..selection.clone() + }), + Some(RuntimeSelection { + endpoint: "ssh://remote".into(), + ..selection.clone() + }), + ] { + assert!(matches!( + resolve(Some(&binding), &[], replacement.as_ref()).await, + Err(TargetError::RuntimeConflict) + )); + } + assert_eq!(server.requests.load(Ordering::Relaxed), 0); + // Application attachment remains HTTP-only even for a runtime-owned binding. + let attached = resolve(Some(&binding), &[], None).await.unwrap(); + assert_eq!(attached.runtime, binding.runtime); + let selected = resolve(Some(&binding), &[], Some(&selection)).await.unwrap(); + assert_eq!(selected.runtime, binding.runtime); +} + +#[tokio::test] +async fn missing_and_malformed_targets_never_fall_back_to_localhost() { + assert!(matches!( + resolve(None, &[], None).await, + Err(TargetError::Missing) + )); + for origin in [ + "http://user:secret@127.0.0.1:9090", + "http://127.0.0.1:9090/api", + "file:///tmp/target", + "http://127.0.0.1:9090?token=secret", + "http://127.0.0.1:0", + ] { + let error = resolve(None, &[origin.into()], None).await.unwrap_err(); + assert!(matches!(error, TargetError::Invalid)); + assert!(!error.to_string().contains("secret")); + } +} + +#[tokio::test] +async fn unreachable_explicit_target_never_uses_a_reachable_saved_address() { + let server = TestTargetServer::start(identity(1)).await; + let binding = binding(&server, 1); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let closed = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + assert!(matches!( + resolve(Some(&binding), &[closed], None).await, + Err(TargetError::Unavailable) + )); + assert_eq!(server.requests.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn malformed_identity_or_native_address_is_not_accepted() { + for case in 0..5 { + let mut value = identity(1); + match case { + 0 => value.version = 2, + 1 => value.cluster_id = uuid::Uuid::nil().to_string(), + 2 => value.group0_management_endpoints = vec![], + 3 => value.group0_rpc_endpoints = vec!["http://127.0.0.1:10100".into()], + _ => value.group0_rpc_endpoints = vec!["127.0.0.1:10100/path".into()], + } + let server = TestTargetServer::start(value).await; + assert!( + matches!( + resolve(None, std::slice::from_ref(&server.origin), None).await, + Err(TargetError::Invalid) + ), + "case {case}" + ); + } +} + +#[tokio::test] +async fn http_attach_and_native_transport_reachability_are_separate() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut value = identity(1); + value.group0_rpc_endpoints = vec![listener.local_addr().unwrap().to_string()]; + let server = TestTargetServer::start(value).await; + let selected = resolve(None, std::slice::from_ref(&server.origin), None) + .await + .unwrap(); + selected.require_native_transport().await.unwrap(); + // Drain the prior connection so its close cannot affect this assertion. + drop(listener.accept().await.unwrap()); + drop(listener); + assert!(matches!( + selected.require_native_transport().await, + Err(TargetError::RpcUnreachable) + )); + assert!(resolve(None, std::slice::from_ref(&server.origin), None) + .await + .is_ok()); +} diff --git a/lib/crowdb-console-shared/tests/common/cluster_target.rs b/lib/crowdb-console-shared/tests/common/cluster_target.rs new file mode 100644 index 000000000..9b3540ab6 --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/cluster_target.rs @@ -0,0 +1,58 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{routing::get, Json, Router}; +use crowdb_protocol::mgmt::cluster::ClusterTargetIdentity; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +pub struct TestTargetServer { + pub origin: String, + pub requests: Arc, + worker: tokio::task::JoinHandle<()>, +} + +impl TestTargetServer { + pub async fn start(identity: ClusterTargetIdentity) -> Self { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(AtomicUsize::new(0)); + let counter = Arc::clone(&requests); + let router = Router::new().route( + "/api/cluster/target", + get(move || { + let counter = Arc::clone(&counter); + let identity = identity.clone(); + async move { + counter.fetch_add(1, Ordering::Relaxed); + Json(identity) + } + }), + ); + let worker = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + Self { + origin, + requests, + worker, + } + } +} + +impl Drop for TestTargetServer { + fn drop(&mut self) { + self.worker.abort(); + } +} + +pub fn identity(id: u128) -> ClusterTargetIdentity { + ClusterTargetIdentity { + version: 1, + cluster_id: uuid::Uuid::from_u128(id).to_string(), + group0_management_endpoints: vec!["http://127.0.0.1:10000".into()], + group0_rpc_endpoints: vec!["127.0.0.1:10100".into()], + } +} diff --git a/lib/crowdb-console-shared/tests/common/default_metadata.rs b/lib/crowdb-console-shared/tests/common/default_metadata.rs new file mode 100644 index 000000000..c47c0a884 --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/default_metadata.rs @@ -0,0 +1,105 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::default_preparation::TestPreparationAuthority; +use crowdb_console_shared::deployment::preparation::{self, DefaultPreparation}; +use crowdb_kv::{ + cluster::{KvServer, PxKvStore}, + common::config::CrowDBConfig, +}; +use crowdb_kv_server::{ + mgmt::{self, operation_registry::AppState}, + store_registry::KvStoreRegistry, +}; +use crowdb_protocol::common::KvServerIdentity; +use std::sync::Arc; + +pub struct TestDefaultMetadata { + pub authority: TestPreparationAuthority, + pub accepted: DefaultPreparation, + pub stores: Vec>, + pub endpoints: Vec, + servers: Vec>, + _root: crowdb_test_harness::test_dirs::TestDir, +} + +impl TestDefaultMetadata { + pub async fn start(members: &[u64]) -> Self { + let authority = TestPreparationAuthority::start(members).await; + let accepted = preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("default-metadata-nodes"); + let mut stores = Vec::new(); + let mut servers = Vec::new(); + let mut endpoints = Vec::new(); + for (index, node) in members.iter().enumerate() { + let path = root.path().join(format!("node-{node}")); + let config = CrowDBConfig { + config_root: path.join("config"), + data_root: path.join("data"), + wal_root: path.join("wal"), + wal_backend: "file".into(), + crowtree_backend: "file".into(), + ..CrowDBConfig::for_tests() + }; + let registry = Arc::new(KvStoreRegistry::with_config(config)); + let store = if index == 0 { + Arc::clone(&authority.store) + } else { + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + store.start().await.unwrap(); + store + }; + registry.add_store(0, &store); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let origin = format!("http://{}", listener.local_addr().unwrap()); + let router = mgmt::router(AppState::new(registry)); + servers.push(tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + })); + authority + .ctx + .sysmd() + .register_kv_server( + KvServerIdentity { + instance_id: *node, + node_id: Some(*node), + }, + &origin, + &[0], + &[], + "ok", + "", + ) + .await + .unwrap(); + endpoints.push(origin); + stores.push(store); + } + Self { + authority, + accepted, + stores, + endpoints, + servers, + _root: root, + } + } + + pub async fn ensure(&self) -> crowdb_console_shared::error::Result { + crowdb_console_shared::ops::kv_logical::ensure_default_group(&self.authority.ctx, &self.accepted) + .await + } +} + +impl Drop for TestDefaultMetadata { + fn drop(&mut self) { + for server in &self.servers { + server.abort(); + } + for store in &self.stores { + store.stop(); + } + } +} diff --git a/lib/crowdb-console-shared/tests/common/default_preparation.rs b/lib/crowdb-console-shared/tests/common/default_preparation.rs new file mode 100644 index 000000000..de5954657 --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/default_preparation.rs @@ -0,0 +1,75 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{ + config::{NodeEntry, RackEntry, ServerEntry}, + deployment::{PreparedBootstrap, CLUSTER_KEY}, + ops::OpContext, + ConsoleConfig, +}; +use crowdb_kv::cluster::{group::PxGroup, KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; +use std::sync::Arc; + +pub struct TestPreparationAuthority { + pub ctx: OpContext, + pub bootstrap: PreparedBootstrap, + pub store: Arc, + _root: crowdb_test_harness::test_dirs::TestDir, +} + +impl TestPreparationAuthority { + pub async fn start(members: &[u64]) -> Self { + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new( + 0, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + store.start().await.unwrap(); + let endpoint = store.listen_addr().unwrap().to_string(); + let mut config = ConsoleConfig { + racks: vec![RackEntry { + id: 4, + name: "local".into(), + }], + ..Default::default() + }; + for member in members { + config.nodes.push(NodeEntry { + id: *member, + rack_id: 4, + host: format!("node-{member}"), + ssh_port: 2222, + ssh_user: "crowdb".into(), + ssh_credential_ref: None, + ssh_key: None, + ssh_password: None, + }); + let mut server = ServerEntry::new(format!("kv-{member}"), format!("http://node-{member}:10000")); + server.node_id = Some(*member); + server.rpc_url = Some(format!("node-{member}:10100")); + config.servers.push(server); + } + let root = crowdb_test_harness::test_dirs::tempdir_in_test_data("default-preparation"); + let bootstrap = + PreparedBootstrap::open(&root.path().join("prepared.json"), &config, members).unwrap(); + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![endpoint.clone()]))); + kv.seed_leader(0, 0, endpoint); + kv.put(0, 0, CLUSTER_KEY, &serde_json::to_vec(&bootstrap).unwrap(), None) + .await + .unwrap(); + let ctx = OpContext::with_shared_client_preserving_hint(kv, &[], config); + Self { + ctx, + bootstrap, + store, + _root: root, + } + } +} + +impl Drop for TestPreparationAuthority { + fn drop(&mut self) { + self.store.stop(); + } +} diff --git a/lib/crowdb-console-shared/tests/common/deployment_group_replay.rs b/lib/crowdb-console-shared/tests/common/deployment_group_replay.rs new file mode 100644 index 000000000..de496bf03 --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/deployment_group_replay.rs @@ -0,0 +1,115 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use axum::{extract::State, http::StatusCode, routing::get, Json, Router}; +use crowdb_console_shared::ops::kv_logical::create_local_group_for_tests; +use crowdb_protocol::mgmt::{AddGroupRequest, GroupCreationStatus, GroupDeploymentIdentity}; +use serde_json::{json, Value}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +struct TestReplayState { + owner: Option, + lost_reply: bool, + lists: AtomicUsize, + posts: AtomicUsize, +} + +pub struct TestDeploymentReplay { + endpoint: String, + state: Arc, + server: tokio::task::JoinHandle<()>, +} + +impl TestDeploymentReplay { + pub async fn start(owner: Option, lost_reply: bool) -> Self { + let state = Arc::new(TestReplayState { + owner, + lost_reply, + lists: AtomicUsize::new(0), + posts: AtomicUsize::new(0), + }); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let router = Router::new() + .route("/stores/0/groups", get(list).post(create)) + .route("/stores/0/groups/1/creation", get(ownership)) + .with_state(state.clone()); + let server = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + Self { + endpoint, + state, + server, + } + } + + pub async fn replay( + &self, + deployment: GroupDeploymentIdentity, + ) -> crowdb_console_shared::error::Result { + create_local_group_for_tests( + &self.endpoint, + 0, + &AddGroupRequest { + group_id: 1, + replica_id: 7, + initial_role: None, + start_election: Some(false), + deployment: Some(deployment), + }, + ) + .await + } + + pub fn posts(&self) -> usize { + self.state.posts.load(Ordering::Relaxed) + } +} + +impl Drop for TestDeploymentReplay { + fn drop(&mut self) { + self.server.abort(); + } +} + +async fn list(State(state): State>) -> Json { + let first = state.lists.fetch_add(1, Ordering::Relaxed) == 0; + Json(if first && state.lost_reply { + json!([]) + } else { + json!([{"group_id":1,"local_replica_id":7,"leader_id":0,"remote_count":0}]) + }) +} + +async fn create(State(state): State>) -> (StatusCode, Json) { + state.posts.fetch_add(1, Ordering::Relaxed); + ( + StatusCode::BAD_GATEWAY, + Json(json!({"error":"upstream response lost after creation"})), + ) +} + +async fn ownership(State(state): State>) -> (StatusCode, Json) { + state.owner.as_ref().map_or_else( + || (StatusCode::NOT_FOUND, Json(json!({}))), + |owner| (StatusCode::OK, Json(serde_json::to_value(owner).unwrap())), + ) +} + +pub fn owner() -> GroupCreationStatus { + GroupCreationStatus { + version: 1, + store_id: 0, + group_id: 1, + replica_id: 7, + deployment: Some(GroupDeploymentIdentity { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + }), + retired: false, + } +} diff --git a/lib/crowdb-console-shared/tests/common/member_evidence.rs b/lib/crowdb-console-shared/tests/common/member_evidence.rs new file mode 100644 index 000000000..ed5221eb5 --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/member_evidence.rs @@ -0,0 +1,166 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{ + bootstrap_intent::BootstrapIntent, + config::{NodeEntry, RackEntry, ServerEntry}, + deployment::{registry::NodeRecord, PreparedBootstrap}, + ops::cluster::readiness::member_evidence, + ConsoleConfig, +}; +use crowdb_protocol::mgmt::{ + cluster::{DependencyKind, DependencyObservation, DependencyState}, + node::{ + deployment::NodeDeploymentPolicy, + observation::{NodeRuntimeObservation, NodeServiceObservation}, + NodeBinding, NodeServiceAction, NodeServiceIntent, + }, + SystemBootstrapIdentity, +}; + +pub struct TestMemberEvidence { + pub bootstrap: PreparedBootstrap, + pub expected: NodeRecord, + pub policy: NodeDeploymentPolicy, + pub intents: Vec, + pub node: NodeRuntimeObservation, +} + +impl TestMemberEvidence { + pub fn new() -> Self { + let identity = SystemBootstrapIdentity { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + configuration_digest: "accepted-digest".into(), + }; + let expected = NodeRecord { + discovery_id: uuid::Uuid::new_v4().to_string(), + node_id: 1, + physical_host_id: "test-host".into(), + rack_id: 1, + host: "node-one".into(), + ssh_port: 22, + ssh_user: "crowdb".into(), + operation_id: uuid::Uuid::new_v4().to_string(), + confirmed: true, + host_key: None, + cancelled: false, + }; + let config = Self::config(); + let bootstrap = PreparedBootstrap { + identity: identity.clone(), + intent: BootstrapIntent::capture(&config, &[1]).unwrap(), + nodes: vec![expected.clone()], + }; + let policy = NodeDeploymentPolicy::default(); + let intents: Vec<_> = ["diskdb", "diskio", "chunkdb", "chunk-kv", "access"] + .into_iter() + .map(|kind| NodeServiceIntent { + cluster_id: identity.cluster_id.clone(), + node_id: 1, + service_id: format!("{kind}-1"), + kind: kind.into(), + operation_id: uuid::Uuid::new_v4().to_string(), + action: NodeServiceAction::Start, + configuration: String::new(), + environment: std::collections::BTreeMap::new(), + }) + .collect(); + let mut services: Vec<_> = intents + .iter() + .map(|intent| NodeServiceObservation { + detail: "Service process and readiness match the current executed operation".into(), + service_id: intent.service_id.clone(), + kind: intent.kind.clone(), + operation_id: Some(intent.operation_id.clone()), + state: DependencyState::Ready, + observed_at_unix_ms: 1000, + max_age_ms: 3000, + reason_code: None, + }) + .collect(); + services.push(NodeServiceObservation { + detail: "KV has current supervised process health".into(), + service_id: "kv".into(), + kind: "kv".into(), + operation_id: None, + state: DependencyState::Ready, + observed_at_unix_ms: 1000, + max_age_ms: 3000, + reason_code: None, + }); + let operation = intents.last().unwrap().operation_id.clone(); + let protocols = ["s3", "iceberg", "dataset"] + .into_iter() + .map(|protocol| DependencyObservation { + id: format!("node/1/protocol/{protocol}"), + kind: DependencyKind::Protocol, + detail: format!("{protocol} authenticated request succeeded"), + required: true, + state: DependencyState::Ready, + observed_at_unix_ms: 1000, + max_age_ms: 3000, + revision: None, + operation_id: Some(operation.clone()), + current_operation_id: Some(operation.clone()), + reason: None, + }) + .collect(); + let node = NodeRuntimeObservation { + version: 1, + discovery_id: expected.discovery_id.clone(), + policy: policy.clone(), + binding: Some(NodeBinding { + node_id: 1, + bootstrap: identity, + management_seeds: vec!["http://node-one:10000".into()], + }), + setup: None, + observed_at_unix_ms: 1000, + services, + protocols, + }; + Self { + bootstrap, + expected, + policy, + intents, + node, + } + } + + fn config() -> ConsoleConfig { + let mut server = ServerEntry::new("kv-1", "http://node-one:10000"); + server.node_id = Some(1); + server.rpc_url = Some("node-one:10100".into()); + ConsoleConfig { + racks: vec![RackEntry { + id: 1, + name: "local".into(), + }], + nodes: vec![NodeEntry { + id: 1, + rack_id: 1, + host: "node-one".into(), + ssh_port: 22, + ssh_user: "crowdb".into(), + ssh_key: None, + ssh_password: None, + ssh_credential_ref: None, + }], + servers: vec![server], + ..Default::default() + } + } + + pub fn project(&self) -> Vec { + member_evidence( + &self.bootstrap, + &self.expected, + &self.policy, + Some(&self.intents), + Some(&self.node), + 1000, + ) + } +} diff --git a/lib/crowdb-console-shared/tests/common/member_recovery.rs b/lib/crowdb-console-shared/tests/common/member_recovery.rs new file mode 100644 index 000000000..e95da684e --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/member_recovery.rs @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::{ + config::NodeEntry, + deployment::recovery::{MemberTrust, TrustedMember}, +}; +use crowdb_protocol::mgmt::{ + node::{deployment::NodeDeploymentPolicy, observation::NodeRuntimeObservation, NodeBinding}, + SystemBootstrapIdentity, +}; +use russh::keys::{key::KeyPair, PublicKeyBase64}; +use std::path::Path; + +pub fn host_key(key: &KeyPair) -> String { + format!( + "ssh-ed25519 {}", + key.clone_public_key().unwrap().public_key_base64() + ) +} + +pub fn trust(key: &KeyPair) -> MemberTrust { + MemberTrust { + version: 1, + bootstrap: SystemBootstrapIdentity { + cluster_id: uuid::Uuid::new_v4().to_string(), + operation_id: uuid::Uuid::new_v4().to_string(), + configuration_digest: "a".repeat(64), + }, + members: vec![TrustedMember { + discovery_id: uuid::Uuid::new_v4().to_string(), + node_id: 1, + admission_operation_id: uuid::Uuid::new_v4().to_string(), + host_key: host_key(key), + }], + } +} + +pub fn observation(trust: &MemberTrust) -> NodeRuntimeObservation { + NodeRuntimeObservation { + version: 1, + discovery_id: trust.members[0].discovery_id.clone(), + policy: NodeDeploymentPolicy::default(), + binding: Some(NodeBinding { + node_id: 1, + bootstrap: trust.bootstrap.clone(), + management_seeds: vec!["http://old-address:10000".into()], + }), + setup: None, + observed_at_unix_ms: 0, + services: vec![], + protocols: vec![], + } +} + +pub fn reply(observation: &NodeRuntimeObservation) -> Vec { + serde_json::to_vec(observation).unwrap() +} + +pub fn node(port: u16, path: &Path, key: &KeyPair) -> NodeEntry { + russh::keys::encode_pkcs8_pem(key, std::fs::File::create(path).unwrap()).unwrap(); + NodeEntry { + id: 1, + rack_id: 1, + host: "127.0.0.1".into(), + ssh_port: port, + ssh_user: "operator".into(), + ssh_key: Some(path.to_string_lossy().into_owned()), + ssh_password: None, + ssh_credential_ref: None, + } +} diff --git a/lib/crowdb-console-shared/tests/common/regression.rs b/lib/crowdb-console-shared/tests/common/regression.rs new file mode 100644 index 000000000..05dfcf35b --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/regression.rs @@ -0,0 +1,300 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +mod inspection; +mod options; +mod ownership; + +use axum::{ + extract::State, + http::{HeaderMap, StatusCode}, + routing::{get, post}, + Json, Router, +}; +use clap::Parser; +use crowdb_console_shared::{ + config::{ConsoleConfig, ServiceType}, + deployment::PreparedBootstrap, + ops::{cluster, OpContext}, +}; +use crowdb_protocol::mgmt::cluster::ClusterTargetIdentity; +use options::TestSetup; +use ownership::{process_start, TestOwnedProcesses}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{self, OpenOptions}, + io::Write, + os::unix::fs::OpenOptionsExt, + path::{Path, PathBuf}, + time::Duration, +}; +use tokio::sync::{mpsc, oneshot}; + +pub type TestResult = Result>; + +#[derive(Parser)] +struct TestFixtureArgs { + #[arg(long)] + root: PathBuf, + #[arg(long)] + state: PathBuf, + #[arg(long)] + owner_pid: u32, + #[arg(last = true)] + setup: Vec, +} + +#[derive(Deserialize)] +#[serde(tag = "action", rename_all = "kebab-case")] +enum TestControl { + Inspect, + Setup { + arguments: Vec, + }, + Reset { + store: u64, + group: u64, + restart_storage: bool, + }, + Stop, +} + +type TestRequest = (TestControl, oneshot::Sender>); + +#[derive(Clone)] +struct TestControlState { + identity: ClusterTargetIdentity, + token: String, + requests: mpsc::Sender, +} + +fn write_private(path: &Path, body: &[u8]) -> TestResult<()> { + if fs::symlink_metadata(path).is_ok_and(|metadata| !metadata.file_type().is_file()) { + return Err("fixture state path must be a regular file".into()); + } + let temporary = path.with_extension("new"); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary)?; + file.write_all(body)?; + file.sync_all()?; + fs::rename(temporary, path)?; + Ok(()) +} + +fn parse_setup(arguments: Vec) -> TestResult { + let setup = TestSetup::try_parse_from(std::iter::once("fixture-setup".to_owned()).chain(arguments))?; + if !matches!(setup.nodes, 1 | 3) { + return Err("fixture requires one or three KV voters".into()); + } + Ok(setup) +} + +async fn setup( + context: &OpContext, + root: &Path, + options: &TestSetup, + owned: &mut TestOwnedProcesses, +) -> TestResult<()> { + let workspace = root.join("services"); + match options.kind.as_str() { + "kv" | "combined" => { + if !context.config().servers.is_empty() { + return Err("KV fixture already exists; stop it before replacing inputs".into()); + } + let (_, members) = + cluster::prepare_local_deploy(context, options.nodes, Some(&workspace), Some(&options.kv())) + .await?; + let config = context.config().clone(); + owned.record(context)?; + let prepared = PreparedBootstrap::open(&root.join("bootstrap.json"), &config, &members)?; + let result = prepared.execute(context).await; + // Bootstrap seals topology and intentionally drops process-local + // fields. This test fixture still owns the native children. + context.config_mut().servers = config.servers; + result?; + if options.kind == "combined" { + cluster::local_deploy_combined_after_kv( + context, + &workspace, + &options.disk(), + &options.chunk(), + &options.diskio_dummy_disk_type, + ) + .await?; + } + } + "diskdb" => { + cluster::local_deploy_diskdb(context, &workspace, &options.disk()).await?; + } + "chunkdb" => { + cluster::local_deploy_chunkdb(context, &workspace, &options.chunk()).await?; + } + _ => return Err("unsupported fixture kind".into()), + } + Ok(()) +} + +fn identity(context: &OpContext, root: &Path) -> TestResult { + let prepared = PreparedBootstrap::load(&root.join("bootstrap.json"))?; + let config = context.config(); + let servers: Vec<_> = config + .servers + .iter() + .filter(|server| server.service_type == ServiceType::PaxosKv) + .collect(); + Ok(ClusterTargetIdentity { + version: 1, + cluster_id: prepared.identity.cluster_id, + group0_management_endpoints: servers.iter().map(|server| server.url.clone()).collect(), + group0_rpc_endpoints: servers + .iter() + .filter_map(|server| server.rpc_url.as_ref()) + .map(|url| url.trim_start_matches("http://").to_owned()) + .collect(), + }) +} + +async fn control( + State(state): State, + headers: HeaderMap, + Json(command): Json, +) -> Result, (StatusCode, String)> { + if headers + .get("X-Crowdb-Fixture-Owner") + .and_then(|value| value.to_str().ok()) + != Some(&state.token) + { + return Err((StatusCode::FORBIDDEN, "fixture ownership token differs".into())); + } + let (reply, received) = oneshot::channel(); + state + .requests + .send((command, reply)) + .await + .map_err(|_| (StatusCode::SERVICE_UNAVAILABLE, "fixture stopped".into()))?; + let value = received + .await + .map_err(|_| (StatusCode::SERVICE_UNAVAILABLE, "fixture stopped".into()))? + .map_err(|error| (StatusCode::BAD_GATEWAY, error))?; + Ok(Json(value)) +} + +async fn command( + context: &OpContext, + root: &Path, + request: TestControl, + owned: &mut TestOwnedProcesses, +) -> TestResult<(bool, serde_json::Value)> { + match request { + TestControl::Inspect => { + return Ok(( + false, + serde_json::json!({"ok": true, + "result": tokio::time::timeout(Duration::from_secs(10), inspection::read(context)).await??}), + )) + } + TestControl::Setup { arguments } => setup(context, root, &parse_setup(arguments)?, owned).await?, + TestControl::Reset { + store, + group, + restart_storage, + } => { + if (store, group) == (0, 0) { + return Err("fixture reset cannot wipe Group 0 authority".into()); + } + let mut result = cluster::clean(context, store, group).await?; + if restart_storage { + result.restarted_services = cluster::restart_storage_services(context).await?; + } + return Ok((false, serde_json::json!({"ok": true, "result": result}))); + } + TestControl::Stop => return Ok((true, serde_json::json!({"ok": true}))), + } + Ok((false, serde_json::json!({"ok": true}))) +} + +#[derive(Serialize)] +struct TestReady<'a> { + endpoint: String, + token: &'a str, + root: &'a Path, + pid: u32, + start: String, +} + +pub async fn run() -> TestResult<()> { + let args = TestFixtureArgs::parse(); + let owner_start = process_start(args.owner_pid).ok_or("fixture owner is not alive")?; + let mut owned = TestOwnedProcesses::new(&args.root)?; + let context = OpContext::new("127.0.0.1:1".into(), vec![], ConsoleConfig::default()); + let result = setup(&context, &args.root, &parse_setup(args.setup)?, &mut owned).await; + owned.record(&context)?; + result?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let endpoint = format!("http://{}", listener.local_addr()?); + let token = uuid::Uuid::new_v4().to_string(); + let (requests, mut receive) = mpsc::channel(8); + let state = TestControlState { + identity: identity(&context, &args.root)?, + token: token.clone(), + requests, + }; + let router = Router::new() + .route( + "/api/cluster/target", + get(|State(state): State| async move { Json(state.identity) }), + ) + .route("/_fixture/control", post(control)) + .with_state(state); + let (stop, stopped) = oneshot::channel(); + let server = tokio::spawn(async move { + axum::serve(listener, router) + .with_graceful_shutdown(async { + let _ = stopped.await; + }) + .await + }); + write_private(&args.state, toml::to_string(&*context.config())?.as_bytes())?; + let ready = TestReady { + endpoint, + token: &token, + root: &args.root, + pid: std::process::id(), + start: process_start(std::process::id()).unwrap(), + }; + write_private( + &args.state.with_extension("fixture.json"), + &serde_json::to_vec(&ready)?, + )?; + let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; + let mut tick = tokio::time::interval(Duration::from_secs(1)); + loop { + tokio::select! { + Some((request, reply)) = receive.recv() => { + let result = command(&context, &args.root, request, &mut owned).await; + owned.record(&context)?; + write_private(&args.state, toml::to_string(&*context.config())?.as_bytes())?; + let stop = result.as_ref().is_ok_and(|(stop, _)| *stop); + let _ = reply.send(result.map(|(_, value)| value).map_err(|error| error.to_string())); + if stop { break; } + } + _ = terminate.recv() => break, + _ = tokio::signal::ctrl_c() => break, + _ = tick.tick() => if process_start(args.owner_pid).as_ref() != Some(&owner_start) { break; }, + } + } + let _ = stop.send(()); + server.await??; + let result = owned.shutdown().await; + write_private( + &args.state.with_extension("fixture-stopped.json"), + &serde_json::to_vec(&serde_json::json!({ + "pid": ready.pid, "start": ready.start, "ok": result.is_ok(), + "error": result.as_ref().err().map(ToString::to_string) + }))?, + )?; + result +} diff --git a/lib/crowdb-console-shared/tests/common/regression/inspection.rs b/lib/crowdb-console-shared/tests/common/regression/inspection.rs new file mode 100644 index 000000000..5f4b9a151 --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/regression/inspection.rs @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::TestResult; +use crowdb_console_shared::ops::OpContext; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_protocol::{ + chunk_kv::{ChunkKvRangeCatalogHead, ChunkKvRangeCatalogPage}, + chunk_stream::{decode_stream_chunk_record, StreamName, StreamRecord}, + key::{ + BinaryKey, ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, StreamChunkRecordKey, + StreamRecordKey, TextKey, + }, +}; +use serde_json::{json, Value}; +use std::collections::BTreeSet; + +async fn get(context: &OpContext, group: u64, key: &[u8]) -> TestResult>> { + match context + .kv() + .get(0, group, key, ReadMode::Linearizable, None) + .await? + { + GetOutcome::Found { value, .. } => Ok(Some(value.to_vec())), + GetOutcome::NotFound => Ok(None), + } +} + +async fn records(context: &OpContext) -> TestResult> { + let outcome = crowdb_console_shared::ops::kv_data::scan(context, 0, 0, b"/chunk-kv/", &[], 256).await?; + Ok(outcome + .items + .into_iter() + .map(|(key, value)| { + json!({ + "key": String::from_utf8_lossy(&key), + "value": serde_json::from_slice::(&value) + .unwrap_or_else(|_| Value::String(String::from_utf8_lossy(&value).into_owned())) + }) + }) + .collect()) +} + +pub async fn read(context: &OpContext) -> TestResult { + let Some(bytes) = get(context, 0, ChunkKvRangeCatalogHeadKey.to_path().as_bytes()).await? else { + return Ok(json!({"catalog_head": null})); + }; + let head: ChunkKvRangeCatalogHead = serde_json::from_slice(&bytes)?; + if head.pages.len() > 128 { + return Err("inspection catalog exceeds its page bound".into()); + } + let mut pages = Vec::new(); + let mut streams = BTreeSet::new(); + for page in &head.pages { + let key = ChunkKvRangeCatalogPageKey { + generation: page.page_generation, + page_index: page.page_index, + }; + let bytes = get(context, 0, key.to_path().as_bytes()) + .await? + .ok_or("catalog page is absent")?; + let page: ChunkKvRangeCatalogPage = serde_json::from_slice(&bytes)?; + for entry in &page.entries { + streams.insert((entry.artifact.stream_name.high, entry.artifact.stream_name.low)); + if let Some(overlay) = &entry.artifact.tail_overlay { + streams.insert((overlay.source_stream_name.high, overlay.source_stream_name.low)); + } + } + if streams.len() > 256 { + return Err("inspection stream count exceeds its bound".into()); + } + pages.push(page); + } + let mut observations = Vec::new(); + for (high, low) in streams { + let name = StreamName { high, low }; + let binding = get(context, 0, &StreamRecordKey { stream_name: name }.to_bytes()).await?; + let binding: Option = + binding.map(|bytes| serde_json::from_slice(&bytes)).transpose()?; + let chunk_record = if let Some(binding) = &binding { + let records = context + .kv() + .scan_reverse( + 0, + binding.metadata_group_id, + &StreamChunkRecordKey::attached_prefix(name), + &[], + &[], + 1, + ReadMode::Linearizable, + None, + false, + None, + ) + .await?; + records + .items + .first() + .map(|(_, value)| decode_stream_chunk_record(value)) + .transpose()? + } else { + None + }; + observations.push(json!({"stream": name, "record": binding, "chunk_record": chunk_record})); + } + Ok( + json!({"catalog_head": head, "pages": pages, "streams": observations, + "records": records(context).await?}), + ) +} diff --git a/lib/crowdb-console-shared/tests/common/regression/options.rs b/lib/crowdb-console-shared/tests/common/regression/options.rs new file mode 100644 index 000000000..8d922aa89 --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/regression/options.rs @@ -0,0 +1,114 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use clap::Parser; +use crowdb_console_shared::ops::cluster::{ + KvDeployTunables, LocalChunkdbDeployConfig, LocalDiskdbDeployConfig, +}; + +#[derive(Clone, Debug, Parser)] +#[allow(clippy::struct_excessive_bools)] // Independent command-line switches for separate services. +pub struct TestSetup { + #[arg(long, value_parser = ["kv", "diskdb", "chunkdb", "combined"])] + pub kind: String, + #[arg(long, default_value_t = 3)] + pub nodes: usize, + #[arg(long)] + rpc_workers: Option, + #[arg(long)] + peer_pool_size: Option, + #[arg(long)] + max_inflight: Option, + #[arg(long)] + coalesce_max_keys: Option, + #[arg(long)] + event_write: bool, + #[arg(long)] + send_queue_capacity: Option, + #[arg(long)] + metrics_interval: Option, + #[arg(long)] + kv_backend: Option, + #[arg(long)] + wal_backend: Option, + #[arg(long)] + no_fsync: bool, + #[arg(long, default_value_t = 1)] + disk_groups_per_node: usize, + #[arg(long, default_value_t = 4)] + disks_per_group: usize, + #[arg(long, default_value_t = 1_099_511_627_776)] + disk_capacity_bytes: u64, + #[arg(long, default_value_t = 274_877_906_944)] + disk_zone_size_bytes: u64, + #[arg(long, default_value_t = 1_048_576)] + disk_unit_size_bytes: u32, + #[arg(long, value_delimiter = ',', default_value = "1")] + data_groups: Vec, + #[arg(long)] + kv_connections: Option, + #[arg(long)] + kv_client_rpc_workers: Option, + #[arg(long)] + free_batch: bool, + #[arg(long)] + free_flush_max_batch: Option, + #[arg(long, default_value_t = 3)] + chunkdb_instances: usize, + #[arg(long)] + diskdb_connections: Option, + #[arg(long)] + diskdb_client_rpc_workers: Option, + #[arg(long)] + diskio_rpc_workers: Option, + #[arg(long, default_value = "null", value_parser = ["null", "mem"])] + pub diskio_dummy_disk_type: String, +} + +impl TestSetup { + pub fn kv(&self) -> KvDeployTunables { + KvDeployTunables { + rpc_workers: self.rpc_workers, + peer_pool_size: self.peer_pool_size, + max_inflight: self.max_inflight, + coalesce_max_keys: self.coalesce_max_keys, + event_write: self.event_write.then_some(true), + send_queue_capacity: self.send_queue_capacity, + metrics_interval: self.metrics_interval, + kv_backend: self.kv_backend.clone(), + wal_backend: self.wal_backend.clone(), + no_fsync: self.no_fsync.then_some(true), + } + } + pub fn disk(&self) -> LocalDiskdbDeployConfig { + LocalDiskdbDeployConfig { + disk_groups_per_node: self.disk_groups_per_node, + disks_per_group: self.disks_per_group, + capacity_bytes: self.disk_capacity_bytes, + zone_size_bytes: self.disk_zone_size_bytes, + unit_size_bytes: self.disk_unit_size_bytes, + data_groups: self.data_groups.clone(), + rpc_workers: self.rpc_workers, + kv_connections: self.kv_connections, + kv_client_rpc_workers: self.kv_client_rpc_workers, + free_batch_enabled: self.free_batch.then_some(true), + free_flush_max_batch: self.free_flush_max_batch, + keepalive_interval_secs: None, + } + } + pub fn chunk(&self) -> LocalChunkdbDeployConfig { + LocalChunkdbDeployConfig { + instance_count: self.chunkdb_instances, + storage_groups: vec![1, 2, 3], + dynamic_ownership: false, + allow_unsafe_ec: false, + rpc_workers: self.rpc_workers, + diskio_rpc_workers: self.diskio_rpc_workers, + kv_connections: self.kv_connections, + kv_client_rpc_workers: self.kv_client_rpc_workers, + diskdb_connections: self.diskdb_connections, + diskdb_client_rpc_workers: self.diskdb_client_rpc_workers, + metrics_interval: self.metrics_interval, + } + } +} diff --git a/lib/crowdb-console-shared/tests/common/regression/ownership.rs b/lib/crowdb-console-shared/tests/common/regression/ownership.rs new file mode 100644 index 000000000..3c8aff921 --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/regression/ownership.rs @@ -0,0 +1,123 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::TestResult; +use crowdb_console_shared::{lifecycle, ops::OpContext}; +use crowdb_protocol::port::namespace::RuntimeNamespace; +use std::{collections::BTreeMap, fs, path::Path, time::Duration}; + +pub fn process_start(pid: u32) -> Option { + #[cfg(target_os = "linux")] + { + let stat = fs::read_to_string(format!("/proc/{pid}/stat")).ok()?; + let fields: Vec<_> = stat.rsplit_once(')')?.1.split_whitespace().collect(); + if fields[0] == "Z" { + return None; + } + fields.get(19).map(|value| (*value).to_owned()) + } + #[cfg(not(target_os = "linux"))] + { + let output = std::process::Command::new("ps") + .args(["-p", &pid.to_string(), "-o", "lstart="]) + .output() + .ok()?; + let value = String::from_utf8(output.stdout).ok()?; + (!value.trim().is_empty()).then(|| value.trim().to_owned()) + } +} + +pub struct TestOwnedProcesses { + processes: BTreeMap, + namespace: Option, +} + +impl TestOwnedProcesses { + pub fn new(root: &Path) -> TestResult { + if fs::read_dir(root)?.next().is_some() { + return Err("regression fixture requires a fresh empty owned directory".into()); + } + Ok(Self { + processes: BTreeMap::new(), + namespace: Some(RuntimeNamespace::persistent(root, "regression-fixture")?), + }) + } + pub fn record(&mut self, context: &OpContext) -> TestResult<()> { + for server in &context.config().servers { + if let Some(pid) = server.pid { + if let Some(start) = process_start(pid) { + self.namespace.as_mut().unwrap().record_process(pid)?; + self.processes.insert(pid, start); + } + } + } + Ok(()) + } + pub fn stop(&mut self) -> TestResult<()> { + let mut errors = Vec::new(); + for (&pid, start) in self.processes.iter().rev() { + if process_start(pid).as_ref() == Some(start) { + if let Err(error) = lifecycle::stop_pid_with_timeout(pid, Duration::from_secs(5)) { + errors.push(format!("stop owned fixture process {pid}: {error}")); + } + } + } + if let Some(namespace) = self.namespace.take() { + if let Err(error) = namespace.release() { + errors.push(format!("release fixture namespace: {error}")); + } + } + if let Err(error) = crowdb_protocol::port::namespace::release_process_ports() { + errors.push(format!("release fixture process ports: {error}")); + } + if !errors.is_empty() { + return Err(errors.join("\n").into()); + } + self.processes.clear(); + Ok(()) + } + + pub async fn shutdown(&mut self) -> TestResult<()> { + let processes = self.processes.clone(); + self.stop()?; + // A zombie is stopped but still owns its Tokio child registration. + // Keep the runtime alive until its reaper tasks release those handles. + tokio::time::timeout(Duration::from_secs(5), async { + loop { + #[cfg(target_os = "linux")] + let pending = processes.iter().any(|(pid, start)| { + fs::read_to_string(format!("/proc/{pid}/stat")) + .ok() + .and_then(|stat| { + stat.rsplit_once(')')? + .1 + .split_whitespace() + .nth(19) + .map(str::to_owned) + }) + .as_ref() + == Some(start) + }); + #[cfg(not(target_os = "linux"))] + let pending = processes + .iter() + .any(|(pid, start)| process_start(*pid).as_ref() == Some(start)); + if !pending { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .map_err(|_| "owned measurement child was not reaped before runtime shutdown")?; + Ok(()) + } +} + +impl Drop for TestOwnedProcesses { + fn drop(&mut self) { + if let Err(error) = self.stop() { + eprintln!("owned fixture cleanup failed: {error}"); + } + } +} diff --git a/lib/crowdb-console-shared/tests/common/regression_fixture.rs b/lib/crowdb-console-shared/tests/common/regression_fixture.rs new file mode 100644 index 000000000..99185e9c2 --- /dev/null +++ b/lib/crowdb-console-shared/tests/common/regression_fixture.rs @@ -0,0 +1,11 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Explicit test-owned native environment for regression measurement scripts. + +mod regression; + +#[tokio::main] +async fn main() -> regression::TestResult<()> { + regression::run().await +} diff --git a/lib/crowdb-console-shared/tests/common/ssh_server.rs b/lib/crowdb-console-shared/tests/common/ssh_server.rs index 9c55d25ad..276f2dbb3 100644 --- a/lib/crowdb-console-shared/tests/common/ssh_server.rs +++ b/lib/crowdb-console-shared/tests/common/ssh_server.rs @@ -8,6 +8,13 @@ use russh::keys::key::{KeyPair, PublicKey}; use russh::server::{Auth, Handler, Msg, Session}; use russh::{Channel, ChannelId, CryptoVec}; +#[derive(Clone)] +#[allow(dead_code)] +pub enum TestReply { + Stdout(Vec), + Failure(String), +} + pub struct TestSshServer { pub port: u16, server: tokio::task::JoinHandle<()>, @@ -19,12 +26,29 @@ impl Drop for TestSshServer { } } +#[allow(dead_code)] impl TestSshServer { pub async fn start(key: PublicKey) -> Self { + Self::start_with_reply(key, None).await + } + + pub async fn start_with_reply(key: PublicKey, reply: Option>) -> Self { + Self::start_with_host_key(key, reply, KeyPair::generate_ed25519().unwrap()).await + } + + pub async fn start_with_host_key(key: PublicKey, reply: Option>, host: KeyPair) -> Self { + Self::start_server(key, reply.map(TestReply::Stdout), host).await + } + + pub async fn start_with_result(key: PublicKey, reply: TestReply) -> Self { + Self::start_server(key, Some(reply), KeyPair::generate_ed25519().unwrap()).await + } + + async fn start_server(key: PublicKey, reply: Option, host: KeyPair) -> Self { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); let config = Arc::new(russh::server::Config { - keys: vec![KeyPair::generate_ed25519().unwrap()], + keys: vec![host], ..Default::default() }); let server = tokio::spawn(async move { @@ -34,7 +58,7 @@ impl TestSshServer { accepted = listener.accept() => { let (stream, _) = accepted.unwrap(); let config = config.clone(); - let handler = TestHandler { key: key.clone() }; + let handler = TestHandler { key: key.clone(), reply: reply.clone() }; sessions.spawn(async move { if let Ok(session) = russh::server::run_stream(config, stream, handler).await { let _ = session.await; @@ -51,6 +75,7 @@ impl TestSshServer { struct TestHandler { key: PublicKey, + reply: Option, } #[async_trait] @@ -78,6 +103,21 @@ impl Handler for TestHandler { session: &mut Session, ) -> Result<(), Self::Error> { session.channel_success(channel); + if let Some(reply) = &self.reply { + match reply { + TestReply::Stdout(bytes) => { + session.data(channel, CryptoVec::from_slice(bytes)); + session.exit_status_request(channel, 0); + } + TestReply::Failure(error) => { + session.extended_data(channel, 1, CryptoVec::from_slice(error.as_bytes())); + session.exit_status_request(channel, 1); + } + } + session.eof(channel); + session.close(channel); + return Ok(()); + } let output = tokio::process::Command::new("/bin/sh") .args(["-c", &String::from_utf8_lossy(data)]) .output() diff --git a/lib/crowdb-console-shared/tests/default_metadata_test.rs b/lib/crowdb-console-shared/tests/default_metadata_test.rs new file mode 100644 index 000000000..4efc8379c --- /dev/null +++ b/lib/crowdb-console-shared/tests/default_metadata_test.rs @@ -0,0 +1,276 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/default_metadata.rs"] +mod default_metadata; +#[path = "common/default_preparation.rs"] +mod default_preparation; + +use crowdb_console_shared::deployment::preparation; +use crowdb_kv::cluster::{group::PxGroup, PxLocalReplica, PxLocalReplicaRole}; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use default_metadata::TestDefaultMetadata; + +async fn ready(fixture: &TestDefaultMetadata) { + tokio::time::timeout(std::time::Duration::from_secs(15), async { + while !fixture.ensure().await.unwrap() { + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + }) + .await + .expect("accepted metadata group did not become readable"); +} + +#[tokio::test] +async fn singleton_default_group_uses_owned_native_creation_and_replay_preserves_user_data() { + let fixture = TestDefaultMetadata::start(&[42]).await; + ready(&fixture).await; + let ctx = &fixture.authority.ctx; + ctx.kv().put(0, 1, b"retained", b"value", None).await.unwrap(); + let before = ctx.membership().read(0, 1).await.unwrap().unwrap(); + let operation = preparation::read(ctx, &fixture.accepted.bootstrap) + .await + .unwrap() + .unwrap(); + assert!(fixture.ensure().await.unwrap()); + assert_eq!( + ctx.membership().read(0, 1).await.unwrap().unwrap().record(), + before.record() + ); + assert_eq!( + preparation::read(ctx, &fixture.accepted.bootstrap) + .await + .unwrap() + .unwrap(), + operation + ); + assert!( + matches!(ctx.kv().get(0, 1, b"retained", ReadMode::Linearizable, None).await.unwrap(), + GetOutcome::Found { value, .. } if value.as_ref() == b"value") + ); +} + +#[tokio::test] +async fn three_owned_native_replicas_publish_native_listeners_and_allow_linearizable_reads() { + let fixture = TestDefaultMetadata::start(&[2, 7, 9]).await; + ready(&fixture).await; + let membership = fixture + .authority + .ctx + .membership() + .read(0, 1) + .await + .unwrap() + .unwrap(); + assert_eq!(membership.record().members.len(), 3); + for ((member, store), management) in membership + .record() + .members + .iter() + .zip(&fixture.stores) + .zip(&fixture.endpoints) + { + use crowdb_kv::cluster::KvServer; + assert_eq!(member.endpoint, store.listen_addr().unwrap().to_string()); + assert_ne!(format!("http://{}", member.endpoint), *management); + } + fixture + .authority + .ctx + .kv() + .put(0, 1, b"three", b"preserved", None) + .await + .unwrap(); + ready(&fixture).await; + assert!( + matches!(fixture.authority.ctx.kv().get(0, 1, b"three", ReadMode::Linearizable, None).await.unwrap(), + GetOutcome::Found { value, .. } if value.as_ref() == b"preserved") + ); +} + +#[tokio::test] +async fn a_matching_unowned_native_replica_cannot_publish_default_membership() { + let fixture = TestDefaultMetadata::start(&[2]).await; + fixture.stores[0].add_group_without_election(PxGroup::new( + 1, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + assert!(fixture.ensure().await.is_err()); + assert!(fixture + .authority + .ctx + .sysmd() + .get_group(0, 1) + .await + .unwrap() + .is_none()); + assert!(fixture + .authority + .ctx + .membership() + .read(0, 1) + .await + .unwrap() + .is_none()); +} + +#[tokio::test] +async fn retired_preparation_cannot_create_native_replicas_or_publish_metadata() { + let fixture = TestDefaultMetadata::start(&[2]).await; + preparation::retire( + &fixture.authority.ctx, + &fixture.accepted.bootstrap, + &fixture.accepted.operation_id, + ) + .await + .unwrap(); + assert!(fixture.ensure().await.is_err()); + assert!(fixture.stores[0].get_group(1).is_none()); + assert!(fixture + .authority + .ctx + .sysmd() + .get_group(0, 1) + .await + .unwrap() + .is_none()); +} + +#[tokio::test] +async fn operator_removal_retires_the_shared_operation_before_deleting_native_replicas() { + let fixture = TestDefaultMetadata::start(&[2]).await; + assert!( + crowdb_console_shared::ops::kv_logical::add_group(&fixture.authority.ctx, 0, 1, 1, &[2]) + .await + .is_err() + ); + assert!(fixture.stores[0].get_group(1).is_none()); + ready(&fixture).await; + crowdb_console_shared::ops::kv_logical::remove_group(&fixture.authority.ctx, 0, 1) + .await + .unwrap(); + let retired = preparation::read(&fixture.authority.ctx, &fixture.accepted.bootstrap) + .await + .unwrap() + .unwrap() + .0; + assert!(retired.retired); + assert_eq!(retired.operation_id, fixture.accepted.operation_id); + assert!(fixture.ensure().await.is_err()); + assert!(fixture.stores[0].get_group(1).is_none()); + assert!(fixture + .authority + .ctx + .membership() + .read(0, 1) + .await + .unwrap() + .is_none()); +} + +#[tokio::test] +async fn an_established_missing_replica_requires_data_recovery_without_creating_a_replacement() { + let fixture = TestDefaultMetadata::start(&[2]).await; + ready(&fixture).await; + let before = fixture + .authority + .ctx + .membership() + .read(0, 1) + .await + .unwrap() + .unwrap(); + assert!(fixture.stores[0].remove_group(1)); + assert!(fixture.ensure().await.is_err()); + assert!(fixture.stores[0].get_group(1).is_none()); + assert_eq!( + fixture + .authority + .ctx + .membership() + .read(0, 1) + .await + .unwrap() + .unwrap() + .record(), + before.record() + ); +} + +#[tokio::test] +async fn a_later_operator_membership_change_is_never_replaced_by_initial_defaults() { + let fixture = TestDefaultMetadata::start(&[2]).await; + ready(&fixture).await; + let membership = fixture.authority.ctx.membership(); + let initial = membership.read(0, 1).await.unwrap().unwrap(); + let changed = membership + .begin_change(0, 1, 1, initial.record().members.clone()) + .await + .unwrap(); + assert!(fixture.ensure().await.is_err()); + assert_eq!( + membership.read(0, 1).await.unwrap().unwrap().record(), + changed.record() + ); + assert!(fixture.stores[0].get_group(1).is_some()); +} + +#[tokio::test] +async fn consumers_resolve_only_the_accepted_group_and_refuse_missing_or_retired_bindings() { + use crowdb_protocol::{ + common::GroupValue, + key::{KvGroupKey, TextKey}, + }; + let fixture = TestDefaultMetadata::start(&[2]).await; + let ctx = &fixture.authority.ctx; + ctx.kv() + .put( + 0, + 0, + KvGroupKey { + store_id: 0, + group_id: 8, + } + .to_path() + .as_bytes(), + &serde_json::to_vec(&GroupValue { + store_id: 0, + group_id: 8, + }) + .unwrap(), + None, + ) + .await + .unwrap(); + assert!(preparation::metadata_binding(ctx, &fixture.accepted.bootstrap) + .await + .is_err()); + ready(&fixture).await; + let metadata = preparation::metadata_binding(ctx, &fixture.accepted.bootstrap) + .await + .unwrap(); + assert_eq!((metadata.store_id, metadata.group_id), (0, 1)); + let storage = preparation::storage_binding(ctx, &fixture.accepted.bootstrap, 4, 2) + .await + .unwrap(); + assert_eq!( + (storage.disk_group_id, storage.store_id, storage.group_id), + (1, 0, 1) + ); + assert!( + preparation::storage_binding(ctx, &fixture.accepted.bootstrap, 9, 2) + .await + .is_err() + ); + preparation::retire(ctx, &fixture.accepted.bootstrap, &fixture.accepted.operation_id) + .await + .unwrap(); + assert!(preparation::metadata_binding(ctx, &fixture.accepted.bootstrap) + .await + .is_err()); + assert!( + preparation::storage_binding(ctx, &fixture.accepted.bootstrap, 4, 2) + .await + .is_err() + ); +} diff --git a/lib/crowdb-console-shared/tests/default_preparation_test.rs b/lib/crowdb-console-shared/tests/default_preparation_test.rs new file mode 100644 index 000000000..3e1851b2d --- /dev/null +++ b/lib/crowdb-console-shared/tests/default_preparation_test.rs @@ -0,0 +1,364 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/default_preparation.rs"] +mod common; + +use common::TestPreparationAuthority; +use crowdb_console_shared::deployment::{ + preparation::{self, DEFAULT_PREPARATION_KEY}, + CLUSTER_KEY, +}; +use crowdb_kv_client::{GetOutcome, ReadMode}; +use crowdb_protocol::{ + common::GroupValue, + key::{KvGroupKey, TextKey}, +}; + +async fn snapshot(authority: &TestPreparationAuthority) -> Option<(Vec, u64)> { + match authority + .ctx + .kv() + .get(0, 0, DEFAULT_PREPARATION_KEY, ReadMode::Linearizable, None) + .await + .unwrap() + { + GetOutcome::Found { value, revision } => Some((value.to_vec(), revision)), + GetOutcome::NotFound => None, + } +} + +#[tokio::test] +async fn simultaneous_runners_converge_on_one_fixed_mapping_and_credential_reference() { + let authority = TestPreparationAuthority::start(&[2, 7, 9]).await; + let attempts = futures::future::join_all( + (0..8).map(|_| preparation::reserve(&authority.ctx, &authority.bootstrap.identity)), + ) + .await; + let winner = attempts[0].as_ref().unwrap(); + for attempt in &attempts { + assert_eq!(attempt.as_ref().unwrap(), winner); + } + assert_eq!((winner.metadata.store_id, winner.metadata.group_id), (0, 1)); + assert_eq!( + winner + .metadata + .replicas + .iter() + .map(|replica| (replica.node_id, replica.replica_id)) + .collect::>(), + vec![(2, 1), (7, 2), (9, 3)] + ); + assert_eq!(winner.storage.len(), 3); + for (index, binding) in winner.storage.iter().enumerate() { + assert_eq!( + ( + binding.rack_id, + binding.disk_group_id, + binding.store_id, + binding.group_id + ), + (4, index as u64 + 1, 0, 1) + ); + } + assert!(winner.credential_ref.ends_with(&winner.operation_id)); + assert_eq!(winner.chunk_kv.node_id, 2); + assert_eq!(winner.chunk_kv.metadata_group_id, winner.metadata.group_id); + assert_eq!(winner.chunk_kv.owner_epoch, 1); + assert_ne!( + winner.chunk_kv.partition_id, + crowdb_protocol::chunk_kv::Id128::default() + ); + assert_ne!( + winner.chunk_kv.stream_name, + crowdb_protocol::chunk_stream::StreamName::default() + ); + assert_ne!(winner.chunk_kv.tree_id, 0); + let before = snapshot(&authority).await.unwrap(); + assert_eq!( + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(), + *winner + ); + assert_eq!(snapshot(&authority).await.unwrap(), before); + assert!(authority.ctx.sysmd().get_group(0, 1).await.unwrap().is_none()); + assert!(authority + .ctx + .sysmd() + .get_disk_group(4, 2, 1) + .await + .unwrap() + .is_none()); + for binding in &winner.storage { + authority + .ctx + .sysmd() + .reserve_disk_group_id(&crowdb_protocol::key::DiskGroupKey { + rack_id: binding.rack_id, + node_id: binding.node_id, + disk_group_id: binding.disk_group_id, + }) + .await + .unwrap(); + } +} + +#[tokio::test] +async fn global_disk_group_reservations_cannot_be_adopted_from_another_node_or_retired_location() { + let authority = TestPreparationAuthority::start(&[2, 7, 9]).await; + let foreign = crowdb_protocol::key::DiskGroupKey { + rack_id: 8, + node_id: 42, + disk_group_id: 2, + }; + authority + .ctx + .sysmd() + .reserve_disk_group_id(&foreign) + .await + .unwrap(); + let error = preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap_err(); + assert!(error.to_string().contains("global owner"), "{error}"); + assert!(snapshot(&authority).await.is_none()); + assert!(authority + .ctx + .sysmd() + .get_disk_group(4, 2, 1) + .await + .unwrap() + .is_none()); +} + +#[tokio::test] +async fn singleton_uses_admitted_node_identity_and_read_does_not_reserve() { + let authority = TestPreparationAuthority::start(&[42]).await; + assert!(preparation::read(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap() + .is_none()); + assert!(snapshot(&authority).await.is_none()); + let winner = preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + assert_eq!(winner.metadata.replicas[0].node_id, 42); + assert_eq!(winner.storage[0].node_id, 42); + assert_eq!(winner.chunk_kv.node_id, 42); + let before = snapshot(&authority).await.unwrap(); + let recovered = preparation::read(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap() + .unwrap(); + assert_eq!(recovered.0, winner); + assert_eq!(recovered.1, before.1); + let other_runner = crowdb_console_shared::ops::OpContext::with_shared_client_preserving_hint( + authority.ctx.kv_arc().clone(), + &[], + crowdb_console_shared::ConsoleConfig::default(), + ); + assert_eq!( + preparation::reserve(&other_runner, &authority.bootstrap.identity) + .await + .unwrap(), + winner + ); + assert_eq!(snapshot(&authority).await.unwrap(), before); +} + +#[tokio::test] +async fn existing_membership_or_storage_binding_cannot_be_adopted_by_default_preparation() { + use crowdb_protocol::{common::BindMapValue, key::BindMapKey, kv_membership::GroupMember}; + let authority = TestPreparationAuthority::start(&[2]).await; + authority + .ctx + .membership() + .create( + 0, + 1, + vec![GroupMember { + replica_id: 1, + node_id: 2, + endpoint: "node-2:10100".into(), + voting: true, + }], + ) + .await + .unwrap(); + assert!( + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .is_err() + ); + assert!(snapshot(&authority).await.is_none()); + let other = TestPreparationAuthority::start(&[2]).await; + let key = BindMapKey { + rack_id: 4, + node_id: 2, + disk_group_id: 1, + } + .to_path(); + let bytes = serde_json::to_vec(&BindMapValue { + store_id: 0, + group_id: 8, + }) + .unwrap(); + other + .ctx + .kv() + .put(0, 0, key.as_bytes(), &bytes, None) + .await + .unwrap(); + assert!(preparation::reserve(&other.ctx, &other.bootstrap.identity) + .await + .is_err()); + assert!(snapshot(&other).await.is_none()); +} + +#[tokio::test] +async fn conflicting_or_missing_authority_never_allocates_default_identity() { + let authority = TestPreparationAuthority::start(&[2]).await; + let mut other = authority.bootstrap.identity.clone(); + other.cluster_id = uuid::Uuid::new_v4().to_string(); + assert!(preparation::reserve(&authority.ctx, &other).await.is_err()); + assert!(snapshot(&authority).await.is_none()); + authority.ctx.kv().delete(0, 0, CLUSTER_KEY, None).await.unwrap(); + assert!( + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .is_err() + ); + assert!(snapshot(&authority).await.is_none()); +} + +#[tokio::test] +async fn preexisting_group_conflicts_and_unrelated_groups_are_never_selected() { + let authority = TestPreparationAuthority::start(&[2]).await; + let key = KvGroupKey { + store_id: 0, + group_id: 8, + } + .to_path(); + authority + .ctx + .kv() + .put( + 0, + 0, + key.as_bytes(), + &serde_json::to_vec(&GroupValue { + store_id: 0, + group_id: 8, + }) + .unwrap(), + None, + ) + .await + .unwrap(); + let winner = preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + assert_eq!(winner.metadata.group_id, 1); + let other = TestPreparationAuthority::start(&[2]).await; + let key = KvGroupKey { + store_id: 0, + group_id: 1, + } + .to_path(); + let bytes = serde_json::to_vec(&GroupValue { + store_id: 0, + group_id: 1, + }) + .unwrap(); + other + .ctx + .kv() + .put(0, 0, key.as_bytes(), &bytes, None) + .await + .unwrap(); + assert!(preparation::reserve(&other.ctx, &other.bootstrap.identity) + .await + .is_err()); + assert!(snapshot(&other).await.is_none()); +} + +#[tokio::test] +async fn corrupt_or_changed_fixed_inputs_are_preserved_and_never_replaced() { + let authority = TestPreparationAuthority::start(&[2]).await; + let winner = preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + let mut changed = winner.clone(); + changed.metadata.replicas[0].node_id = 9; + let mut wrong_version = winner.clone(); + wrong_version.version = 2; + let mut changed_reference = winner.clone(); + changed_reference.credential_ref = "access/other".into(); + for bytes in [ + serde_json::to_vec(&changed).unwrap(), + serde_json::to_vec(&wrong_version).unwrap(), + serde_json::to_vec(&changed_reference).unwrap(), + b"invalid".to_vec(), + ] { + authority + .ctx + .kv() + .put(0, 0, DEFAULT_PREPARATION_KEY, &bytes, None) + .await + .unwrap(); + let before = snapshot(&authority).await.unwrap(); + assert!( + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .is_err() + ); + assert_eq!(snapshot(&authority).await.unwrap(), before); + } +} + +#[tokio::test] +async fn explicit_retirement_is_idempotent_and_prevents_automatic_recreation() { + let authority = TestPreparationAuthority::start(&[2]).await; + let winner = preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap(); + let before = snapshot(&authority).await.unwrap(); + assert!(preparation::retire( + &authority.ctx, + &authority.bootstrap.identity, + &uuid::Uuid::new_v4().to_string() + ) + .await + .is_err()); + assert_eq!(snapshot(&authority).await.unwrap(), before); + preparation::retire( + &authority.ctx, + &authority.bootstrap.identity, + &winner.operation_id, + ) + .await + .unwrap(); + let retired = snapshot(&authority).await.unwrap(); + assert!( + preparation::read(&authority.ctx, &authority.bootstrap.identity) + .await + .unwrap() + .unwrap() + .0 + .retired + ); + preparation::retire( + &authority.ctx, + &authority.bootstrap.identity, + &winner.operation_id, + ) + .await + .unwrap(); + assert!( + preparation::reserve(&authority.ctx, &authority.bootstrap.identity) + .await + .is_err() + ); + assert_eq!(snapshot(&authority).await.unwrap(), retired); +} diff --git a/lib/crowdb-console-shared/tests/deployment_group_replay_test.rs b/lib/crowdb-console-shared/tests/deployment_group_replay_test.rs new file mode 100644 index 000000000..1f472c02f --- /dev/null +++ b/lib/crowdb-console-shared/tests/deployment_group_replay_test.rs @@ -0,0 +1,41 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/deployment_group_replay.rs"] +mod common; + +use common::{owner, TestDeploymentReplay}; + +#[tokio::test] +async fn a_lost_creation_reply_is_read_back_without_reissuing_the_mutation() { + let accepted = owner(); + let deployment = accepted.deployment.clone().unwrap(); + let fixture = TestDeploymentReplay::start(Some(accepted), true).await; + assert!(!fixture.replay(deployment.clone()).await.unwrap()); + assert_eq!(fixture.posts(), 1); + assert!(!fixture.replay(deployment).await.unwrap()); + assert_eq!(fixture.posts(), 1); +} + +#[tokio::test] +async fn matching_replica_numbers_never_adopt_unknown_foreign_or_retired_creation() { + let expected = owner(); + let deployment = expected.deployment.clone().unwrap(); + let mut retired = expected.clone(); + retired.retired = true; + let mut wrong_replica = expected.clone(); + wrong_replica.replica_id = 8; + for accepted in [None, Some(owner()), Some(retired), Some(wrong_replica)] { + let fixture = TestDeploymentReplay::start(accepted, false).await; + assert!(fixture.replay(deployment.clone()).await.is_err()); + assert_eq!(fixture.posts(), 0); + } +} + +#[tokio::test] +async fn a_foreign_owner_after_an_unknown_create_outcome_remains_a_conflict() { + let deployment = owner().deployment.unwrap(); + let fixture = TestDeploymentReplay::start(Some(owner()), true).await; + assert!(fixture.replay(deployment).await.is_err()); + assert_eq!(fixture.posts(), 1); +} diff --git a/lib/crowdb-console-shared/tests/lifecycle_restart_test.rs b/lib/crowdb-console-shared/tests/lifecycle_restart_test.rs index 175916756..5894f0d44 100644 --- a/lib/crowdb-console-shared/tests/lifecycle_restart_test.rs +++ b/lib/crowdb-console-shared/tests/lifecycle_restart_test.rs @@ -62,6 +62,14 @@ async fn retained_launch_restarts_with_a_new_pid() { assert!(lifecycle::process_is_alive(new_pid)); lifecycle::stop_pid_with_timeout(new_pid, Duration::from_secs(5)).unwrap(); + #[cfg(target_os = "linux")] + tokio::time::timeout(Duration::from_secs(2), async { + while std::path::Path::new(&format!("/proc/{new_pid}")).exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("restarted child retained its process registration after exit"); let _ = child.wait(); } diff --git a/lib/crowdb-console-shared/tests/local_group_replay_test.rs b/lib/crowdb-console-shared/tests/local_group_replay_test.rs index ed0df331c..2d1fd667f 100644 --- a/lib/crowdb-console-shared/tests/local_group_replay_test.rs +++ b/lib/crowdb-console-shared/tests/local_group_replay_test.rs @@ -54,6 +54,7 @@ async fn replay(replica: u64, race: bool) -> (crowdb_console_shared::error::Resu replica_id: 10, initial_role: None, start_election: Some(false), + deployment: None, }; let result = create_local_group_for_tests(&endpoint, 0, &request).await; let posts = state.posts.load(Ordering::Relaxed); diff --git a/lib/crowdb-console-shared/tests/member_evidence_test.rs b/lib/crowdb-console-shared/tests/member_evidence_test.rs new file mode 100644 index 000000000..0dc09521c --- /dev/null +++ b/lib/crowdb-console-shared/tests/member_evidence_test.rs @@ -0,0 +1,174 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/member_evidence.rs"] +mod member_evidence; + +use crowdb_console_shared::ops::cluster::readiness::member_evidence as project; +use crowdb_protocol::mgmt::{ + cluster::{DependencyKind, DependencyState}, + node::NodeServiceAction, +}; +use member_evidence::TestMemberEvidence; + +#[test] +fn each_required_service_and_protocol_has_its_own_current_fact() { + let fixture = TestMemberEvidence::new(); + let facts = fixture.project(); + assert_eq!(facts.len(), 10); + assert!(facts + .iter() + .all(|fact| fact.state == DependencyState::Ready && fact.operation_id == fact.current_operation_id)); + assert_eq!( + facts + .iter() + .filter(|fact| fact.kind == DependencyKind::Service) + .count(), + 6 + ); + assert_eq!( + facts + .iter() + .filter(|fact| fact.kind == DependencyKind::Protocol) + .count(), + 3 + ); +} + +#[test] +fn aggregate_preserves_protocol_evidence_and_exact_http_failure() { + let mut fixture = TestMemberEvidence::new(); + let s3 = fixture + .node + .protocols + .iter_mut() + .find(|fact| fact.id.ends_with("/s3")) + .unwrap(); + s3.state = DependencyState::Unknown; + s3.detail = "S3 signed ListBuckets returned HTTP 403; expected 200".into(); + s3.reason = Some(crowdb_protocol::mgmt::cluster::ReadinessReason { + dependency: s3.id.clone(), + code: "protocol-http-status".into(), + detail: s3.detail.clone(), + }); + let facts = fixture.project(); + let s3 = facts.iter().find(|fact| fact.id.ends_with("/s3")).unwrap(); + assert_eq!(s3.reason.as_ref().unwrap().code, "protocol-http-status"); + assert_eq!(s3.reason.as_ref().unwrap().detail, s3.detail); + assert!(s3.detail.contains("HTTP 403")); + let iceberg = facts.iter().find(|fact| fact.id.ends_with("/iceberg")).unwrap(); + assert!(iceberg.detail.contains("authenticated request succeeded")); +} + +#[test] +fn an_unreachable_member_and_a_pending_binding_cannot_be_replaced_by_other_health() { + let mut fixture = TestMemberEvidence::new(); + let missing = project( + &fixture.bootstrap, + &fixture.expected, + &fixture.policy, + Some(&fixture.intents), + None, + 1000, + ); + assert_eq!(missing.len(), 1); + assert_eq!(missing[0].kind, DependencyKind::Member); + assert_eq!(missing[0].state, DependencyState::Unknown); + fixture.node.binding = None; + assert_eq!(fixture.project()[0].state, DependencyState::Initializing); +} + +#[test] +fn a_foreign_uuid_discovery_identity_or_formation_policy_fails_closed() { + let original = TestMemberEvidence::new(); + for variant in 0..3 { + let mut fixture = TestMemberEvidence::new(); + match variant { + 0 => { + fixture.node.binding.as_mut().unwrap().bootstrap.cluster_id = + original.bootstrap.identity.cluster_id.clone(); + } + 1 => fixture.node.discovery_id = original.expected.discovery_id.clone(), + _ => fixture.node.policy.cluster = "foreign".into(), + } + let facts = fixture.project(); + assert_eq!(facts.len(), 1); + assert_eq!(facts[0].kind, DependencyKind::Identity); + assert_eq!(facts[0].state, DependencyState::Failed); + } +} + +#[test] +fn current_stop_wins_over_all_historical_ready_facts() { + let mut fixture = TestMemberEvidence::new(); + let access = fixture + .intents + .iter_mut() + .find(|intent| intent.kind == "access") + .unwrap(); + access.action = NodeServiceAction::Stop; + access.operation_id = uuid::Uuid::new_v4().to_string(); + let facts = fixture.project(); + let stopped: Vec<_> = facts + .iter() + .filter(|fact| fact.state == DependencyState::Stopped) + .collect(); + assert_eq!(stopped.len(), 4); + assert!(stopped + .iter() + .all(|fact| fact.operation_id == fact.current_operation_id)); +} + +#[test] +fn a_new_start_requires_new_execution_and_new_protocol_observations() { + let mut fixture = TestMemberEvidence::new(); + fixture + .intents + .iter_mut() + .find(|intent| intent.kind == "access") + .unwrap() + .operation_id = uuid::Uuid::new_v4().to_string(); + let facts = fixture.project(); + let superseded: Vec<_> = facts + .iter() + .filter(|fact| fact.operation_id != fact.current_operation_id) + .collect(); + assert_eq!(superseded.len(), 4); +} + +#[test] +fn missing_intent_and_missing_protocol_proofs_remain_unknown() { + let mut fixture = TestMemberEvidence::new(); + fixture.intents.clear(); + assert_eq!( + fixture + .project() + .iter() + .filter(|fact| fact.state == DependencyState::Unknown) + .count(), + 8 + ); + let mut fixture = TestMemberEvidence::new(); + fixture.node.protocols.clear(); + assert_eq!( + fixture + .project() + .iter() + .filter(|fact| fact.kind == DependencyKind::Protocol && fact.state == DependencyState::Unknown) + .count(), + 3 + ); +} + +#[test] +fn duplicate_service_or_protocol_identity_is_a_conflict() { + for protocol in [false, true] { + let mut fixture = TestMemberEvidence::new(); + if protocol { + fixture.node.protocols.push(fixture.node.protocols[0].clone()); + } else { + fixture.node.services.push(fixture.node.services[0].clone()); + } + assert_eq!(fixture.project()[0].state, DependencyState::Failed); + } +} diff --git a/lib/crowdb-console-shared/tests/member_recovery_test.rs b/lib/crowdb-console-shared/tests/member_recovery_test.rs new file mode 100644 index 000000000..7a0a420b4 --- /dev/null +++ b/lib/crowdb-console-shared/tests/member_recovery_test.rs @@ -0,0 +1,377 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::os::unix::fs::{symlink, PermissionsExt}; + +use crowdb_console_shared::deployment::recovery::MemberTrust; +use crowdb_test_harness::test_dirs::tempdir_in_test_data; +use russh::keys::key::KeyPair; + +#[path = "common/member_recovery.rs"] +mod recovery; +#[path = "common/ssh_server.rs"] +#[allow(dead_code)] +mod ssh_server; + +#[path = "common/member_evidence.rs"] +#[allow(dead_code)] +mod member_evidence; + +#[test] +fn trust_must_match_the_sealed_bootstrap_members_and_admission() { + use sha2::{Digest, Sha256}; + + let mut accepted = member_evidence::TestMemberEvidence::new().bootstrap; + let mut trust = recovery::trust(&KeyPair::generate_ed25519().unwrap()); + accepted.nodes[0].host_key = Some(trust.members[0].host_key.clone()); + accepted.identity.configuration_digest = hex::encode(Sha256::digest( + serde_json::to_vec(&(&accepted.intent, &accepted.nodes)).unwrap(), + )); + trust.bootstrap = accepted.identity.clone(); + trust.members[0] + .discovery_id + .clone_from(&accepted.nodes[0].discovery_id); + trust.members[0] + .admission_operation_id + .clone_from(&accepted.nodes[0].operation_id); + trust.validate_bootstrap(&accepted).unwrap(); + let mut changed = trust.clone(); + changed.members[0].host_key = recovery::host_key(&KeyPair::generate_ed25519().unwrap()); + assert!(changed.validate_bootstrap(&accepted).is_err()); + changed = trust.clone(); + changed.members[0].admission_operation_id = uuid::Uuid::new_v4().to_string(); + assert!(changed.validate_bootstrap(&accepted).is_err()); + changed = trust.clone(); + changed.members[0].node_id = 2; + assert!(changed.validate_bootstrap(&accepted).is_err()); + changed = trust; + changed.bootstrap.cluster_id = uuid::Uuid::new_v4().to_string(); + assert!(changed.validate_bootstrap(&accepted).is_err()); +} + +#[test] +fn trust_rejects_duplicate_identity_shared_keys_and_invalid_versions() { + let key = KeyPair::generate_ed25519().unwrap(); + let accepted = recovery::trust(&key); + accepted.validate().unwrap(); + let mut changed = accepted.clone(); + changed.version = 2; + assert!(changed.validate().is_err()); + changed = accepted.clone(); + let mut duplicate = changed.members[0].clone(); + duplicate.node_id = 2; + duplicate.discovery_id = uuid::Uuid::new_v4().to_string(); + changed.members.push(duplicate); + assert!( + changed.validate().is_err(), + "two member identities cannot share one host key" + ); + changed.members[1].host_key = recovery::host_key(&KeyPair::generate_ed25519().unwrap()); + changed.validate().unwrap(); + changed.members[1].discovery_id = changed.members[0].discovery_id.clone(); + assert!(changed.validate().is_err()); + changed = accepted; + changed.members[0].host_key = "ssh-ed25519 invalid".into(); + assert!(changed.validate().is_err()); +} + +#[test] +fn sealed_trust_is_private_immutable_and_never_relearned() { + let dir = tempdir_in_test_data("member-trust-seal"); + let path = dir.path().join("trust.json"); + let accepted = recovery::trust(&KeyPair::generate_ed25519().unwrap()); + assert!(MemberTrust::load(&path).is_err()); + accepted.seal(&path).unwrap(); + accepted.seal(&path).unwrap(); + assert_eq!(MemberTrust::load(&path).unwrap(), accepted); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let original = std::fs::read(&path).unwrap(); + let mut changed = accepted.clone(); + changed.members[0].host_key = recovery::host_key(&KeyPair::generate_ed25519().unwrap()); + assert!(changed.seal(&path).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), original); + let link = dir.path().join("linked.json"); + symlink(&path, &link).unwrap(); + assert!(MemberTrust::load(&link).is_err()); + assert!(accepted.seal(&link).is_err()); + std::fs::write(&path, "corrupt").unwrap(); + assert!(accepted.seal(&path).is_err()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "corrupt"); +} + +#[test] +fn concurrent_sealers_cannot_replace_the_winning_identity() { + let dir = tempdir_in_test_data("member-trust-race"); + let path = dir.path().join("trust.json"); + let first = recovery::trust(&KeyPair::generate_ed25519().unwrap()); + let mut second = first.clone(); + second.members[0].host_key = recovery::host_key(&KeyPair::generate_ed25519().unwrap()); + std::thread::scope(|scope| { + let a = scope.spawn(|| first.seal(&path)); + let b = scope.spawn(|| second.seal(&path)); + assert_ne!(a.join().unwrap().is_ok(), b.join().unwrap().is_ok()); + }); + let winner = MemberTrust::load(&path).unwrap(); + assert!(winner == first || winner == second); +} + +#[tokio::test] +async fn pinned_identity_survives_endpoint_changes_without_address_tofu() { + let dir = tempdir_in_test_data("member-recovery-move"); + let address_cache = dir.path().join("address-cache"); + std::fs::write(&address_cache, "deliberately invalid address-keyed trust").unwrap(); + std::env::set_var("CROWDB_KV_KNOWN_HOSTS", &address_cache); + let key = KeyPair::generate_ed25519().unwrap(); + let host = KeyPair::generate_ed25519().unwrap(); + let trust = recovery::trust(&host); + let observed = recovery::observation(&trust); + let reply = recovery::reply(&observed); + let first = ssh_server::TestSshServer::start_with_host_key( + key.clone_public_key().unwrap(), + Some(reply.clone()), + host.clone(), + ) + .await; + let mut node = recovery::node(first.port, &dir.path().join("key"), &key); + assert_eq!(trust.observe(&node).await.unwrap(), observed); + drop(first); + let second = + ssh_server::TestSshServer::start_with_host_key(key.clone_public_key().unwrap(), Some(reply), host) + .await; + node.ssh_port = second.port; + assert_eq!(trust.observe(&node).await.unwrap(), observed); + let impostor = ssh_server::TestSshServer::start_with_reply( + key.clone_public_key().unwrap(), + Some(recovery::reply(&observed)), + ) + .await; + node.ssh_port = impostor.port; + assert!( + trust.observe(&node).await.is_err(), + "self-reported identity cannot override host-key pins" + ); + assert_eq!( + std::fs::read_to_string(address_cache).unwrap(), + "deliberately invalid address-keyed trust" + ); +} + +#[tokio::test] +async fn valid_host_key_cannot_claim_another_cluster_or_member() { + let dir = tempdir_in_test_data("member-recovery-binding"); + let key = KeyPair::generate_ed25519().unwrap(); + let host = KeyPair::generate_ed25519().unwrap(); + let trust = recovery::trust(&host); + for variant in 0..4 { + let mut observed = recovery::observation(&trust); + match variant { + 0 => observed.discovery_id = uuid::Uuid::new_v4().to_string(), + 1 => observed.binding.as_mut().unwrap().node_id = 2, + 2 => observed.binding.as_mut().unwrap().bootstrap.cluster_id = uuid::Uuid::new_v4().to_string(), + _ => observed.binding = None, + } + let server = ssh_server::TestSshServer::start_with_host_key( + key.clone_public_key().unwrap(), + Some(recovery::reply(&observed)), + host.clone(), + ) + .await; + let node = recovery::node(server.port, &dir.path().join("key"), &key); + assert!( + trust.observe(&node).await.is_err(), + "variant {variant} must be rejected" + ); + } +} + +#[tokio::test] +async fn authenticated_endpoint_requires_the_contacted_address_and_live_incarnation() { + use crowdb_protocol::mgmt::node::recovery::{MemberEndpoint, MemberRecoveryIdentity}; + let dir = tempdir_in_test_data("member-current-address"); + let key = KeyPair::generate_ed25519().unwrap(); + let host = KeyPair::generate_ed25519().unwrap(); + let trust = recovery::trust(&host); + let identity = MemberRecoveryIdentity { + version: 1, + binding: recovery::observation(&trust).binding.unwrap(), + endpoint: MemberEndpoint { + node_id: 1, + discovery_id: trust.members[0].discovery_id.clone(), + boot_id: uuid::Uuid::new_v4().to_string(), + addresses: vec!["127.0.0.1".parse().unwrap()], + }, + }; + for variant in 0..4 { + let mut reply = identity.clone(); + match variant { + 1 => reply.endpoint.addresses = vec!["127.0.0.2".parse().unwrap()], + 2 => reply.endpoint.boot_id = uuid::Uuid::nil().to_string(), + 3 => reply.binding.bootstrap.cluster_id = uuid::Uuid::new_v4().to_string(), + _ => {} + } + let server = ssh_server::TestSshServer::start_with_host_key( + key.clone_public_key().unwrap(), + Some(serde_json::to_vec(&reply).unwrap()), + host.clone(), + ) + .await; + let node = recovery::node(server.port, &dir.path().join("key"), &key); + let result = trust.endpoint(&node).await; + assert_eq!(result.is_ok(), variant == 0, "variant {variant}: {result:?}"); + } +} + +#[tokio::test] +async fn pinned_control_rejects_oversized_malformed_and_password_fallback() { + let dir = tempdir_in_test_data("member-recovery-reply"); + let key = KeyPair::generate_ed25519().unwrap(); + let host = KeyPair::generate_ed25519().unwrap(); + let trust = recovery::trust(&host); + for reply in [vec![b'x'; 65537], b"not json".to_vec()] { + let server = ssh_server::TestSshServer::start_with_host_key( + key.clone_public_key().unwrap(), + Some(reply), + host.clone(), + ) + .await; + let node = recovery::node(server.port, &dir.path().join("key"), &key); + assert!(trust.observe(&node).await.is_err()); + } + let mut node = recovery::node(1, &dir.path().join("key"), &key); + node.ssh_password = Some("must-not-be-used".into()); + assert!(trust + .observe(&node) + .await + .unwrap_err() + .to_string() + .contains("local key")); +} + +#[test] +fn address_rebind_updates_only_owned_internal_listeners() { + let source = r#" +http_advertise_addr = "192.0.2.1:7400" +[server] +bind_address = "192.0.2.1" +kv_server_mgmt_seeds = ["http://192.0.2.1:10000"] +rpc_listen_addr = "192.0.2.1:7102" +http_listen_addr = "0.0.0.0:7101" +[s3] +listen = "192.0.2.1:9091" +public_uri = "http://192.0.2.1:49091" +[dataset] +listen = "192.0.2.8:9093" +"#; + let result = crowdb_console_shared::deployment::recovery::rebind_service( + source, + "192.0.2.1".parse().unwrap(), + "192.0.2.2".parse().unwrap(), + &["http://192.0.2.2:10000".into()], + ) + .unwrap(); + let config: toml::Value = toml::from_str(&result).unwrap(); + assert_eq!( + config["server"]["kv_server_mgmt_seeds"][0].as_str(), + Some("http://192.0.2.2:10000") + ); + assert_eq!(config["server"]["bind_address"].as_str(), Some("192.0.2.2")); + assert_eq!( + config["server"]["rpc_listen_addr"].as_str(), + Some("192.0.2.2:7102") + ); + assert_eq!( + config["server"]["http_listen_addr"].as_str(), + Some("0.0.0.0:7101") + ); + assert_eq!(config["s3"]["listen"].as_str(), Some("192.0.2.2:9091")); + assert_eq!( + config["s3"]["public_uri"].as_str(), + Some("http://192.0.2.1:49091") + ); + assert_eq!(config["dataset"]["listen"].as_str(), Some("192.0.2.8:9093")); + assert_eq!( + crowdb_console_shared::deployment::recovery::rebind_service( + &result, + "192.0.2.1".parse().unwrap(), + "192.0.2.2".parse().unwrap(), + &["http://192.0.2.2:10000".into()], + ) + .unwrap(), + result + ); +} + +#[test] +fn authenticated_admission_pin_survives_serialization_and_address_changes() { + let mut node = member_evidence::TestMemberEvidence::new().expected; + let key = recovery::host_key(&KeyPair::generate_ed25519().unwrap()); + node.pin_host_key(&format!("{key} authenticated-host")).unwrap(); + let accepted = node.host_key.clone(); + node.host = "192.0.2.42".into(); + node.pin_host_key(&format!("{key} updated-comment")).unwrap(); + assert_eq!(node.host_key, accepted); + let bytes = serde_json::to_vec(&node).unwrap(); + let retained: crowdb_console_shared::deployment::registry::NodeRecord = + serde_json::from_slice(&bytes).unwrap(); + assert_eq!(retained.host_key, accepted); + let changed = recovery::host_key(&KeyPair::generate_ed25519().unwrap()); + assert!(node.pin_host_key(&changed).is_err()); + assert!(node.pin_host_key("ssh-ed25519 invalid").is_err()); + assert_eq!(node.host_key, accepted); +} + +#[test] +fn confirmed_authority_alone_extends_and_retires_later_member_trust() { + use crowdb_console_shared::deployment::registry::NodeRegistry; + let initial = recovery::trust(&KeyPair::generate_ed25519().unwrap()); + let mut node = member_evidence::TestMemberEvidence::new().expected; + node.discovery_id.clone_from(&initial.members[0].discovery_id); + node.operation_id + .clone_from(&initial.members[0].admission_operation_id); + node.host_key = Some(initial.members[0].host_key.clone()); + node.confirmed = true; + let mut later = node.clone(); + later.discovery_id = uuid::Uuid::new_v4().to_string(); + later.operation_id = uuid::Uuid::new_v4().to_string(); + later.node_id = 2; + later.host_key = None; + later + .pin_host_key(&recovery::host_key(&KeyPair::generate_ed25519().unwrap())) + .unwrap(); + later.confirmed = false; + let mut registry = NodeRegistry { + cluster_id: initial.bootstrap.cluster_id.clone(), + nodes: vec![node, later], + }; + assert_eq!(initial.confirmed_registry(®istry).unwrap(), initial); + registry.nodes[1].confirmed = true; + let current = initial.confirmed_registry(®istry).unwrap(); + assert_eq!(current.members.len(), 2); + assert_eq!(current.members[0], initial.members[0]); + registry.nodes[1].cancelled = true; + assert_eq!(initial.confirmed_registry(®istry).unwrap(), initial); + registry.nodes[1].cancelled = false; + let accepted = registry.clone(); + for change in [ + "cluster", + "initial-key", + "initial-operation", + "missing-pin", + "shared-key", + ] { + let mut changed = accepted.clone(); + match change { + "cluster" => changed.cluster_id = uuid::Uuid::new_v4().to_string(), + "initial-key" => changed.nodes[0].host_key = changed.nodes[1].host_key.clone(), + "initial-operation" => changed.nodes[0].operation_id = uuid::Uuid::new_v4().to_string(), + "missing-pin" => changed.nodes[1].host_key = None, + "shared-key" => changed.nodes[1].host_key = changed.nodes[0].host_key.clone(), + _ => unreachable!(), + } + assert!(initial.confirmed_registry(&changed).is_err(), "{change}"); + } + assert_eq!(initial.members.len(), 1, "initial sealed trust remains immutable"); +} diff --git a/lib/crowdb-console-shared/tests/ops_hardware_test.rs b/lib/crowdb-console-shared/tests/ops_hardware_test.rs index 46720d80f..47e2afe18 100644 --- a/lib/crowdb-console-shared/tests/ops_hardware_test.rs +++ b/lib/crowdb-console-shared/tests/ops_hardware_test.rs @@ -141,7 +141,7 @@ async fn remove_node_with_server_conflict() { { let mut cfg = ctx.config_mut(); cfg.servers.push(crowdb_console_shared::config::ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "10".into(), url: "http://127.0.0.1:10000".into(), node_id: Some(10), diff --git a/lib/crowdb-console-shared/tests/ops_kv_server_test.rs b/lib/crowdb-console-shared/tests/ops_kv_server_test.rs index 94347d95f..a41a03bc7 100644 --- a/lib/crowdb-console-shared/tests/ops_kv_server_test.rs +++ b/lib/crowdb-console-shared/tests/ops_kv_server_test.rs @@ -58,7 +58,7 @@ async fn stop_no_pid_not_found() { { let mut cfg = ctx.config_mut(); cfg.add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "10".into(), url: "http://127.0.0.1:10000".into(), node_id: Some(10), @@ -85,7 +85,7 @@ async fn list_returns_all_servers() { { let mut cfg = ctx.config_mut(); cfg.add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "10".into(), url: "http://127.0.0.1:10000".into(), node_id: Some(10), @@ -121,7 +121,7 @@ async fn delete_with_server_succeeds_when_no_replicas() { { let mut cfg = ctx.config_mut(); cfg.add_server(ServerEntry { - dataset_origin: None, + access_origins: std::collections::BTreeMap::new(), id: "10".into(), url: "http://127.0.0.1:10000".into(), node_id: Some(10), diff --git a/lib/crowdb-console-shared/tests/s3_bench_test.rs b/lib/crowdb-console-shared/tests/s3_bench_test.rs index 660812676..e69670189 100644 --- a/lib/crowdb-console-shared/tests/s3_bench_test.rs +++ b/lib/crowdb-console-shared/tests/s3_bench_test.rs @@ -5,7 +5,6 @@ use std::time::Duration; use axum::{body::Body, extract::Request, http::StatusCode, response::Response, Router}; use crowdb_console_shared::ops::s3_bench::{self, MixWeights, S3BenchConfig, S3BenchWorkload}; -use crowdb_test_harness::test_dirs::TestDir; #[test] fn mixed_weights_are_case_insensitive_and_deterministic() { @@ -35,7 +34,7 @@ fn mixed_weights_reject_ambiguous_or_invalid_terms() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn memory_mix_exercises_every_s3_operation() { +async fn attached_mix_exercises_every_s3_operation_without_local_lifecycle() { // Workload accounting is a component proof; real S3 flows have container owners. let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = format!("http://{}", listener.local_addr().unwrap()); @@ -44,11 +43,8 @@ async fn memory_mix_exercises_every_s3_operation() { .await .unwrap(); }); - let runtime = TestDir::new("s3-memory-bench").expect("create benchmark runtime"); - let work_dir = runtime.path().to_path_buf(); - let result = s3_bench::run_workload_for_tests( + let result = s3_bench::run( S3BenchConfig { - work_dir, workload: S3BenchWorkload::Mix, object_size: 1024, dataset_objects: 8, @@ -57,14 +53,14 @@ async fn memory_mix_exercises_every_s3_operation() { duration: Duration::from_secs(10), warmup_operations: 4, seed: 7, - memory_budget_bytes: 2 * 1024 * 1024 * 1024, + max_dataset_bytes: 2 * 1024 * 1024 * 1024, list_limit: 4, mix: MixWeights::default(), }, - endpoint, + crowdb_console_shared::ops::s3::S3HttpClient::new(endpoint).unwrap(), ) .await - .expect("memory S3 benchmark"); + .expect("attached S3 benchmark"); peer.abort(); assert_eq!(result.total_operations, 200); assert_eq!(result.total_errors, 0); @@ -72,7 +68,7 @@ async fn memory_mix_exercises_every_s3_operation() { assert!(result.by_operation.read.is_some()); assert!(result.by_operation.range_read.is_some()); assert!(result.by_operation.list.is_some()); - assert_eq!(result.backing.diskio, "mem"); + assert!(result.bucket.starts_with("crowdb-bench-")); assert!(!result.object_bytes_verified); } diff --git a/lib/crowdb-console-shared/tests/s3_local_state_test.rs b/lib/crowdb-console-shared/tests/s3_local_state_test.rs deleted file mode 100644 index 099ab9785..000000000 --- a/lib/crowdb-console-shared/tests/s3_local_state_test.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -//! Local launch persistence stays component-local; packaged storage recovery is E2E. - -use crowdb_console_shared::ops::s3; -use crowdb_test_harness::test_dirs::TestDir; - -fn state() -> TestDir { - let directory = TestDir::new("s3-local-state").unwrap(); - std::fs::write( - directory.path().join("s3-mini-cluster.json"), - r#"{"version":1,"endpoint":"http://127.0.0.1:7500","tenant":"local"}"#, - ) - .unwrap(); - std::fs::write( - s3::config_path(directory.path()), - r#"version = 1 -group0_seeds = ["http://127.0.0.1:7000"] -[[service]] -id = "kv-1" -url = "http://127.0.0.1:7000" -node_id = 1 -[local_launches.kv-1] -program = "crowdb-kv-server" -args = ["--management-port", "7000"] -workdir = "/owned/node1" -env_file = "/owned/private.env" -"#, - ) - .unwrap(); - directory -} - -#[test] -fn stop_persists_only_local_launch_inputs_and_preserves_data() { - let directory = state(); - let sentinel = directory.path().join("operator-data"); - std::fs::write(&sentinel, b"preserve").unwrap(); - let (before, record) = s3::load(directory.path()).unwrap(); - let stopped = s3::stop(directory.path()).unwrap(); - assert_eq!(stopped.running_services, 0); - assert_eq!(stopped.total_services, 1); - let (after, reopened) = s3::load(directory.path()).unwrap(); - assert_eq!(after.servers, before.servers); - assert_eq!(after.local_launches, before.local_launches); - assert_eq!(reopened.endpoint, record.endpoint); - assert!(after.racks.is_empty()); - assert!(after.nodes.is_empty()); - let saved = std::fs::read_to_string(s3::config_path(directory.path())).unwrap(); - for forbidden in ["[[rack]]", "[[node]]", "[[store]]", "CROWDB_S3_MASTER_KEY"] { - assert!(!saved.contains(forbidden), "{saved}"); - } - assert!(!directory.path().join("console.toml").exists()); - assert_eq!(std::fs::read(sentinel).unwrap(), b"preserve"); - s3::delete(directory.path()).unwrap(); - assert!(!directory.path().exists()); -} - -#[test] -fn inline_master_key_is_rejected_without_rewriting_state() { - let directory = state(); - let path = s3::config_path(directory.path()); - let mut saved = std::fs::read_to_string(&path).unwrap(); - saved.push_str("\n[local_launches.kv-1.env]\nCROWDB_S3_MASTER_KEY = 'private-key'\n"); - std::fs::write(&path, &saved).unwrap(); - let error = s3::stop(directory.path()).unwrap_err(); - assert!(error.to_string().contains("inline secrets"), "{error}"); - assert_eq!(std::fs::read_to_string(path).unwrap(), saved); -} diff --git a/lib/crowdb-console-shared/tests/s3_mini_cluster_test.rs b/lib/crowdb-console-shared/tests/s3_mini_cluster_test.rs deleted file mode 100644 index 28dd25004..000000000 --- a/lib/crowdb-console-shared/tests/s3_mini_cluster_test.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2026-present Gian -// Licensed under the Apache License, Version 2.0. - -// Packaged storage lifecycle and node faults belong to the S3 container suite. -// Keep local ownership and launch-state validation independent of servers. -use crowdb_console_shared::ops::s3; -use crowdb_test_harness::test_dirs::TestDir; - -#[test] -fn foreign_nonempty_directory_is_not_a_cluster() { - let dir = TestDir::new("s3-mini-foreign").expect("create test directory"); - std::fs::write(dir.path().join("owned-by-user"), b"keep").expect("write sentinel"); - - let error = s3::status(dir.path()).expect_err("foreign directory must be rejected"); - - assert!(error.to_string().contains("is not a CROWDB S3 mini-cluster")); - assert_eq!(std::fs::read(dir.path().join("owned-by-user")).unwrap(), b"keep"); -} - -#[test] -fn incomplete_marker_fails_closed() { - let dir = TestDir::new("s3-mini-incomplete").expect("create test directory"); - std::fs::write(dir.path().join("s3-mini-cluster.json"), b"{}").expect("write marker"); - - let error = s3::status(dir.path()).expect_err("incomplete marker must be rejected"); - - assert!(error.to_string().contains("config error")); -} - -#[test] -fn local_launch_state_rejects_topology_and_legacy_console_file() { - let dir = TestDir::new("s3-mini-local-only").expect("create test directory"); - std::fs::write( - dir.path().join("s3-mini-cluster.json"), - r#"{"version":1,"endpoint":"http://127.0.0.1:16000","tenant":"local"}"#, - ) - .unwrap(); - std::fs::write(dir.path().join("console.toml"), "[[rack]]\nid = 1\n").unwrap(); - assert!( - s3::status(dir.path()).is_err(), - "legacy topology must not be loaded" - ); - std::fs::write( - dir.path().join("s3-local-state.toml"), - "version = 1\ngroup0_seeds = ['http://127.0.0.1:10000']\n[[rack]]\nid = 1\n", - ) - .unwrap(); - let error = s3::status(dir.path()).expect_err("local state cannot contain topology"); - assert!(error.to_string().contains("unknown field"), "{error}"); -} diff --git a/lib/crowdb-console-shared/tests/s3_transport_test.rs b/lib/crowdb-console-shared/tests/s3_transport_test.rs new file mode 100644 index 000000000..e2320265a --- /dev/null +++ b/lib/crowdb-console-shared/tests/s3_transport_test.rs @@ -0,0 +1,78 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_console_shared::ops::s3::{sign_request, S3Credentials, S3HttpClient}; +use reqwest::{header::HeaderMap, Method}; + +fn credentials() -> S3Credentials { + S3Credentials { + access: "test-access".into(), + secret: "private-secret".into(), + region: "us-east-1".into(), + session: Some("private-session-token".into()), + } +} + +#[test] +fn selected_origins_reject_embedded_credentials_and_non_origin_paths() { + for endpoint in [ + "http://user:secret@127.0.0.1:9091", + "http://127.0.0.1:9091/api", + "http://127.0.0.1:9091?token=private-secret", + "file:///private-secret", + "http://127.0.0.1:0", + ] { + let error = S3HttpClient::signed(endpoint.into(), credentials()) + .err() + .expect("invalid endpoint"); + assert!(!error.to_string().contains("private-secret")); + } + let mut empty = credentials(); + empty.secret.clear(); + assert!(S3HttpClient::signed("http://127.0.0.1:9091".into(), empty).is_err()); +} + +#[test] +fn authentication_headers_are_sensitive_in_structured_diagnostics() { + let mut headers = HeaderMap::new(); + let url = "http://127.0.0.1:9091/bucket/key?prefix=a%20b&list-type=2" + .parse() + .unwrap(); + sign_request(&credentials(), &Method::GET, &url, b"", &mut headers).unwrap(); + assert!(headers["authorization"].is_sensitive()); + assert!(headers["x-amz-security-token"].is_sensitive()); + let debug = format!("{headers:?}"); + assert!(!debug.contains("private-session-token")); + assert!(!debug.contains("private-secret")); +} + +#[tokio::test] +async fn selected_s3_redirects_are_reported_without_forwarding_credentials() { + use axum::{http::StatusCode, routing::get, Router}; + let destination = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let location = format!("http://{}", destination.local_addr().unwrap()); + let first = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", first.local_addr().unwrap()); + let router = Router::new().route( + "/", + get(move || { + let location = location.clone(); + async move { (StatusCode::TEMPORARY_REDIRECT, [("location", location)]) } + }), + ); + let server = tokio::spawn(async move { + axum::serve(first, router).await.unwrap(); + }); + let client = S3HttpClient::signed(endpoint, credentials()).unwrap(); + let reply = client + .request_exchange(Method::GET, None, None, &[], None, None) + .await + .unwrap(); + assert_eq!(reply.status, StatusCode::TEMPORARY_REDIRECT); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), destination.accept()) + .await + .is_err() + ); + server.abort(); +} diff --git a/lib/crowdb-diskio-client/Cargo.toml b/lib/crowdb-diskio-client/Cargo.toml index 656821d1b..906bf33c9 100644 --- a/lib/crowdb-diskio-client/Cargo.toml +++ b/lib/crowdb-diskio-client/Cargo.toml @@ -32,4 +32,4 @@ tracing = { workspace = true } crowdb-kv = { path = "../crowdb-kv" } crowdb-diskio-client = { path = ".", features = ["test-util"] } crowdb-rpc-ffi = { path = "../crowdb-rpc/ffi", features = ["test-util"] } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time"] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "net", "io-util"] } diff --git a/lib/crowdb-diskio-client/src/client.rs b/lib/crowdb-diskio-client/src/client.rs index 5b43033e9..daa1614ff 100644 --- a/lib/crowdb-diskio-client/src/client.rs +++ b/lib/crowdb-diskio-client/src/client.rs @@ -36,6 +36,8 @@ pub struct WireWriteTarget { pub zone_index: u32, pub zone_offset: u64, pub ordering_zone_offset: u64, + pub sync: bool, + pub write_create_time_ms: u64, } /// Disk I/O return codes (mirrors `FBDiskIoRetCode`). @@ -49,9 +51,8 @@ pub enum DiskIoRetCode { PartialWrite = 4, InvalidAlignment = 5, ConnectionError = 6, - /// Compatibility response from an older `DiskIO` server. - StaleAllocation = 7, - OldRequest = 8, + OldRequest = 7, + FsyncFailed = 8, } #[derive(Clone, Copy)] @@ -61,6 +62,8 @@ struct WriteTarget { zone_offset: u64, size: u32, ordering_zone_offset: u64, + sync: bool, + write_create_time_ms: u64, } impl From for DiskIoRetCode { @@ -72,8 +75,8 @@ impl From for DiskIoRetCode { 4 => Self::PartialWrite, 5 => Self::InvalidAlignment, 6 => Self::ConnectionError, - 7 => Self::StaleAllocation, - 8 => Self::OldRequest, + 7 => Self::OldRequest, + 8 => Self::FsyncFailed, _ => Self::IoError, } } @@ -165,6 +168,8 @@ impl WireClient { zone_offset, size, ordering_zone_offset: zone_offset, + sync: false, + write_create_time_ms: unix_time_ms(), }, Buffer::from_vec(data), ) @@ -196,6 +201,8 @@ impl WireClient { zone_offset, size, ordering_zone_offset: zone_offset, + sync: false, + write_create_time_ms: unix_time_ms(), }, Buffer::from_owned_bytes(data), ) @@ -224,6 +231,8 @@ impl WireClient { zone_offset: target.zone_offset, size, ordering_zone_offset: target.ordering_zone_offset, + sync: target.sync, + write_create_time_ms: target.write_create_time_ms, }, Buffer::from_owned_bytes(data), ) @@ -258,7 +267,8 @@ impl WireClient { zone_offset: target.zone_offset, size, ordering_zone_offset: target.ordering_zone_offset, - write_create_time_ms: unix_time_ms(), + write_create_time_ms: target.write_create_time_ms, + sync: target.sync, }, ); fbb.finish(off, None); @@ -295,7 +305,8 @@ impl WireClient { zone_offset: target.zone_offset, size: target.size, ordering_zone_offset: target.ordering_zone_offset, - write_create_time_ms: unix_time_ms(), + write_create_time_ms: target.write_create_time_ms, + sync: target.sync, }, ); fbb.finish(off, None); @@ -392,6 +403,22 @@ impl WireClient { parse_ret_code(&resp) } + /// Parse the server's proof that a successful write was synchronized. + /// + /// # Errors + /// Rejects failed or invalid replies and returns explicit synchronization completion. + pub async fn await_synchronized_write_response(future: CallFuture) -> WireResult { + let response = future.await.map_err(WireError::from)?; + parse_ret_code(&response)?; + let control = response + .control + .as_ref() + .ok_or_else(|| WireError::Protocol("missing write response control".into()))?; + let reply = flatbuffers::root::(control.bytes()) + .map_err(|error| WireError::Protocol(format!("invalid write response: {error}")))?; + Ok(reply.sync_completed()) + } + /// Parse a read response from a completed `CallFuture`. /// Returns (`ret_code`, data) on success. /// diff --git a/lib/crowdb-diskio-client/src/semantic.rs b/lib/crowdb-diskio-client/src/semantic.rs index fe596705c..fcb207d96 100644 --- a/lib/crowdb-diskio-client/src/semantic.rs +++ b/lib/crowdb-diskio-client/src/semantic.rs @@ -24,6 +24,8 @@ use crate::{DiskId, DiskioError, DiskioResult, DiskioStatus, SegmentTarget}; mod connection; mod native_routes; pub use native_routes::NativeDiskRouteResolver; +mod drain; +mod write; mod write_views; const TOPOLOGY_REFRESH_INTERVAL_MS: u64 = 5_000; @@ -55,6 +57,7 @@ pub enum Durability { pub struct OperationOptions { pub lane: TrafficLane, pub deadline: Instant, + pub retry_writes: bool, } impl OperationOptions { @@ -63,6 +66,7 @@ impl OperationOptions { Self { lane: TrafficLane::Normal, deadline: Instant::now() + timeout, + retry_writes: true, } } @@ -71,6 +75,13 @@ impl OperationOptions { self.lane = TrafficLane::Priority; self } + + /// Submit a write once; an uncertain response cannot authorize reuse. + #[must_use] + pub fn without_write_retries(mut self) -> Self { + self.retry_writes = false; + self + } } /// Discovery, pool, and retry bounds for a semantic client. @@ -216,6 +227,12 @@ enum WritePayload { Views(Arc<[Bytes]>), } +#[derive(Clone, Copy, PartialEq, Eq)] +enum WriteMode { + Data(Durability), + Drain, +} + impl WritePayload { fn len(&self) -> usize { match self { @@ -527,131 +544,6 @@ impl DiskioClient { Err(DiskioError::DeadlineExceeded) } - /// Write caller-owned bytes to an exact segment-relative range. - /// - /// # Errors - /// - /// Returns a typed input, topology, backpressure, disk, durability, or - /// ambiguous-outcome error. - pub async fn write( - &self, - target: SegmentTarget, - offset: u64, - data: Bytes, - durability: Durability, - options: OperationOptions, - ) -> DiskioResult<()> { - self.write_payload( - target, - offset, - WritePayload::Contiguous(data), - durability, - options, - ) - .await - } - - async fn write_payload( - &self, - target: SegmentTarget, - offset: u64, - data: WritePayload, - durability: Durability, - options: OperationOptions, - ) -> DiskioResult<()> { - self.refresh_topology_if_due().await; - if data.is_empty() { - return Err(DiskioError::InvalidInput("write data must be nonempty".into())); - } - let (zone_offset, _) = target.checked_range(offset, data.len())?; - let _inflight = self.begin_operation(OperationKind::Write, options.lane)?; - let mut backoff = self.config.reconnect_initial_backoff; - let mut last_error = None; - let mut only_backpressure = true; - for attempt in 0..self.config.retry_attempts { - let (route, selected) = self.select(target.disk_id, options.lane)?; - let write_target = WireWriteTarget { - disk_id: target.disk_id, - zone_index: target.zone_index, - zone_offset, - ordering_zone_offset: target.segment_base, - }; - let submitted = match &data { - WritePayload::Contiguous(data) => { - self.wire - .write_segment_bytes(&self.server, &selected, write_target, data.clone()) - } - WritePayload::Views(views) => self.wire.write_segment_views( - &self.server, - &selected, - write_target, - views.iter().cloned().collect(), - ), - }; - let future = match submitted { - Ok(future) => future, - Err(error) => { - let error = Self::classify_wire(error, OperationKind::Write); - only_backpressure &= matches!(error, DiskioError::Backpressure(_)); - last_error = Some(error); - if attempt + 1 < self.config.retry_attempts { - if self - .prepare_retry(&route, options.lane, &selected, &mut backoff, options.deadline) - .await - .is_err() - { - break; - } - continue; - } - break; - } - }; - match self - .await_write(future, options.deadline, OperationKind::Write) - .await - { - Ok(()) => { - if durability == Durability::Fsync { - self.fsync_inner(target.disk_id, options, true).await?; - } - return Ok(()); - } - Err(DiskioError::PartialWrite) => return Err(DiskioError::PartialWrite), - Err( - error @ (DiskioError::DiskFailure(_) - | DiskioError::DiskNotFound - | DiskioError::InvalidInput(_)), - ) => { - return Err(error); - } - Err(error) => { - only_backpressure = false; - last_error = Some(error); - if attempt + 1 < self.config.retry_attempts - && self - .prepare_retry(&route, options.lane, &selected, &mut backoff, options.deadline) - .await - .is_err() - { - break; - } - } - } - } - self.counters.ambiguous_writes.fetch_add(1, Ordering::Relaxed); - if only_backpressure { - let Some(error @ DiskioError::Backpressure(_)) = last_error.as_ref() else { - return Err(DiskioError::AmbiguousWrite("completion was not observed".into())); - }; - return Err(error.clone()); - } - Err(DiskioError::AmbiguousWrite(last_error.map_or_else( - || "completion was not observed".into(), - |error| error.to_string(), - ))) - } - /// Flush all prior writes to one disk on the selected lane. /// /// # Errors @@ -660,7 +552,7 @@ impl DiskioClient { /// error. pub async fn fsync(&self, disk_id: DiskId, options: OperationOptions) -> DiskioResult<()> { self.refresh_topology_if_due().await; - self.fsync_inner(disk_id, options, false).await + self.fsync_inner(disk_id, options).await } /// Return an operation policy using the configured default deadline. @@ -724,16 +616,6 @@ impl DiskioClient { }) } - fn begin_nested_operation(&self, kind: OperationKind) -> InflightGuard<'_> { - InflightGuard { - counters: &self.counters, - kind, - started: Instant::now(), - counted_inflight: false, - lane_inflight: None, - } - } - fn select(&self, disk_id: DiskId, lane: TrafficLane) -> DiskioResult<(DiskRoute, SelectedConnection)> { let route = self.routes.load().routes.get(&disk_id).cloned().ok_or_else(|| { DiskioError::TopologyUnavailable(format!( @@ -808,17 +690,8 @@ impl DiskioClient { Ok(()) } - async fn fsync_inner( - &self, - disk_id: DiskId, - options: OperationOptions, - durability_context: bool, - ) -> DiskioResult<()> { - let _inflight = if durability_context { - self.begin_nested_operation(OperationKind::Fsync) - } else { - self.begin_operation(OperationKind::Fsync, options.lane)? - }; + async fn fsync_inner(&self, disk_id: DiskId, options: OperationOptions) -> DiskioResult<()> { + let _inflight = self.begin_operation(OperationKind::Fsync, options.lane)?; let mut backoff = self.config.reconnect_initial_backoff; let mut last_error = None; for attempt in 0..self.config.retry_attempts { @@ -846,11 +719,7 @@ impl DiskioClient { { Ok(()) => return Ok(()), Err(error @ (DiskioError::DiskFailure(_) | DiskioError::DiskNotFound)) => { - return if durability_context { - Err(DiskioError::DurabilityFailure(error.to_string())) - } else { - Err(error) - }; + return Err(error); } Err(error) => { last_error = Some(error); @@ -869,11 +738,7 @@ impl DiskioClient { || "fsync completion was not observed".into(), |error| error.to_string(), ); - if durability_context { - Err(DiskioError::DurabilityFailure(message)) - } else { - Err(DiskioError::TransportUnavailable(message)) - } + Err(DiskioError::TransportUnavailable(message)) } async fn await_write( @@ -937,6 +802,9 @@ impl DiskioClient { WireError::Protocol(message) => DiskioError::Protocol(message), WireError::IoError(DiskIoRetCode::DiskNotExist) => DiskioError::DiskNotFound, WireError::IoError(DiskIoRetCode::PartialWrite) => DiskioError::PartialWrite, + WireError::IoError(DiskIoRetCode::FsyncFailed) => { + DiskioError::DurabilityFailure("DiskIO synchronized write flush failed".into()) + } WireError::IoError(DiskIoRetCode::InvalidAlignment) => { DiskioError::InvalidInput("DiskIO rejected address alignment".into()) } diff --git a/lib/crowdb-diskio-client/src/semantic/drain.rs b/lib/crowdb-diskio-client/src/semantic/drain.rs new file mode 100644 index 000000000..80f228782 --- /dev/null +++ b/lib/crowdb-diskio-client/src/semantic/drain.rs @@ -0,0 +1,26 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Segment ordering barriers after caller-proven exclusion of old producers. + +use super::{Bytes, DiskioClient, DiskioResult, OperationOptions, SegmentTarget, WriteMode, WritePayload}; + +impl DiskioClient { + /// Wait for prior segment writes, then synchronize them through one reply. + /// The caller must independently exclude future writes before treating this + /// as authority to observe a final prefix or reuse physical storage. + /// + /// # Errors + /// Returns input, topology, timeout, queue, disk, synchronization or ambiguous + /// outcome errors. A failed or unproven response never establishes a drain. + pub async fn drain_segment(&self, target: SegmentTarget, options: OperationOptions) -> DiskioResult<()> { + self.write_payload( + target, + 0, + WritePayload::Contiguous(Bytes::new()), + WriteMode::Drain, + options, + ) + .await + } +} diff --git a/lib/crowdb-diskio-client/src/semantic/write.rs b/lib/crowdb-diskio-client/src/semantic/write.rs new file mode 100644 index 000000000..d185fc8cc --- /dev/null +++ b/lib/crowdb-diskio-client/src/semantic/write.rs @@ -0,0 +1,179 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded writes requiring explicit synchronization completion. + +use super::{ + unix_time_ms, Bytes, DiskioClient, DiskioError, DiskioResult, Durability, Instant, OperationKind, + OperationOptions, Ordering, SegmentTarget, WireClient, WireWriteTarget, WriteMode, WritePayload, +}; +use crate::client::WireError; + +impl DiskioClient { + /// Write caller-owned bytes to an exact segment-relative range. + /// + /// # Errors + /// + /// Returns a typed input, topology, backpressure, disk, durability, or + /// ambiguous-outcome error. + pub async fn write( + &self, + target: SegmentTarget, + offset: u64, + data: Bytes, + durability: Durability, + options: OperationOptions, + ) -> DiskioResult<()> { + self.write_payload( + target, + offset, + WritePayload::Contiguous(data), + WriteMode::Data(durability), + options, + ) + .await + } + + pub(super) async fn write_payload( + &self, + target: SegmentTarget, + offset: u64, + data: WritePayload, + mode: WriteMode, + options: OperationOptions, + ) -> DiskioResult<()> { + if Instant::now() >= options.deadline { + return Err(DiskioError::DeadlineExceeded); + } + let write_create_time_ms = unix_time_ms(); + self.refresh_topology_if_due().await; + if data.is_empty() && mode != WriteMode::Drain { + return Err(DiskioError::InvalidInput("write data must be nonempty".into())); + } + let (zone_offset, _) = target.checked_range(offset, data.len())?; + let _inflight = self.begin_operation(OperationKind::Write, options.lane)?; + let mut backoff = self.config.reconnect_initial_backoff; + let mut last_error = None; + let mut only_backpressure = true; + let attempts = if options.retry_writes { + self.config.retry_attempts + } else { + 1 + }; + for attempt in 0..attempts { + if Instant::now() >= options.deadline { + if attempt == 0 { + return Err(DiskioError::DeadlineExceeded); + } + break; + } + let (route, selected) = self.select(target.disk_id, options.lane)?; + let write_target = WireWriteTarget { + disk_id: target.disk_id, + zone_index: target.zone_index, + zone_offset, + ordering_zone_offset: target.segment_base, + sync: matches!(mode, WriteMode::Data(Durability::Fsync) | WriteMode::Drain), + write_create_time_ms, + }; + let future = match self.submit_payload(&selected, write_target, &data) { + Ok(future) => future, + Err(error) => { + let error = Self::classify_wire(error, OperationKind::Write); + only_backpressure &= matches!(error, DiskioError::Backpressure(_)); + last_error = Some(error); + if attempt + 1 < attempts { + if self + .prepare_retry(&route, options.lane, &selected, &mut backoff, options.deadline) + .await + .is_err() + { + break; + } + continue; + } + break; + } + }; + match self.await_synchronized_write(future, options.deadline).await { + Ok(synchronized) => { + if write_target.sync && !synchronized { + return Err(DiskioError::DurabilityFailure( + "synchronized write response omitted completion proof".into(), + )); + } + return Ok(()); + } + Err(DiskioError::PartialWrite) => return Err(DiskioError::PartialWrite), + Err( + error @ (DiskioError::DiskFailure(_) + | DiskioError::DiskNotFound + | DiskioError::InvalidInput(_) + | DiskioError::DurabilityFailure(_)), + ) => { + return Err(error); + } + Err(error) => { + only_backpressure = false; + last_error = Some(error); + if attempt + 1 < attempts + && self + .prepare_retry(&route, options.lane, &selected, &mut backoff, options.deadline) + .await + .is_err() + { + break; + } + } + } + } + self.counters.ambiguous_writes.fetch_add(1, Ordering::Relaxed); + if only_backpressure { + let Some(error @ DiskioError::Backpressure(_)) = last_error.as_ref() else { + return Err(DiskioError::AmbiguousWrite("completion was not observed".into())); + }; + return Err(error.clone()); + } + Err(DiskioError::AmbiguousWrite(last_error.map_or_else( + || "completion was not observed".into(), + |error| error.to_string(), + ))) + } + + fn submit_payload( + &self, + connection: &crowdb_rpc_ffi::SelectedConnection, + target: WireWriteTarget, + data: &WritePayload, + ) -> Result { + match data { + WritePayload::Contiguous(data) => { + self.wire + .write_segment_bytes(&self.server, connection, target, data.clone()) + } + WritePayload::Views(views) => self.wire.write_segment_views( + &self.server, + connection, + target, + views.iter().cloned().collect(), + ), + } + } + + async fn await_synchronized_write( + &self, + future: crowdb_rpc_ffi::CallFuture, + deadline: Instant, + ) -> DiskioResult { + if Instant::now() >= deadline { + return Err(DiskioError::DeadlineExceeded); + } + tokio::time::timeout_at( + tokio::time::Instant::from_std(deadline), + WireClient::await_synchronized_write_response(future), + ) + .await + .map_err(|_| DiskioError::DeadlineExceeded)? + .map_err(|error| Self::classify_wire(error, OperationKind::Write)) + } +} diff --git a/lib/crowdb-diskio-client/src/semantic/write_views.rs b/lib/crowdb-diskio-client/src/semantic/write_views.rs index dc8ef3b0b..73aeeb00d 100644 --- a/lib/crowdb-diskio-client/src/semantic/write_views.rs +++ b/lib/crowdb-diskio-client/src/semantic/write_views.rs @@ -2,7 +2,8 @@ // Licensed under the Apache License, Version 2.0. use super::{ - Bytes, DiskioClient, DiskioError, DiskioResult, Durability, OperationOptions, SegmentTarget, WritePayload, + Bytes, DiskioClient, DiskioError, DiskioResult, Durability, OperationOptions, SegmentTarget, WriteMode, + WritePayload, }; impl DiskioClient { @@ -12,8 +13,8 @@ impl DiskioClient { /// /// Returns a typed input, topology, backpressure, disk, durability, or /// ambiguous-outcome error. Larger view lists use consecutive bounded RPC - /// frames without copying payload. Frames complete in byte order; requested - /// fsync follows the whole range. + /// frames without copying payload. Frames complete in byte order; the last + /// request synchronizes the whole range when requested. pub async fn write_views( &self, target: SegmentTarget, @@ -40,7 +41,7 @@ impl DiskioClient { target, offset, WritePayload::Views(data.into()), - durability, + WriteMode::Data(durability), options, ) .await; @@ -49,8 +50,8 @@ impl DiskioClient { let mut views = data.into_iter(); // All frames belong to one strip's disk block. Wait for each reply // before submitting the next frame, even at aligned boundaries: RPC - // arrival order across connections is not guaranteed, and server-side - // partial-block padding can overwrite a later frame's stored bytes. + // arrival order across connections is not guaranteed. This preserves + // the contiguous prefix and makes final synchronization cover all data. // Callers may overlap independent strip blocks and different disks // holding the same strip's shards or mirror replicas. loop { @@ -66,15 +67,16 @@ impl DiskioClient { target, position, WritePayload::Views(batch.into()), - Durability::Buffered, + WriteMode::Data(if views.len() == 0 { + durability + } else { + Durability::Buffered + }), options, ) .await?; position += length as u64; } - if durability == Durability::Fsync { - self.fsync(target.disk_id, options).await?; - } Ok(()) } } diff --git a/lib/crowdb-diskio-client/src/topology.rs b/lib/crowdb-diskio-client/src/topology.rs index f3f162909..6ab709e34 100644 --- a/lib/crowdb-diskio-client/src/topology.rs +++ b/lib/crowdb-diskio-client/src/topology.rs @@ -96,14 +96,11 @@ pub(crate) async fn discover( service: &ServiceRegistryClient, hardware: &HardwareClient, ) -> DiskioResult { - let instances = service - .read_all_diskio_instances() - .await + let (instances, disks) = tokio::join!(service.read_all_diskio_instances(), hardware.list_all_disks()); + let instances = instances .map_err(|error| DiskioError::TopologyUnavailable(format!("read DiskIO instances: {error}")))?; - let mut disks = hardware - .list_all_disks() - .await - .map_err(|error| DiskioError::TopologyUnavailable(format!("read hardware disks: {error}")))?; + let mut disks = + disks.map_err(|error| DiskioError::TopologyUnavailable(format!("read hardware disks: {error}")))?; disks.retain(|disk| { !matches!( crowdb_protocol::common::HwStatus::try_from(disk.value.status), @@ -118,15 +115,14 @@ pub(crate) async fn discover_disk( hardware: &HardwareClient, key: &crowdb_protocol::key::DiskKey, ) -> DiskioResult { - let value = hardware - .get_disk(key.rack_id, key.node_id, key.disk_group_id, &key.disk_id) - .await + let (value, instances) = tokio::join!( + hardware.get_disk(key.rack_id, key.node_id, key.disk_group_id, &key.disk_id), + service.read_all_diskio_instances() + ); + let value = value .map_err(|error| DiskioError::TopologyUnavailable(error.to_string()))? .ok_or_else(|| DiskioError::TopologyUnavailable("administrative disk membership missing".into()))?; - let instances = service - .read_all_diskio_instances() - .await - .map_err(|error| DiskioError::TopologyUnavailable(error.to_string()))?; + let instances = instances.map_err(|error| DiskioError::TopologyUnavailable(error.to_string()))?; let owners = instances .into_iter() .map(|(_, instance)| instance) diff --git a/lib/crowdb-diskio-client/tests/common/synchronized_peer.rs b/lib/crowdb-diskio-client/tests/common/synchronized_peer.rs new file mode 100644 index 000000000..50b6c5ce4 --- /dev/null +++ b/lib/crowdb-diskio-client/tests/common/synchronized_peer.rs @@ -0,0 +1,198 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_diskio_client::{DiskId, DiskioClient, DiskioClientConfig, TestDiskRoute}; +use crowdb_protocol::diskio_fb::{ + FBDiskFsyncRequest, FBDiskFsyncResponse, FBDiskFsyncResponseArgs, FBDiskIoRetCode, FBDiskWriteRequest, + FBDiskWriteResponse, FBDiskWriteResponseArgs, +}; +use crowdb_protocol::fb::FBMsgType; +use flatbuffers::FlatBufferBuilder; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, +}; +use std::time::Duration; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::Notify; +use tokio::task::{JoinHandle, JoinSet}; + +#[derive(Default)] +pub struct TestCounts { + pub writes: AtomicUsize, + pub sync_requests: AtomicUsize, + pub fsyncs: AtomicUsize, + pub bytes: AtomicUsize, + pub requests: Mutex>, +} + +pub struct TestDiskioPeer { + pub counts: Arc, + pub gate: Arc, + address: String, + worker: JoinHandle<()>, +} + +impl TestDiskioPeer { + pub async fn start(sync_proof: bool, delayed: bool, code: FBDiskIoRetCode) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap().to_string(); + let counts = Arc::new(TestCounts::default()); + let gate = Arc::new(Notify::new()); + let worker = tokio::spawn({ + let counts = Arc::clone(&counts); + let gate = Arc::clone(&gate); + async move { + let mut connections = JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let (stream, _) = accepted.unwrap(); + connections.spawn(serve(stream, Arc::clone(&counts), + Arc::clone(&gate), sync_proof, delayed, code)); + } + done = connections.join_next(), if !connections.is_empty() => { + done.unwrap().unwrap(); + } + } + } + } + }); + Self { + counts, + gate, + address, + worker, + } + } + + pub fn client(&self) -> DiskioClient { + self.client_with_attempts(1) + } + + pub fn client_with_attempts(&self, retry_attempts: usize) -> DiskioClient { + DiskioClient::connect_for_tests( + vec![self.route(DiskId::new(1, 2))], + DiskioClientConfig { + retry_attempts, + ..Default::default() + }, + ) + .unwrap() + } + + pub fn route(&self, disk_id: DiskId) -> TestDiskRoute { + TestDiskRoute { + disk_id, + rack_id: 1, + node_id: disk_id.high, + disk_group_id: 1, + instance_id: disk_id.high, + endpoint: self.address.clone(), + } + } + + pub async fn wait_for_write(&self) { + tokio::time::timeout(Duration::from_secs(3), async { + let mut tick = tokio::time::interval(Duration::from_millis(1)); + while self.counts.writes.load(Ordering::Acquire) == 0 { + tick.tick().await; + } + }) + .await + .unwrap(); + } +} + +impl Drop for TestDiskioPeer { + fn drop(&mut self) { + self.worker.abort(); + } +} + +async fn serve( + mut stream: TcpStream, + counts: Arc, + gate: Arc, + sync_proof: bool, + delayed: bool, + code: FBDiskIoRetCode, +) { + loop { + let mut header = [0; 14]; + if stream.read_exact(&mut header).await.is_err() { + return; + } + assert_eq!(u16::from_le_bytes(header[..2].try_into().unwrap()), 0xca70); + let kind = u16::from_le_bytes(header[2..4].try_into().unwrap()); + let length = u32::from_le_bytes(header[4..8].try_into().unwrap()) as usize; + let data_length = u32::from_le_bytes(header[8..12].try_into().unwrap()) as usize; + assert_eq!(header[12], 14); + assert!(length <= 4096 && data_length <= 1024 * 1024); + let mut control = vec![0; length]; + let mut data = vec![0; data_length]; + stream.read_exact(&mut control).await.unwrap(); + stream.read_exact(&mut data).await.unwrap(); + let (id, sync) = if kind == FBMsgType::EDiskWriteRequest.0 as u16 { + let request = flatbuffers::root::(&control).unwrap(); + assert_eq!(data.len(), request.size() as usize); + assert!(data.iter().all(|byte| *byte == 7)); + assert_eq!(request.ordering_zone_offset(), 0); + counts.bytes.fetch_add(data.len(), Ordering::AcqRel); + counts + .requests + .lock() + .unwrap() + .push((request.zone_offset(), request.write_create_time_ms())); + counts.writes.fetch_add(1, Ordering::Release); + if request.sync() { + counts.sync_requests.fetch_add(1, Ordering::Relaxed); + } + (request.id(), request.sync()) + } else { + assert_eq!(kind, FBMsgType::EDiskFsyncRequest.0 as u16); + counts.fsyncs.fetch_add(1, Ordering::Relaxed); + ( + flatbuffers::root::(&control).unwrap().id(), + false, + ) + }; + if sync && delayed { + gate.notified().await; + } + let mut builder = FlatBufferBuilder::new(); + if kind == FBMsgType::EDiskWriteRequest.0 as u16 { + let reply = FBDiskWriteResponse::create( + &mut builder, + &FBDiskWriteResponseArgs { + id, + ret_code: code, + sync_completed: sync_proof && sync && code == FBDiskIoRetCode::Success, + ..Default::default() + }, + ); + builder.finish(reply, None); + } else { + let reply = FBDiskFsyncResponse::create( + &mut builder, + &FBDiskFsyncResponseArgs { + id, + ..Default::default() + }, + ); + builder.finish(reply, None); + } + let reply = builder.finished_data(); + let mut response = Vec::with_capacity(14 + reply.len()); + response.extend_from_slice(&0xca70_u16.to_le_bytes()); + response.extend_from_slice(&(kind + 1).to_le_bytes()); + response.extend_from_slice(&u32::try_from(reply.len()).unwrap().to_le_bytes()); + response.extend_from_slice(&0_u32.to_le_bytes()); + response.extend_from_slice(&[14, 0]); + response.extend_from_slice(reply); + if stream.write_all(&response).await.is_err() { + return; + } + } +} diff --git a/lib/crowdb-diskio-client/tests/segment_drain_test.rs b/lib/crowdb-diskio-client/tests/segment_drain_test.rs new file mode 100644 index 000000000..cc384a443 --- /dev/null +++ b/lib/crowdb-diskio-client/tests/segment_drain_test.rs @@ -0,0 +1,61 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/synchronized_peer.rs"] +mod peer; + +use bytes::Bytes; +use crowdb_diskio_client::{DiskId, DiskioError, Durability, OperationOptions, SegmentTarget}; +use crowdb_protocol::diskio_fb::FBDiskIoRetCode; +use std::sync::atomic::Ordering; +use std::time::Duration; + +#[tokio::test] +async fn drain_requires_a_proven_empty_ordered_synchronized_request() { + for proof in [true, false] { + let peer = peer::TestDiskioPeer::start(proof, proof, FBDiskIoRetCode::Success).await; + let target = SegmentTarget::new(DiskId::new(1, 2), 0, 0, 1, 4096).unwrap(); + let client = peer.client(); + let operation = tokio::spawn(async move { + client + .drain_segment(target, OperationOptions::within(Duration::from_secs(3))) + .await + }); + peer.wait_for_write().await; + if proof { + assert!(!operation.is_finished()); + peer.gate.notify_one(); + } + let result = operation.await.unwrap(); + if proof { + result.unwrap(); + } else { + assert!( + matches!(result, Err(DiskioError::DurabilityFailure(_))), + "{result:?}" + ); + } + assert_eq!(peer.counts.writes.load(Ordering::Acquire), 1); + assert_eq!(peer.counts.sync_requests.load(Ordering::Acquire), 1); + assert_eq!(peer.counts.bytes.load(Ordering::Acquire), 0); + assert_eq!(peer.counts.fsyncs.load(Ordering::Acquire), 0); + } +} + +#[tokio::test] +async fn an_empty_data_write_cannot_implicitly_become_a_drain() { + let peer = peer::TestDiskioPeer::start(true, false, FBDiskIoRetCode::Success).await; + let target = SegmentTarget::new(DiskId::new(1, 2), 0, 0, 1, 4096).unwrap(); + let result = peer + .client() + .write( + target, + 0, + Bytes::new(), + Durability::Fsync, + OperationOptions::within(Duration::from_secs(3)), + ) + .await; + assert!(matches!(result, Err(DiskioError::InvalidInput(_))), "{result:?}"); + assert_eq!(peer.counts.writes.load(Ordering::Acquire), 0); +} diff --git a/lib/crowdb-diskio-client/tests/synchronized_write_test.rs b/lib/crowdb-diskio-client/tests/synchronized_write_test.rs new file mode 100644 index 000000000..1f993f03a --- /dev/null +++ b/lib/crowdb-diskio-client/tests/synchronized_write_test.rs @@ -0,0 +1,198 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/synchronized_peer.rs"] +mod peer; + +use bytes::Bytes; +use crowdb_diskio_client::{DiskId, DiskioError, Durability, OperationOptions, SegmentTarget}; +use crowdb_protocol::diskio_fb::FBDiskIoRetCode; +use peer::TestDiskioPeer; +use std::sync::atomic::Ordering; +use std::time::Duration; + +fn target() -> SegmentTarget { + SegmentTarget::new(DiskId::new(1, 2), 0, 0, 1024, 4096).unwrap() +} +fn options() -> OperationOptions { + OperationOptions::within(Duration::from_secs(3)) +} + +#[tokio::test] +async fn synchronized_write_waits_for_its_proof_and_needs_no_fsync_rpc() { + let peer = TestDiskioPeer::start(true, true, FBDiskIoRetCode::Success).await; + let client = peer.client(); + let operation = tokio::spawn(async move { + client + .write( + target(), + 0, + Bytes::from_static(&[7; 32]), + Durability::Fsync, + options(), + ) + .await + }); + peer.wait_for_write().await; + assert!(!operation.is_finished()); + assert_eq!(peer.counts.fsyncs.load(Ordering::Relaxed), 0); + peer.gate.notify_one(); + operation.await.unwrap().unwrap(); + assert_eq!(peer.counts.writes.load(Ordering::Relaxed), 1); + assert_eq!(peer.counts.sync_requests.load(Ordering::Relaxed), 1); + assert_eq!(peer.counts.fsyncs.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn expired_write_deadline_rejects_before_submission() { + let peer = TestDiskioPeer::start(true, false, FBDiskIoRetCode::Success).await; + let mut expired = options(); + expired.deadline = std::time::Instant::now(); + assert_eq!( + peer.client() + .write( + target(), + 0, + Bytes::from_static(&[7; 32]), + Durability::Fsync, + expired + ) + .await, + Err(DiskioError::DeadlineExceeded) + ); + assert_eq!(peer.counts.writes.load(Ordering::Acquire), 0); +} + +#[tokio::test] +async fn initialization_never_retries_an_uncertain_response() { + let peer = TestDiskioPeer::start(true, false, FBDiskIoRetCode::ConnectionError).await; + let result = peer + .client_with_attempts(3) + .write( + target(), + 0, + Bytes::from_static(&[7; 32]), + Durability::Fsync, + options().without_write_retries(), + ) + .await; + assert!( + matches!(result, Err(DiskioError::AmbiguousWrite(_))), + "{result:?}" + ); + assert_eq!(peer.counts.writes.load(Ordering::Acquire), 1); +} + +#[tokio::test] +async fn retries_keep_the_original_write_creation_time() { + let peer = TestDiskioPeer::start(true, false, FBDiskIoRetCode::ConnectionError).await; + let result = peer + .client_with_attempts(3) + .write( + target(), + 0, + Bytes::from_static(&[7; 32]), + Durability::Fsync, + options(), + ) + .await; + assert!( + matches!(result, Err(DiskioError::AmbiguousWrite(_))), + "{result:?}" + ); + let requests = peer.counts.requests.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert!(requests[0].1 > 0); + assert!(requests.iter().all(|request| request == &requests[0])); +} + +#[tokio::test] +async fn synchronized_response_without_proof_fails_without_extra_io() { + let peer = TestDiskioPeer::start(false, false, FBDiskIoRetCode::Success).await; + let error = peer + .client() + .write( + target(), + 0, + Bytes::from_static(&[7; 32]), + Durability::Fsync, + options(), + ) + .await + .unwrap_err(); + assert!(matches!(error, DiskioError::DurabilityFailure(_))); + assert_eq!(peer.counts.writes.load(Ordering::Relaxed), 1); + assert_eq!(peer.counts.sync_requests.load(Ordering::Relaxed), 1); + assert_eq!(peer.counts.fsyncs.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn buffered_write_retains_no_flush_semantics() { + let peer = TestDiskioPeer::start(true, false, FBDiskIoRetCode::Success).await; + peer.client() + .write( + target(), + 0, + Bytes::from_static(&[7; 32]), + Durability::Buffered, + options(), + ) + .await + .unwrap(); + assert_eq!(peer.counts.writes.load(Ordering::Relaxed), 1); + assert_eq!(peer.counts.sync_requests.load(Ordering::Relaxed), 0); + assert_eq!(peer.counts.fsyncs.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn bounded_view_splits_synchronize_only_the_final_ordered_request() { + let peer = TestDiskioPeer::start(true, false, FBDiskIoRetCode::Success).await; + let views = vec![Bytes::from_static(&[7]); 17]; + peer.client() + .write_views(target(), 0, views, Durability::Fsync, options()) + .await + .unwrap(); + assert_eq!(peer.counts.bytes.load(Ordering::Relaxed), 17); + assert_eq!(peer.counts.writes.load(Ordering::Relaxed), 2); + assert_eq!( + peer.counts + .requests + .lock() + .unwrap() + .iter() + .map(|request| request.0) + .collect::>(), + vec![0, 16] + ); + assert_eq!(peer.counts.sync_requests.load(Ordering::Relaxed), 1); + assert_eq!(peer.counts.fsyncs.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn failed_flush_or_write_never_produces_success_or_a_fallback_request() { + for code in [ + FBDiskIoRetCode::FsyncFailed, + FBDiskIoRetCode::IoError, + FBDiskIoRetCode::PartialWrite, + ] { + let peer = TestDiskioPeer::start(true, false, code).await; + let error = peer + .client() + .write( + target(), + 0, + Bytes::from_static(&[7; 32]), + Durability::Fsync, + options(), + ) + .await + .unwrap_err(); + match code { + FBDiskIoRetCode::FsyncFailed => assert!(matches!(error, DiskioError::DurabilityFailure(_))), + FBDiskIoRetCode::IoError => assert!(matches!(error, DiskioError::DiskFailure(_))), + _ => assert_eq!(error, DiskioError::PartialWrite), + } + assert_eq!(peer.counts.writes.load(Ordering::Relaxed), 1); + assert_eq!(peer.counts.fsyncs.load(Ordering::Relaxed), 0); + } +} diff --git a/lib/crowdb-kv-client/src/binding/range.rs b/lib/crowdb-kv-client/src/binding/range.rs index bd598bb96..3c1c41aee 100644 --- a/lib/crowdb-kv-client/src/binding/range.rs +++ b/lib/crowdb-kv-client/src/binding/range.rs @@ -60,15 +60,21 @@ impl RangeBindingClient { /// or unsupported changes to the fixed service assignment. pub async fn refresh(&self) -> Result<()> { let maps = ChunkSlotMapClient::new(Arc::clone(&self.kv)); - let dynamic = maps.read_dynamic_service_snapshot().await?; - let map = if let Some(snapshot) = &dynamic { - snapshot.service().clone() - } else { - maps.read_service().await? - }; - let endpoints: HashMap<_, _> = ServiceRegistryClient::from_shared(Arc::clone(&self.kv)) - .read_all_chunkdb_instances() - .await? + let service = ServiceRegistryClient::from_shared(Arc::clone(&self.kv)); + let (layout, instances) = tokio::join!( + async { + let dynamic = maps.read_dynamic_service_snapshot().await?; + let map = if let Some(snapshot) = &dynamic { + snapshot.service().clone() + } else { + maps.read_service().await? + }; + Ok::<_, Error>((dynamic, map)) + }, + service.read_all_chunkdb_instances() + ); + let (dynamic, map) = layout?; + let endpoints: HashMap<_, _> = instances? .into_iter() .map(|(id, instance)| (id, instance.rpc_endpoint)) .collect(); diff --git a/lib/crowdb-kv-client/src/client/topology.rs b/lib/crowdb-kv-client/src/client/topology.rs index b04415231..b8bf73d95 100644 --- a/lib/crowdb-kv-client/src/client/topology.rs +++ b/lib/crowdb-kv-client/src/client/topology.rs @@ -271,21 +271,23 @@ impl TopologyCache { Ok(mut body) => { valid_response = true; address::resolve_local_endpoints(&mut body, seed); - let endpoint = required_group.and_then(|(store_id, group_id)| { - let store = body.stores.iter().find(|store| store.store_id == store_id)?; - let group = store.groups.iter().find(|group| group.group_id == group_id)?; - if group.leader_id == 0 { - None - } else if group.local_replica.id == group.leader_id { - store.listen_addr.clone() - } else { - group - .remotes - .iter() - .find(|remote| remote.id == group.leader_id) - .map(|remote| remote.endpoint.clone()) - } - }); + let endpoint = required_group + .and_then(|(store_id, group_id)| { + let store = body.stores.iter().find(|store| store.store_id == store_id)?; + let group = store.groups.iter().find(|group| group.group_id == group_id)?; + if group.leader_id == 0 { + None + } else if group.local_replica.id == group.leader_id { + store.listen_addr.clone() + } else { + group + .remotes + .iter() + .find(|remote| remote.id == group.leader_id) + .map(|remote| remote.endpoint.clone()) + } + }) + .filter(|endpoint| !endpoint.is_empty()); if required_group.is_some() && endpoint.is_none() { continue; } @@ -330,12 +332,17 @@ impl TopologyCache { .find(|r| r.id == leader_id) .map(|r| r.endpoint.clone()) }; + // Recovery may fence an unverified peer by publishing no route. + // Such a peer is unavailable, not a usable RPC destination. + let leader = leader.filter(|endpoint| !endpoint.is_empty()); let mut replicas: Vec = Vec::with_capacity(group.remotes.len() + 1); - if let Some(addr) = &local_endpoint { + if let Some(addr) = local_endpoint.as_ref().filter(|endpoint| !endpoint.is_empty()) { replicas.push(addr.clone()); } for r in &group.remotes { - replicas.push(r.endpoint.clone()); + if !r.endpoint.is_empty() { + replicas.push(r.endpoint.clone()); + } } fresh_routes.push(FreshRoute { key, diff --git a/lib/crowdb-kv-client/src/hardware/hierarchy/mutation.rs b/lib/crowdb-kv-client/src/hardware/hierarchy/mutation.rs index 8fbca5f0b..bad07b9a1 100644 --- a/lib/crowdb-kv-client/src/hardware/hierarchy/mutation.rs +++ b/lib/crowdb-kv-client/src/hardware/hierarchy/mutation.rs @@ -4,6 +4,7 @@ //! Shared revision fence for status and hierarchy membership changes. use crowdb_protocol::key::{RackKey, TextKey}; +use std::hash::{DefaultHasher, Hash, Hasher}; use crate::{BatchOp, CrowdbKvClient, Error, GetOutcome, ReadMode, Result}; @@ -40,7 +41,9 @@ where T: serde::de::DeserializeOwned + serde::Serialize + PartialEq, F: Fn(&mut T) -> Result<()>, { - for _ in 0..10 { + let mut last_error = Error::CasBusy; + let mut uncertain = false; + for attempt in 0..10 { let rack_path = RackKey { rack_id }.to_path(); let rack = read(kv, &rack_path) .await? @@ -69,9 +72,26 @@ where .await { Ok(_) => return Ok(()), - Err(Error::CasFailed { .. } | Error::CasBusy | Error::OutcomeUnknown) => {} + Err(error @ (Error::CasFailed { .. } | Error::CasBusy | Error::OutcomeUnknown)) => { + uncertain |= matches!(error, Error::OutcomeUnknown); + last_error = error; + if attempt < 9 { + // Give the in-flight rack mutation time to settle before + // re-reading its revision. Spread competing records across + // the next window instead of synchronizing their retries. + let mut hash = DefaultHasher::new(); + (path, attempt).hash(&mut hash); + let window = 1 << attempt.min(6); + let delay = window + hash.finish() % window; + tokio::time::sleep(std::time::Duration::from_millis(delay)).await; + } + } Err(error) => return Err(error), } } - Err(Error::OutcomeUnknown) + Err(if uncertain { + Error::OutcomeUnknown + } else { + last_error + }) } diff --git a/lib/crowdb-kv-client/src/lib.rs b/lib/crowdb-kv-client/src/lib.rs index b5ffc47c9..c3e14f990 100644 --- a/lib/crowdb-kv-client/src/lib.rs +++ b/lib/crowdb-kv-client/src/lib.rs @@ -51,8 +51,9 @@ pub use hardware::{ pub use membership::{GroupMembershipClient, GroupMembershipSnapshot}; pub use metrics::{ClientMetrics, ClientMetricsSnapshot, LeaderChangeEpisode, WindowLatencySnapshot}; pub use service::{ - DomainMonitorClient, ServiceDiscoveryClient, ServiceRegistryClient, WatchNotify, WatchNotifyClient, - WatchSubscription, + DomainMonitorClient, ServiceDiscoveryClient, ServiceRegistryClient, StreamProcessAuthority, + StreamProcessContext, StreamProcessLeaseClient, StreamProcessLeaseError, StreamProcessLeaseStatus, + StreamProcessLeaseTiming, WatchNotify, WatchNotifyClient, WatchSubscription, }; pub use transport::{KVClusterAdmin, KVClusterMetaClient, KvRpcTransport}; diff --git a/lib/crowdb-kv-client/src/service.rs b/lib/crowdb-kv-client/src/service.rs index 45bdee46b..7bcfd39e8 100644 --- a/lib/crowdb-kv-client/src/service.rs +++ b/lib/crowdb-kv-client/src/service.rs @@ -6,9 +6,14 @@ pub mod discovery; pub mod domain_monitor; pub mod registry; +mod stream_lease; pub mod watch_notify; pub use discovery::ServiceDiscoveryClient; pub use domain_monitor::DomainMonitorClient; pub use registry::ServiceRegistryClient; +pub use stream_lease::{ + StreamProcessAuthority, StreamProcessContext, StreamProcessLeaseClient, StreamProcessLeaseError, + StreamProcessLeaseStatus, StreamProcessLeaseTiming, +}; pub use watch_notify::{WatchNotify, WatchNotifyClient, WatchSubscription}; diff --git a/lib/crowdb-kv-client/src/service/registry.rs b/lib/crowdb-kv-client/src/service/registry.rs index b2ff2d083..c119f4671 100644 --- a/lib/crowdb-kv-client/src/service/registry.rs +++ b/lib/crowdb-kv-client/src/service/registry.rs @@ -466,32 +466,6 @@ impl ServiceRegistryClient { // ── chunk-kv convenience wrappers ────────────────────────────── impl ServiceRegistryClient { - /// Register a chunk-KV owner and its current partition/load observation. - pub async fn register_chunk_kv( - &self, - instance_id: InstanceId, - rpc_endpoint: &str, - observation: &crowdb_protocol::common::ChunkKvExtra, - ) -> Result<()> { - let extra = ServiceExtra { - diskdb: None, - kv_server: None, - chunk_kv: Some(observation.clone()), - }; - self.register("chunk-kv", instance_id, rpc_endpoint, &extra).await - } - - /// Heartbeat a chunk-KV owner with a complete replacement observation. - pub async fn heartbeat_chunk_kv( - &self, - instance_id: InstanceId, - rpc_endpoint: &str, - observation: &crowdb_protocol::common::ChunkKvExtra, - ) -> Result<()> { - self.register_chunk_kv(instance_id, rpc_endpoint, observation) - .await - } - /// Read raw chunk-KV owner observations, including expired registrations. pub async fn read_all_chunk_kv_observations( &self, @@ -510,6 +484,7 @@ impl ServiceRegistryClient { })?; Ok(crowdb_protocol::chunk_kv::ChunkKvInstanceObservation { instance_id, + process_epoch: observation.process_epoch, rpc_endpoint: value.rpc_endpoint, last_heartbeat_ms: value.last_heartbeat_ms, capacity_bytes: observation.capacity_bytes, diff --git a/lib/crowdb-kv-client/src/service/stream_lease.rs b/lib/crowdb-kv-client/src/service/stream_lease.rs new file mode 100644 index 000000000..9e2debea5 --- /dev/null +++ b/lib/crowdb-kv-client/src/service/stream_lease.rs @@ -0,0 +1,228 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Conditional Group-0 process leases for Stream and WAL owners. + +use std::sync::Arc; + +use crowdb_protocol::chunk_stream::{StreamProcessIdentity, StreamProcessLease, StreamPurpose}; +use crowdb_protocol::key::{StreamProcessLeaseKey, TextKey}; + +use crate::{CrowdbKvClient, Error, GetOutcome, ReadMode}; + +mod authority; +mod observation; +pub use authority::{ + StreamProcessAuthority, StreamProcessContext, StreamProcessLeaseStatus, StreamProcessLeaseTiming, +}; + +#[derive(Debug, thiserror::Error)] +pub enum StreamProcessLeaseError { + #[error("invalid Stream process lease: {0}")] + Invalid(String), + #[error("Stream owner process has been replaced")] + Replaced, + #[error("Stream owner process lease has expired")] + Expired, + #[error("Stream process lease outcome remains uncertain")] + OutcomeUnknown, + #[error(transparent)] + Kv(#[from] Error), +} + +type Result = std::result::Result; + +pub struct StreamProcessLeaseClient { + kv: Arc, +} + +impl StreamProcessLeaseClient { + #[must_use] + pub fn new(kv: Arc) -> Self { + Self { kv } + } + + /// Capture a predecessor once before attempting a new process claim. + /// # Errors + /// Rejects malformed identity, unsupported records and unavailable Group 0. + pub async fn load(&self, purpose: StreamPurpose, instance_id: u64) -> Result> { + Ok(self.read(purpose, instance_id).await?.map(|(record, _)| record)) + } + + /// Claim only from one fixed predecessor; an uncertain result reuses the nonce. + /// Adoption still requires the returned predecessor boundary, late-request + /// exclusion and ordered IO completion; the claim alone proves none of those. + /// # Errors + /// Rejects changed predecessors, expired own retries and invalid timing. + pub async fn claim( + &self, + purpose: StreamPurpose, + instance_id: u64, + boot_nonce: [u8; 16], + predecessor: Option, + now_ms: u64, + lease_ms: u64, + ) -> Result { + validate_claim(purpose, instance_id, boot_nonce, predecessor)?; + let expires_at_ms = expiry(now_ms, lease_ms)?; + let epoch = predecessor + .map_or(Some(1), |previous| previous.process_epoch.checked_add(1)) + .ok_or_else(|| invalid("process epoch overflow"))?; + for attempt in 0..8 { + let current = self.read(purpose, instance_id).await?; + if let Some((record, _)) = ¤t { + if record.identity.boot_nonce == boot_nonce && record.identity.process_epoch == epoch { + if now_ms >= record.expires_at_ms { + return Err(StreamProcessLeaseError::Expired); + } + return Ok(record.clone()); + } + } + if current.as_ref().map(|(record, _)| record.identity) != predecessor { + return Err(StreamProcessLeaseError::Replaced); + } + let mut record = StreamProcessLease { + identity: StreamProcessIdentity { + purpose, + instance_id, + process_epoch: epoch, + boot_nonce, + }, + expires_at_ms, + prior_exclusion_deadline_ms: 0, + grant_sequence: 0, + grant_expires_at_ms: 0, + }; + if let Some((previous, _)) = ¤t { + record.prior_exclusion_deadline_ms = previous + .prior_exclusion_deadline_ms + .max(previous.expires_at_ms) + .max(previous.grant_expires_at_ms); + record.grant_sequence = previous.grant_sequence; + record.grant_expires_at_ms = previous.grant_expires_at_ms; + } + let revision = current.as_ref().map_or(0, |(_, revision)| *revision); + if self.write(&record, revision, attempt).await? { + return Ok(record); + } + } + Err(StreamProcessLeaseError::OutcomeUnknown) + } + + /// Renew the same process without reviving a recorded expiry or replacement. + /// # Errors + /// Returns a fence, record, availability or unresolved-write error. + pub async fn renew( + &self, + identity: StreamProcessIdentity, + now_ms: u64, + lease_ms: u64, + ) -> Result { + let expires_at_ms = expiry(now_ms, lease_ms)?; + for attempt in 0..8 { + let Some((mut record, revision)) = self.read(identity.purpose, identity.instance_id).await? + else { + return Err(StreamProcessLeaseError::Replaced); + }; + if record.identity != identity { + return Err(StreamProcessLeaseError::Replaced); + } + if now_ms >= record.expires_at_ms { + return Err(StreamProcessLeaseError::Expired); + } + record.expires_at_ms = record.expires_at_ms.max(expires_at_ms); + if self.write(&record, revision, attempt).await? { + return Ok(record); + } + } + Err(StreamProcessLeaseError::OutcomeUnknown) + } + + async fn read( + &self, + purpose: StreamPurpose, + instance_id: u64, + ) -> Result> { + if instance_id == 0 { + return Err(invalid("zero owner instance")); + } + let key = StreamProcessLeaseKey { purpose, instance_id }.to_path(); + match self + .kv + .get(0, 0, key.as_bytes(), ReadMode::Linearizable, None) + .await? + { + GetOutcome::NotFound => Ok(None), + GetOutcome::Found { value, revision } => { + let record: StreamProcessLease = + serde_json::from_slice(&value).map_err(|e| invalid(&e.to_string()))?; + record.validate().map_err(|e| invalid(&e.to_string()))?; + if record.identity.purpose != purpose || record.identity.instance_id != instance_id { + return Err(invalid("record differs from its owner key")); + } + Ok(Some((record, revision))) + } + } + } + + async fn write(&self, record: &StreamProcessLease, revision: u64, attempt: u32) -> Result { + record.validate().map_err(|e| invalid(&e.to_string()))?; + let identity = record.identity; + let key = StreamProcessLeaseKey { + purpose: identity.purpose, + instance_id: identity.instance_id, + } + .to_path(); + let value = serde_json::to_vec(record).map_err(|e| invalid(&e.to_string()))?; + match self.kv.put_cas(0, 0, key.as_bytes(), &value, revision).await { + Ok(_) => Ok(true), + Err(Error::CasBusy) => { + wait_for_cas(attempt).await; + Ok(false) + } + Err(Error::CasFailed { .. } | Error::OutcomeUnknown) => Ok(false), + Err(error) => Err(error.into()), + } + } +} + +// Renewal and discovery can contend on the same lease. Let the in-flight +// conditional write complete instead of burning the bounded retry budget. +async fn wait_for_cas(attempt: u32) { + if attempt < 7 { + tokio::time::sleep(std::time::Duration::from_millis(1 << attempt.min(4))).await; + } +} + +fn invalid(message: &str) -> StreamProcessLeaseError { + StreamProcessLeaseError::Invalid(message.into()) +} + +fn expiry(now: u64, lease: u64) -> Result { + if lease == 0 { + return Err(invalid("zero lease duration")); + } + now.checked_add(lease) + .ok_or_else(|| invalid("lease deadline overflow")) +} + +fn validate_claim( + purpose: StreamPurpose, + instance_id: u64, + nonce: [u8; 16], + previous: Option, +) -> Result<()> { + if instance_id == 0 + || nonce == [0; 16] + || previous.is_some_and(|previous| { + previous.purpose != purpose + || previous.instance_id != instance_id + || previous.process_epoch == 0 + || previous.boot_nonce == [0; 16] + || previous.boot_nonce == nonce + }) + { + return Err(invalid("invalid process claim identity or predecessor")); + } + Ok(()) +} diff --git a/lib/crowdb-kv-client/src/service/stream_lease/authority.rs b/lib/crowdb-kv-client/src/service/stream_lease/authority.rs new file mode 100644 index 000000000..876e6e31b --- /dev/null +++ b/lib/crowdb-kv-client/src/service/stream_lease/authority.rs @@ -0,0 +1,232 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Lock-free local projection of one fixed Group-0 process lease. + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use arc_swap::ArcSwap; +use crowdb_protocol::{ + chunk_stream::{StreamProcessIdentity, StreamProcessLease}, + timing::{DEFAULT_SELF_FENCE_MARGIN_MS, MAX_WRITE_CLOCK_SKEW_MS}, +}; +use serde::Serialize; + +use super::{invalid, Result, StreamProcessLeaseError}; + +#[derive(Clone, Copy, Debug)] +pub struct StreamProcessLeaseTiming { + pub max_clock_skew_ms: u64, + pub self_fence_margin_ms: u64, +} + +impl Default for StreamProcessLeaseTiming { + fn default() -> Self { + Self { + max_clock_skew_ms: MAX_WRITE_CLOCK_SKEW_MS, + self_fence_margin_ms: DEFAULT_SELF_FENCE_MARGIN_MS, + } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct StreamProcessContext { + identity: StreamProcessIdentity, + deadline: Instant, +} + +impl StreamProcessContext { + #[must_use] + pub const fn identity(&self) -> StreamProcessIdentity { + self.identity + } + #[must_use] + pub const fn deadline(&self) -> Instant { + self.deadline + } +} + +struct LeaseState { + lease: Arc, + deadline: Instant, + fenced: bool, +} + +#[derive(Debug, Serialize)] +pub struct StreamProcessLeaseStatus { + pub lease: StreamProcessLease, + pub remaining_ms: u64, + pub fenced: bool, +} + +pub struct StreamProcessAuthority { + identity: StreamProcessIdentity, + timing: StreamProcessLeaseTiming, + state: ArcSwap, +} + +impl StreamProcessAuthority { + /// Project the authoritative expiry using wall/monotonic clocks sampled together. + /// # Errors + /// Rejects invalid records, unsafe timing, expired receipts and clock overflow. + pub fn new( + lease: StreamProcessLease, + timing: StreamProcessLeaseTiming, + wall_ms: u64, + received: Instant, + ) -> Result { + let deadline = project(&lease, timing, wall_ms, received)?; + if Instant::now() >= deadline { + return Err(StreamProcessLeaseError::Expired); + } + Ok(Self { + identity: lease.identity, + timing, + state: ArcSwap::from_pointee(LeaseState { + lease: Arc::new(lease), + deadline, + fenced: false, + }), + }) + } + + /// Observe renewal without extending an admitted IO context or reviving a gap. + /// # Errors + /// A different process fences immediately; a late renewal cannot clear a fence. + pub fn observe(&self, lease: StreamProcessLease, wall_ms: u64, received: Instant) -> Result<()> { + lease.validate().map_err(|error| invalid(&error.to_string()))?; + if lease.identity != self.identity { + self.fence(); + return Err(StreamProcessLeaseError::Replaced); + } + let deadline = match project(&lease, self.timing, wall_ms, received) { + Ok(deadline) => deadline, + Err(error) => { + let superseded = lease.expires_at_ms <= self.state.load().lease.expires_at_ms; + return if matches!(&error, StreamProcessLeaseError::Expired) && superseded { + self.live_deadline(received).map(|_| ()) + } else { + Err(error) + }; + } + }; + let lease = Arc::new(lease); + self.state.rcu(|current| { + let now = Instant::now().max(received); + if current.fenced || now >= current.deadline { + return fenced(current); + } + if lease.expires_at_ms <= current.lease.expires_at_ms { + return Arc::clone(current); + } + Arc::new(LeaseState { + lease: Arc::clone(&lease), + deadline, + fenced: now >= deadline, + }) + }); + self.live_deadline(received).map(|_| ()) + } + + /// Capture a fixed absolute deadline before IO admission. + /// # Errors + /// Returns a lease fence instead of creating a new IO budget after expiry. + pub fn capture(&self, now: Instant) -> Result { + Ok(StreamProcessContext { + identity: self.identity, + deadline: self.live_deadline(now)?, + }) + } + + /// Validate current ownership before acknowledgment; continuous renewal is allowed. + /// # Errors + /// Rejects a replaced context or any local expiry/revocation. + pub fn validate_context(&self, context: &StreamProcessContext, now: Instant) -> Result<()> { + if context.identity != self.identity { + return Err(StreamProcessLeaseError::Replaced); + } + self.live_deadline(now).map(|_| ()) + } + + /// Permanently revoke this local boot after confirmed replacement or shutdown. + pub fn fence(&self) { + self.state.rcu(fenced); + } + + #[must_use] + pub fn status(&self, now: Instant) -> StreamProcessLeaseStatus { + let state = self.state.load(); + let now = now.max(Instant::now()); + let is_fenced = state.fenced || now >= state.deadline; + StreamProcessLeaseStatus { + lease: state.lease.as_ref().clone(), + remaining_ms: if is_fenced { + 0 + } else { + u64::try_from((state.deadline - now).as_millis()).unwrap_or(u64::MAX) + }, + fenced: is_fenced, + } + } + + fn live_deadline(&self, now: Instant) -> Result { + let now = now.max(Instant::now()); + let state = self.state.load(); + if !state.fenced && now < state.deadline { + return Ok(state.deadline); + } + drop(state); + self.state.rcu(|current| { + if now >= current.deadline || current.fenced { + fenced(current) + } else { + Arc::clone(current) + } + }); + let current = self.state.load(); + if current.fenced || now >= current.deadline { + Err(StreamProcessLeaseError::Expired) + } else { + Ok(current.deadline) + } + } +} + +fn fenced(current: &Arc) -> Arc { + if current.fenced { + return Arc::clone(current); + } + Arc::new(LeaseState { + lease: Arc::clone(¤t.lease), + deadline: current.deadline, + fenced: true, + }) +} + +fn project( + lease: &StreamProcessLease, + timing: StreamProcessLeaseTiming, + wall_ms: u64, + received: Instant, +) -> Result { + lease.validate().map_err(|error| invalid(&error.to_string()))?; + if timing.max_clock_skew_ms > MAX_WRITE_CLOCK_SKEW_MS { + return Err(invalid("owner clock skew exceeds shared recovery bound")); + } + let margin = timing + .max_clock_skew_ms + .checked_add(timing.self_fence_margin_ms) + .ok_or_else(|| invalid("owner safety margin overflow"))?; + let remaining = lease + .expires_at_ms + .checked_sub(wall_ms) + .and_then(|remaining| remaining.checked_sub(margin)) + .filter(|remaining| *remaining > 0) + .ok_or(StreamProcessLeaseError::Expired)?; + received + .checked_add(Duration::from_millis(remaining)) + .ok_or_else(|| invalid("owner monotonic deadline overflow")) +} diff --git a/lib/crowdb-kv-client/src/service/stream_lease/observation.rs b/lib/crowdb-kv-client/src/service/stream_lease/observation.rs new file mode 100644 index 000000000..1e7e5cfed --- /dev/null +++ b/lib/crowdb-kv-client/src/service/stream_lease/observation.rs @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Observation publication fenced by the same process lease as serving grants. + +use super::{invalid, Result, StreamProcessLeaseClient, StreamProcessLeaseError}; +use crate::{BatchOp, Error}; +use crowdb_protocol::{ + chunk_stream::{StreamProcessIdentity, StreamPurpose}, + common::{ChunkKvExtra, InstanceValue, ServiceExtra}, + key::{InstanceKey, StreamProcessLeaseKey, TextKey}, +}; + +impl StreamProcessLeaseClient { + /// Publish one WAL owner's observation only while its exact boot is current. + /// # Errors + /// Rejects expired/replaced boots, malformed observations and unavailable CAS. + pub async fn publish_chunk_kv_observation( + &self, + identity: StreamProcessIdentity, + endpoint: &str, + observation: &ChunkKvExtra, + now_ms: u64, + ) -> Result<()> { + if identity.purpose != StreamPurpose::Wal || observation.process_epoch != identity.process_epoch { + return Err(invalid("observation differs from its WAL process")); + } + let value = InstanceValue { + instance_id: identity.instance_id, + rpc_endpoint: endpoint.into(), + last_heartbeat_ms: now_ms, + extra: Some(ServiceExtra { + diskdb: None, + kv_server: None, + chunk_kv: Some(observation.clone()), + }), + }; + let observation_op = BatchOp::Put { + key: InstanceKey { + service: "chunk-kv".into(), + instance_id: identity.instance_id, + } + .to_path() + .into_bytes() + .into(), + value: serde_json::to_vec(&value) + .map_err(|error| invalid(&error.to_string()))? + .into(), + }; + self.mutate_observation(identity, observation_op, now_ms).await + } + + /// Withdraw discovery without deleting the persistent process/expiry fence. + /// # Errors + /// A replaced boot cannot withdraw its successor's observation. + pub async fn withdraw_chunk_kv_observation( + &self, + identity: StreamProcessIdentity, + now_ms: u64, + ) -> Result<()> { + if identity.purpose != StreamPurpose::Wal { + return Err(invalid("not a WAL process")); + } + let op = BatchOp::Delete { + key: InstanceKey { + service: "chunk-kv".into(), + instance_id: identity.instance_id, + } + .to_path() + .into_bytes() + .into(), + }; + self.mutate_observation(identity, op, now_ms).await + } + + async fn mutate_observation( + &self, + identity: StreamProcessIdentity, + observation_op: BatchOp, + now_ms: u64, + ) -> Result<()> { + let lease_path = StreamProcessLeaseKey { + purpose: identity.purpose, + instance_id: identity.instance_id, + } + .to_path(); + for attempt in 0..8 { + let Some((lease, revision)) = self.read(identity.purpose, identity.instance_id).await? else { + return Err(StreamProcessLeaseError::Replaced); + }; + if lease.identity != identity { + return Err(StreamProcessLeaseError::Replaced); + } + if now_ms >= lease.expires_at_ms { + return Err(StreamProcessLeaseError::Expired); + } + let ops = [ + observation_op.clone(), + BatchOp::Put { + key: lease_path.clone().into_bytes().into(), + value: serde_json::to_vec(&lease) + .map_err(|error| invalid(&error.to_string()))? + .into(), + }, + ]; + match self + .kv + .batch_write_cas(0, 0, &ops, lease_path.as_bytes(), revision) + .await + { + Ok(_) => return Ok(()), + Err(Error::CasBusy) => super::wait_for_cas(attempt).await, + Err(Error::CasFailed { .. } | Error::OutcomeUnknown) => {} + Err(error) => return Err(error.into()), + } + } + Err(StreamProcessLeaseError::OutcomeUnknown) + } +} diff --git a/lib/crowdb-kv-client/tests/common/stream_lease.rs b/lib/crowdb-kv-client/tests/common/stream_lease.rs new file mode 100644 index 000000000..db95b5286 --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/stream_lease.rs @@ -0,0 +1,34 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use std::sync::Arc; + +use crowdb_kv::cluster::{ + group::PxGroup, kv_server::KvServer, PxKvStore, PxLocalReplica, PxLocalReplicaRole, +}; +use crowdb_kv_client::{ClientConfig, CrowdbKvClient}; + +pub struct TestStreamLeaseKv { + pub store: Arc, + pub kv: Arc, +} + +impl TestStreamLeaseKv { + pub async fn start() -> Self { + let store = Arc::new(PxKvStore::new(0, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new( + 0, + PxLocalReplica::new(1, PxLocalReplicaRole::Leader), + )); + store.start().await.unwrap(); + let kv = Arc::new(CrowdbKvClient::new(ClientConfig::new(vec![]))); + kv.seed_leader(0, 0, store.listen_addr().unwrap().to_string()); + Self { store, kv } + } +} + +impl Drop for TestStreamLeaseKv { + fn drop(&mut self) { + self.store.stop(); + } +} diff --git a/lib/crowdb-kv-client/tests/common/stream_lease_clock.rs b/lib/crowdb-kv-client/tests/common/stream_lease_clock.rs new file mode 100644 index 000000000..abf8edb72 --- /dev/null +++ b/lib/crowdb-kv-client/tests/common/stream_lease_clock.rs @@ -0,0 +1,23 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::chunk_stream::{StreamProcessIdentity, StreamProcessLease, StreamPurpose}; + +pub struct TestStreamLeaseRecord; + +impl TestStreamLeaseRecord { + pub fn at(expires_at_ms: u64) -> StreamProcessLease { + StreamProcessLease { + identity: StreamProcessIdentity { + purpose: StreamPurpose::Wal, + instance_id: 7, + process_epoch: 1, + boot_nonce: [1; 16], + }, + expires_at_ms, + prior_exclusion_deadline_ms: 0, + grant_sequence: 0, + grant_expires_at_ms: 0, + } + } +} diff --git a/lib/crowdb-kv-client/tests/disk_status_test.rs b/lib/crowdb-kv-client/tests/disk_status_test.rs index a959d576b..94a0a79d5 100644 --- a/lib/crowdb-kv-client/tests/disk_status_test.rs +++ b/lib/crowdb-kv-client/tests/disk_status_test.rs @@ -9,7 +9,7 @@ use crowdb_kv::cluster::{ use crowdb_kv_client::{BatchOp, ClientConfig, CrowdbKvClient, GetOutcome, HardwareClient, ReadMode}; use crowdb_protocol::{ common::{DiskId, HwStatus, RackValue}, - diskdb::rpc::DiskValue, + diskdb::rpc::{DiskGroupValue, DiskValue}, key::{DiskKey, RackKey, TextKey}, }; @@ -51,6 +51,51 @@ async fn concurrent_status_changes_reobserve_the_shared_rack_fence() { ); } +#[tokio::test] +async fn concurrent_disk_group_changes_do_not_exhaust_retries_on_inflight_rack_updates() { + let fixture = TestHardware::start().await; + let hardware = Arc::new(fixture.hardware.clone()); + for group in 1..=32 { + hardware + .add_disk_group(1, 2, group, &DiskGroupValue::default()) + .await + .unwrap(); + } + for status in [ + HwStatus::Maintenance, + HwStatus::Suspect, + HwStatus::Missing, + HwStatus::Bad, + ] { + let ready = Arc::new(tokio::sync::Barrier::new(32)); + let mut tasks = tokio::task::JoinSet::new(); + for group in 1..=32 { + let hardware = hardware.clone(); + let ready = ready.clone(); + tasks.spawn(async move { + ready.wait().await; + (group, hardware.set_disk_group_status(1, 2, group, status).await) + }); + } + while let Some(result) = tasks.join_next().await { + let (group, outcome) = result.unwrap(); + outcome.unwrap_or_else(|error| panic!("group {group} status {status:?}: {error}")); + } + for group in 1..=32 { + assert_eq!( + hardware + .get_disk_group(1, 2, group) + .await + .unwrap() + .unwrap() + .value + .status, + status as i32 + ); + } + } +} + #[tokio::test] async fn status_change_invalidates_an_already_prepared_hierarchy_removal() { let fixture = TestHardware::start().await; diff --git a/lib/crowdb-kv-client/tests/multi_seed_discovery_test.rs b/lib/crowdb-kv-client/tests/multi_seed_discovery_test.rs index 604e9ac94..9e8c28909 100644 --- a/lib/crowdb-kv-client/tests/multi_seed_discovery_test.rs +++ b/lib/crowdb-kv-client/tests/multi_seed_discovery_test.rs @@ -36,6 +36,7 @@ async fn group_missing_on_first_seed_is_discovered_on_another_member() { let (first, first_task) = seed(stores[0].clone()).await; let (second, second_task) = seed(stores[1].clone()).await; let client = CrowdbKvClient::new(ClientConfig::new(vec![first, second])); + client.put(11, 1, b"first", b"member-one", None).await.unwrap(); let written = client.put(11, 2, b"overlap", b"member-two", None).await; first_task.abort(); second_task.abort(); @@ -50,3 +51,47 @@ async fn group_missing_on_first_seed_is_discovered_on_another_member() { "a healthy unrelated first seed must not hide the target group: {written:?}" ); } + +#[tokio::test] +async fn an_unverified_empty_leader_route_does_not_hide_another_members_local_listener() { + let store = Arc::new(PxKvStore::new(11, "127.0.0.1:0".parse().unwrap())); + store.add_group(PxGroup::new( + 2, + PxLocalReplica::new(2, PxLocalReplicaRole::Leader), + )); + store.start().await.unwrap(); + let mut blocked = store.status(); + blocked.groups[0].local_replica.id = 1; + blocked.groups[0].local_replica_id = 1; + blocked.groups[0].remotes = vec![crowdb_protocol::mgmt::RemoteStatus { + id: 2, + endpoint: String::new(), + voting: true, + ..Default::default() + }]; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let first = format!("http://{}", listener.local_addr().unwrap()); + let app = Router::new().route( + "/topology", + get(move || { + let blocked = blocked.clone(); + async move { Json(serde_json::json!({"stores": [blocked]})) } + }), + ); + let first_task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let (second, second_task) = seed(store.clone()).await; + let client = CrowdbKvClient::new(ClientConfig::new(vec![first, second])); + let written = tokio::time::timeout( + std::time::Duration::from_secs(2), + client.put(11, 2, b"verified", b"leader", None), + ) + .await; + first_task.abort(); + second_task.abort(); + store.stop(); + store.join().await; + assert!( + matches!(written, Ok(Ok(_))), + "an empty recovery route is unavailable, not a leader endpoint: {written:?}" + ); +} diff --git a/lib/crowdb-kv-client/tests/stream_process_authority_test.rs b/lib/crowdb-kv-client/tests/stream_process_authority_test.rs new file mode 100644 index 000000000..0b8a52b83 --- /dev/null +++ b/lib/crowdb-kv-client/tests/stream_process_authority_test.rs @@ -0,0 +1,184 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/stream_lease_clock.rs"] +mod fixture; + +use crowdb_kv_client::{StreamProcessAuthority, StreamProcessLeaseError, StreamProcessLeaseTiming}; +use fixture::TestStreamLeaseRecord as Test; +use std::time::{Duration, Instant}; + +#[test] +fn continuous_renewal_preserves_ownership_without_extending_admitted_io() { + let start = Instant::now(); + let owner = StreamProcessAuthority::new( + Test::at(11_000), + StreamProcessLeaseTiming::default(), + 1_000, + start, + ) + .unwrap(); + let admitted = owner.capture(start).unwrap(); + assert_eq!(admitted.deadline(), start + Duration::from_secs(8)); + owner + .observe(Test::at(12_000), 2_000, start + Duration::from_secs(1)) + .unwrap(); + let later = start + Duration::from_millis(8_500); + owner.validate_context(&admitted, later).unwrap(); + assert_eq!(admitted.deadline(), start + Duration::from_secs(8)); + assert_eq!( + owner.capture(later).unwrap().deadline(), + start + Duration::from_secs(9) + ); + assert_eq!(owner.status(later).remaining_ms, 500); +} + +#[test] +fn a_gap_cannot_be_cleared_by_a_later_successful_renewal() { + let start = Instant::now(); + let owner = StreamProcessAuthority::new( + Test::at(11_000), + StreamProcessLeaseTiming::default(), + 1_000, + start, + ) + .unwrap(); + let admitted = owner.capture(start).unwrap(); + assert!(matches!( + owner.observe(Test::at(21_000), 11_000, start + Duration::from_secs(10)), + Err(StreamProcessLeaseError::Expired) + )); + assert!(matches!( + owner.validate_context(&admitted, start + Duration::from_secs(11)), + Err(StreamProcessLeaseError::Expired) + )); + assert!(matches!( + owner.observe(Test::at(31_000), 21_000, start + Duration::from_secs(20)), + Err(StreamProcessLeaseError::Expired) + )); + assert!(owner.status(start).fenced); + assert_eq!(owner.status(start).remaining_ms, 0); +} + +#[test] +fn an_out_of_order_expired_observation_does_not_revoke_a_continuously_renewed_owner() { + let start = Instant::now(); + let first = Test::at(11_000); + let owner = StreamProcessAuthority::new(first.clone(), StreamProcessLeaseTiming::default(), 1_000, start) + .unwrap(); + let admitted = owner.capture(start).unwrap(); + owner + .observe(Test::at(12_000), 2_000, start + Duration::from_secs(1)) + .unwrap(); + owner + .observe(Test::at(18_000), 8_000, start + Duration::from_secs(7)) + .unwrap(); + owner + .observe(first, 12_000, start + Duration::from_secs(11)) + .unwrap(); + owner + .validate_context(&admitted, start + Duration::from_secs(11)) + .unwrap(); + assert_eq!( + owner.capture(start + Duration::from_secs(11)).unwrap().deadline(), + start + Duration::from_secs(15) + ); +} + +#[test] +fn a_different_boot_fences_old_contexts_and_cannot_install_in_place() { + let start = Instant::now(); + let first = Test::at(11_000); + let owner = StreamProcessAuthority::new(first.clone(), StreamProcessLeaseTiming::default(), 1_000, start) + .unwrap(); + let admitted = owner.capture(start).unwrap(); + let mut next = Test::at(22_000); + next.identity.process_epoch = 2; + next.identity.boot_nonce = [2; 16]; + next.prior_exclusion_deadline_ms = first.expires_at_ms; + assert!(matches!( + owner.observe(next.clone(), 2_000, start + Duration::from_secs(1)), + Err(StreamProcessLeaseError::Replaced) + )); + assert!(owner.status(start).fenced); + let replacement = StreamProcessAuthority::new( + next, + StreamProcessLeaseTiming::default(), + 2_000, + start + Duration::from_secs(1), + ) + .unwrap(); + assert!(matches!( + replacement.validate_context(&admitted, start), + Err(StreamProcessLeaseError::Replaced) + )); +} + +#[test] +fn unsafe_clock_bounds_and_margin_overflow_are_rejected() { + let timing = StreamProcessLeaseTiming { + max_clock_skew_ms: 1_001, + self_fence_margin_ms: 1_000, + }; + assert!(matches!( + StreamProcessAuthority::new(Test::at(11_000), timing, 1_000, Instant::now()), + Err(StreamProcessLeaseError::Invalid(_)) + )); + let timing = StreamProcessLeaseTiming { + max_clock_skew_ms: 1, + self_fence_margin_ms: u64::MAX, + }; + assert!(matches!( + StreamProcessAuthority::new(Test::at(11_000), timing, 1_000, Instant::now()), + Err(StreamProcessLeaseError::Invalid(_)) + )); +} + +#[test] +fn observing_expiry_cannot_be_undone_by_a_backdated_renewal() { + let start = Instant::now(); + let owner = StreamProcessAuthority::new( + Test::at(11_000), + StreamProcessLeaseTiming::default(), + 1_000, + start, + ) + .unwrap(); + let admitted = owner.capture(start).unwrap(); + assert!(matches!( + owner.capture(start + Duration::from_secs(9)), + Err(StreamProcessLeaseError::Expired) + )); + assert!(matches!( + owner.observe(Test::at(12_000), 2_000, start + Duration::from_secs(1)), + Err(StreamProcessLeaseError::Expired) + )); + assert!(matches!( + owner.validate_context(&admitted, start), + Err(StreamProcessLeaseError::Expired) + )); +} + +#[test] +fn malformed_observations_do_not_revoke_an_existing_live_lease() { + let start = Instant::now(); + let owner = StreamProcessAuthority::new( + Test::at(11_000), + StreamProcessLeaseTiming::default(), + 1_000, + start, + ) + .unwrap(); + let admitted = owner.capture(start).unwrap(); + let mut malformed = Test::at(12_000); + malformed.grant_sequence = 1; + assert!(matches!( + owner.observe(malformed, 2_000, start + Duration::from_secs(1)), + Err(StreamProcessLeaseError::Invalid(_)) + )); + owner + .validate_context(&admitted, start + Duration::from_secs(1)) + .unwrap(); + assert!(!owner.status(start).fenced); + assert_eq!(owner.capture(start).unwrap().deadline(), admitted.deadline()); +} diff --git a/lib/crowdb-kv-client/tests/stream_process_lease_test.rs b/lib/crowdb-kv-client/tests/stream_process_lease_test.rs new file mode 100644 index 000000000..927583235 --- /dev/null +++ b/lib/crowdb-kv-client/tests/stream_process_lease_test.rs @@ -0,0 +1,194 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/stream_lease.rs"] +mod fixture; + +use crowdb_kv_client::{ + StreamProcessAuthority, StreamProcessLeaseClient, StreamProcessLeaseError, StreamProcessLeaseTiming, +}; +use crowdb_protocol::{ + chunk_stream::StreamPurpose, + key::{StreamProcessLeaseKey, TextKey}, +}; +use fixture::TestStreamLeaseKv; + +#[tokio::test] +async fn a_fixed_boot_is_idempotent_and_replaced_processes_cannot_renew_or_reclaim() { + let fixture = TestStreamLeaseKv::start().await; + let client = StreamProcessLeaseClient::new(fixture.kv.clone()); + let first = client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_000, 100) + .await + .unwrap(); + assert_eq!( + client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_010, 100) + .await + .unwrap(), + first + ); + let renewed = client.renew(first.identity, 1_050, 100).await.unwrap(); + let next = client + .claim(StreamPurpose::Wal, 7, [2; 16], Some(first.identity), 1_060, 100) + .await + .unwrap(); + assert_eq!(next.identity.process_epoch, 2); + assert_eq!(next.prior_exclusion_deadline_ms, renewed.expires_at_ms); + assert!(matches!( + client.renew(first.identity, 1_061, 100).await, + Err(StreamProcessLeaseError::Replaced) + )); + assert!(matches!( + client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_061, 100) + .await, + Err(StreamProcessLeaseError::Replaced) + )); + assert_eq!( + client + .claim(StreamPurpose::Wal, 7, [2; 16], Some(first.identity), 1_061, 100) + .await + .unwrap(), + next + ); + assert!( + client + .renew(next.identity, 1_070, 100) + .await + .unwrap() + .expires_at_ms + > next.expires_at_ms + ); +} + +#[tokio::test] +async fn simultaneous_claimants_cannot_steal_a_changed_predecessor() { + let fixture = TestStreamLeaseKv::start().await; + let client = StreamProcessLeaseClient::new(fixture.kv.clone()); + let (left, right) = tokio::join!( + client.claim(StreamPurpose::Stream, 9, [1; 16], None, 1_000, 100), + client.claim(StreamPurpose::Stream, 9, [2; 16], None, 1_000, 100), + ); + let (winner, loser_nonce) = match (left, right) { + (Ok(winner), Err(StreamProcessLeaseError::Replaced)) => (winner, [2; 16]), + (Err(StreamProcessLeaseError::Replaced), Ok(winner)) => (winner, [1; 16]), + outcome => panic!("two claims did not choose exactly one boot: {outcome:?}"), + }; + assert!(matches!( + client + .claim(StreamPurpose::Stream, 9, loser_nonce, None, 1_010, 100) + .await, + Err(StreamProcessLeaseError::Replaced) + )); + assert_eq!(client.load(StreamPurpose::Stream, 9).await.unwrap(), Some(winner)); + assert!(client.load(StreamPurpose::Wal, 9).await.unwrap().is_none()); +} + +#[tokio::test] +async fn expiry_requires_a_new_boot_and_preserves_the_old_exclusion_boundary() { + let fixture = TestStreamLeaseKv::start().await; + let client = StreamProcessLeaseClient::new(fixture.kv.clone()); + let first = client + .claim(StreamPurpose::Stream, 7, [1; 16], None, 1_000, 100) + .await + .unwrap(); + assert!(matches!( + client.renew(first.identity, 1_100, 100).await, + Err(StreamProcessLeaseError::Expired) + )); + assert!(matches!( + client + .claim(StreamPurpose::Stream, 7, [1; 16], None, 1_100, 100) + .await, + Err(StreamProcessLeaseError::Expired) + )); + let next = client + .claim( + StreamPurpose::Stream, + 7, + [2; 16], + Some(first.identity), + 1_100, + 100, + ) + .await + .unwrap(); + assert_eq!(next.prior_exclusion_deadline_ms, first.expires_at_ms); +} + +#[tokio::test] +async fn replacement_retains_grant_fences_and_rejects_partial_records() { + let fixture = TestStreamLeaseKv::start().await; + let kv = fixture.kv.clone(); + let client = StreamProcessLeaseClient::new(kv.clone()); + let mut first = client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_000, 100) + .await + .unwrap(); + first.grant_sequence = 9; + first.grant_expires_at_ms = 2_000; + let key = StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id: 7, + } + .to_path(); + kv.put(0, 0, key.as_bytes(), &serde_json::to_vec(&first).unwrap(), None) + .await + .unwrap(); + let renewed = client.renew(first.identity, 1_020, 100).await.unwrap(); + assert_eq!(renewed.grant_sequence, first.grant_sequence); + assert_eq!(renewed.grant_expires_at_ms, first.grant_expires_at_ms); + let next = client + .claim(StreamPurpose::Wal, 7, [2; 16], Some(first.identity), 1_050, 100) + .await + .unwrap(); + assert_eq!(next.prior_exclusion_deadline_ms, 2_000); + assert_eq!(next.grant_sequence, 9); + let mut partial = serde_json::to_value(&next).unwrap(); + partial + .as_object_mut() + .unwrap() + .remove("prior_exclusion_deadline_ms"); + kv.put(0, 0, key.as_bytes(), &serde_json::to_vec(&partial).unwrap(), None) + .await + .unwrap(); + assert!(matches!( + client.load(StreamPurpose::Wal, 7).await, + Err(StreamProcessLeaseError::Invalid(_)) + )); +} + +#[tokio::test] +async fn confirmed_group0_replacement_revokes_the_old_local_context() { + let fixture = TestStreamLeaseKv::start().await; + let client = StreamProcessLeaseClient::new(fixture.kv.clone()); + let first = client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_000, 12_000) + .await + .unwrap(); + let now = std::time::Instant::now(); + let owner = + StreamProcessAuthority::new(first.clone(), StreamProcessLeaseTiming::default(), 1_000, now).unwrap(); + let context = owner.capture(now).unwrap(); + let next = client + .claim( + StreamPurpose::Wal, + 7, + [2; 16], + Some(first.identity), + 1_100, + 12_000, + ) + .await + .unwrap(); + assert!(matches!( + owner.observe(next, 1_100, std::time::Instant::now()), + Err(StreamProcessLeaseError::Replaced) + )); + assert!(matches!( + owner.validate_context(&context, std::time::Instant::now()), + Err(StreamProcessLeaseError::Expired) + )); + assert!(owner.status(std::time::Instant::now()).fenced); +} diff --git a/lib/crowdb-kv-client/tests/stream_process_observation_test.rs b/lib/crowdb-kv-client/tests/stream_process_observation_test.rs new file mode 100644 index 000000000..61fbde58c --- /dev/null +++ b/lib/crowdb-kv-client/tests/stream_process_observation_test.rs @@ -0,0 +1,226 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/stream_lease.rs"] +mod fixture; + +use crowdb_kv_client::{GetOutcome, ReadMode, StreamProcessLeaseClient, StreamProcessLeaseError}; +use crowdb_protocol::{chunk_stream::StreamPurpose, common::ChunkKvExtra, key::InstanceKey}; +use fixture::TestStreamLeaseKv; + +#[tokio::test] +async fn replaced_boot_cannot_overwrite_or_withdraw_its_successor_observation() { + let fixture = TestStreamLeaseKv::start().await; + let client = StreamProcessLeaseClient::new(fixture.kv.clone()); + let first = client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_000, 100) + .await + .unwrap(); + let mut observation = ChunkKvExtra { + process_epoch: 1, + ..ChunkKvExtra::default() + }; + client + .publish_chunk_kv_observation(first.identity, "first", &observation, 1_001) + .await + .unwrap(); + let second = client + .claim(StreamPurpose::Wal, 7, [2; 16], Some(first.identity), 1_002, 100) + .await + .unwrap(); + observation.process_epoch = second.identity.process_epoch; + client + .publish_chunk_kv_observation(second.identity, "second", &observation, 1_003) + .await + .unwrap(); + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 7, + } + .to_path(); + let before = fixture + .kv + .get(0, 0, path.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap(); + observation.process_epoch = 1; + assert!(matches!( + client + .publish_chunk_kv_observation(first.identity, "stale", &observation, 1_004) + .await, + Err(StreamProcessLeaseError::Replaced) + )); + assert!(matches!( + client.withdraw_chunk_kv_observation(first.identity, 1_004).await, + Err(StreamProcessLeaseError::Replaced) + )); + let after = fixture + .kv + .get(0, 0, path.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap(); + match (before, after) { + ( + GetOutcome::Found { + value: before, + revision: before_revision, + }, + GetOutcome::Found { + value: after, + revision: after_revision, + }, + ) => { + assert_eq!(before, after); + assert_eq!(before_revision, after_revision); + } + other => panic!("successor observation changed: {other:?}"), + } + client + .withdraw_chunk_kv_observation(second.identity, 1_005) + .await + .unwrap(); + assert!(matches!( + fixture + .kv + .get(0, 0, path.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap(), + GetOutcome::NotFound + )); + assert_eq!(client.load(StreamPurpose::Wal, 7).await.unwrap().unwrap(), second); +} + +#[tokio::test] +async fn expiry_and_wrong_observation_epoch_cannot_publish_discovery() { + let fixture = TestStreamLeaseKv::start().await; + let client = StreamProcessLeaseClient::new(fixture.kv.clone()); + let lease = client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_000, 100) + .await + .unwrap(); + let mut observation = ChunkKvExtra { + process_epoch: 2, + ..ChunkKvExtra::default() + }; + assert!(matches!( + client + .publish_chunk_kv_observation(lease.identity, "wrong", &observation, 1_001) + .await, + Err(StreamProcessLeaseError::Invalid(_)) + )); + observation.process_epoch = 1; + assert!(matches!( + client + .publish_chunk_kv_observation(lease.identity, "expired", &observation, 1_100) + .await, + Err(StreamProcessLeaseError::Expired) + )); + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 7, + } + .to_path(); + assert!(matches!( + fixture + .kv + .get(0, 0, path.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap(), + GetOutcome::NotFound + )); +} + +#[tokio::test] +async fn observation_waits_for_an_inflight_renewal_instead_of_exhausting_busy_retries() { + exercise_inflight_lease_update(true).await; +} + +#[tokio::test] +async fn renewal_waits_for_an_inflight_observation_instead_of_exhausting_busy_retries() { + exercise_inflight_lease_update(false).await; +} + +async fn exercise_inflight_lease_update(renewal_first: bool) { + use crowdb_protocol::key::{StreamProcessLeaseKey, TextKey}; + use std::{sync::Arc, time::Duration}; + + let fixture = TestStreamLeaseKv::start().await; + let client = Arc::new(StreamProcessLeaseClient::new(fixture.kv.clone())); + let lease = client + .claim(StreamPurpose::Wal, 7, [1; 16], None, 1_000, 10_000) + .await + .unwrap(); + let group = fixture.store.get_group(0).unwrap(); + let permits = group.try_acquire_all_inflight_permits(); + let observation = ChunkKvExtra { + process_epoch: 1, + ..ChunkKvExtra::default() + }; + let updating = Arc::clone(&client); + let first_observation = observation.clone(); + let first = tokio::spawn(async move { + if renewal_first { + updating.renew(lease.identity, 1_001, 10_000).await.map(|_| ()) + } else { + updating + .publish_chunk_kv_observation(lease.identity, "live", &first_observation, 1_001) + .await + } + }); + let key = StreamProcessLeaseKey { + purpose: StreamPurpose::Wal, + instance_id: 7, + } + .to_path(); + tokio::time::timeout(Duration::from_secs(1), async { + while !group.conditional_write_pending_for_tests(key.as_bytes()) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let second = async { + if renewal_first { + client + .publish_chunk_kv_observation(lease.identity, "live", &observation, 1_002) + .await + } else { + client.renew(lease.identity, 1_002, 10_000).await.map(|_| ()) + } + }; + let release = async { + tokio::time::sleep(Duration::from_millis(20)).await; + drop(permits); + }; + let (result, ()) = tokio::join!(second, release); + first.await.unwrap().unwrap(); + assert!( + result.is_ok(), + "short lease-key contention must not starve either update: {result:?}" + ); + let path = InstanceKey { + service: "chunk-kv".into(), + instance_id: 7, + } + .to_path(); + let record = fixture + .kv + .get(0, 0, path.as_bytes(), ReadMode::Linearizable, None) + .await + .unwrap(); + let GetOutcome::Found { value, .. } = record else { + panic!("published observation missing"); + }; + let value: crowdb_protocol::common::InstanceValue = serde_json::from_slice(&value).unwrap(); + assert_eq!(value.rpc_endpoint, "live"); + assert_eq!(value.last_heartbeat_ms, if renewal_first { 1_002 } else { 1_001 }); + assert_eq!( + client + .load(StreamPurpose::Wal, 7) + .await + .unwrap() + .unwrap() + .expires_at_ms, + if renewal_first { 11_001 } else { 11_002 } + ); +} diff --git a/lib/crowdb-kv/src/cluster.rs b/lib/crowdb-kv/src/cluster.rs index 8b881f9be..af6c57e8c 100644 --- a/lib/crowdb-kv/src/cluster.rs +++ b/lib/crowdb-kv/src/cluster.rs @@ -19,6 +19,7 @@ pub mod group_operations; pub(crate) mod group_owner_fence; pub mod group_prepare; pub mod group_propose; +mod group_routes; #[cfg(feature = "test-util")] mod group_tests; pub mod kv_server; diff --git a/lib/crowdb-kv/src/cluster/group.rs b/lib/crowdb-kv/src/cluster/group.rs index 9fc48eb4a..3105f9493 100644 --- a/lib/crowdb-kv/src/cluster/group.rs +++ b/lib/crowdb-kv/src/cluster/group.rs @@ -280,6 +280,8 @@ pub struct PxGroup { pub(crate) coalesce_round_gate: parking_lot::Mutex>>, #[cfg(feature = "test-util")] pub(crate) coalesce_empty_drain_gate: std::sync::atomic::AtomicU8, + #[cfg(feature = "test-util")] + pub(crate) repair_gate: std::sync::atomic::AtomicU8, /// R36 self-weak back-reference, set once the group is wrapped in an /// [`Arc`] via [`Self::set_self_weak`]. The coalescer's per-batch timer /// task upgrades this to spawn a flush without the group holding a @@ -439,6 +441,8 @@ impl PxGroup { coalesce_round_gate: parking_lot::Mutex::new(None), #[cfg(feature = "test-util")] coalesce_empty_drain_gate: std::sync::atomic::AtomicU8::new(0), + #[cfg(feature = "test-util")] + repair_gate: std::sync::atomic::AtomicU8::new(0), self_weak: OnceLock::new(), coalescer: parking_lot::Mutex::new(None), coalesced_rounds_inflight: std::sync::atomic::AtomicU64::new(0), @@ -831,6 +835,17 @@ impl PxGroup { // The first slot above the contiguous frontier is, by definition, not // yet chosen locally — the lowest hole to fill. let gap_slot = contiguous + 1; + #[cfg(feature = "test-util")] + if self + .repair_gate + .compare_exchange(1, 2, Ordering::AcqRel, Ordering::Acquire) + .is_ok() + { + while self.repair_gate.load(Ordering::Acquire) == 2 { + tokio::task::yield_now().await; + } + self.repair_gate.store(0, Ordering::Release); + } let quorum = self.quorum(); let group_id = self.group_id; debug!( diff --git a/lib/crowdb-kv/src/cluster/group_election.rs b/lib/crowdb-kv/src/cluster/group_election.rs index 3d7e23a21..f208dd947 100644 --- a/lib/crowdb-kv/src/cluster/group_election.rs +++ b/lib/crowdb-kv/src/cluster/group_election.rs @@ -341,6 +341,7 @@ impl LeaderElection for PxGroup { pub(crate) enum StepDownReason { HigherTerm(u64), LeaseUnrenewable, + RecoveryIncomplete, Admin, } @@ -421,15 +422,17 @@ impl PxGroup { h.step_downs_lease.inc(); } } - StepDownReason::Admin => { - // Counted synchronously when the strict-fence request is - // accepted so status agrees with the API response. + StepDownReason::RecoveryIncomplete | StepDownReason::Admin => { + // Administrative step-downs are counted when accepted; failed + // recovery is neither a lease expiry nor a higher-term event. } } tenure_cancel.cancel(); let target_term = match reason { StepDownReason::HigherTerm(t) => t.max(my_term), - StepDownReason::LeaseUnrenewable | StepDownReason::Admin => my_term, + StepDownReason::LeaseUnrenewable | StepDownReason::RecoveryIncomplete | StepDownReason::Admin => { + my_term + } }; // `become_follower` clears the local replica's believed leader id // so observers reading `group.leader_id` after this point see diff --git a/lib/crowdb-kv/src/cluster/group_election_leader.rs b/lib/crowdb-kv/src/cluster/group_election_leader.rs index 474ed744e..a20f7df10 100644 --- a/lib/crowdb-kv/src/cluster/group_election_leader.rs +++ b/lib/crowdb-kv/src/cluster/group_election_leader.rs @@ -35,7 +35,7 @@ fn spawn_cancel_bridge(parent: CancellationToken, child: CancellationToken) { ); } -use crate::cluster::group::{PendingReadBarrier, PxGroup, RemoteReplicaKind}; +use crate::cluster::group::{PendingReadBarrier, PxGroup, RemoteReplicaKind, RepairOutcome}; use crate::cluster::group_election::{ HeartbeatOutcome, LeaderElection, PendingLeaderHandoff, ReadBarrierOutcome, StepDownReason, }; @@ -341,6 +341,61 @@ impl PxGroup { outcome } + fn start_leader_recovery( + self: &Arc, + cfg: &PxElectionConfig, + tenure_cancel: &CancellationToken, + ) -> Option> { + if let Some(handoff) = self.pending_leader_handoff.lock().take() { + let group_for_task = self.clone(); + let cfg_for_task = *cfg; + let cancel_for_task = tenure_cancel.clone(); + Some(tokio::spawn( + async move { + group_for_task + .run_bulk_phase1( + handoff.term, + handoff.peer_floor, + handoff.peer_ceiling, + cfg_for_task, + cancel_for_task, + ) + .await; + } + .instrument(tracing::Span::current()), + )) + } else { + self.leader_read_ready.store(true, Ordering::Release); + None + } + } + + fn start_gap_repair(self: &Arc, cancel: &CancellationToken) -> Option> { + if self.local_replica().contiguous_chosen() >= self.local_replica().last_chosen_slot() { + return None; + } + let group = Arc::clone(self); + let cancel = cancel.clone(); + let ceiling = self.local_replica().last_chosen_slot(); + Some(tokio::spawn(async move { + // Drain the observed backlog in one independent pass. Tying each + // hole to a heartbeat tick makes a brief outage stall apply fences + // for heartbeat_interval * abandoned_slots after quorum returns. + while group.local_replica().contiguous_chosen() < ceiling { + tokio::select! { + biased; + () = cancel.cancelled() => return, + outcome = group.repair_once() => { + if !matches!(outcome, RepairOutcome::Filled { .. }) { + return; + } + } + } + tokio::task::yield_now().await; + } + })) + } + /// Leader-state inner loop: tick heartbeats + lease bookkeeping /// until step-down (higher term, lease unrenewable, admin) or /// driver cancel. @@ -398,27 +453,7 @@ impl PxGroup { // Consume the handoff stashed by finalize_leader and spawn bulk // Phase 1 on the per-tenure cancel token. - if let Some(handoff) = self.pending_leader_handoff.lock().take() { - let group_for_task = self.clone(); - let cfg_for_task = *cfg; - let cancel_for_task = tenure_cancel.clone(); - tokio::spawn( - async move { - group_for_task - .run_bulk_phase1( - handoff.term, - handoff.peer_floor, - handoff.peer_ceiling, - cfg_for_task, - cancel_for_task, - ) - .await; - } - .instrument(tracing::Span::current()), - ); - } else { - self.leader_read_ready.store(true, Ordering::Release); - } + let recovery_task = self.start_leader_recovery(cfg, &tenure_cancel); // Establish the first quorum heartbeat before waiting on the ticker. // This makes lease readiness deterministic when the election driver @@ -433,6 +468,7 @@ impl PxGroup { ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); // First tick fires immediately; consume it so the loop starts clean. ticker.tick().await; + let mut gap_repair: Option> = None; info!( group_id, @@ -457,6 +493,17 @@ impl PxGroup { tenure_cancel.cancel(); return; } + // A failed recovery or an uncertain conditional proposal closes + // reads for the entire tenure. Heartbeats alone cannot resolve + // that uncertainty; a new election must recover the accepted log. + if recovery_task + .as_ref() + .map_or(true, tokio::task::JoinHandle::is_finished) + && !self.leader_read_ready() + { + self.step_down(&tenure_cancel, leader_term, StepDownReason::RecoveryIncomplete); + return; + } tokio::select! { biased; () = cancel.cancelled() => { @@ -482,12 +529,11 @@ impl PxGroup { self.step_down(&tenure_cancel, leader_term, StepDownReason::LeaseUnrenewable); return; } - // Opportunistic background repair: close the lowest - // gap in the open prefix so the contiguous frontier - // (and group safe-slot) can advance past abandoned - // slots. A no-gap leader returns immediately without - // any RPCs, so steady state pays nothing. - let _ = self.repair_once().await; + // Keep at most one repair in flight. Prepare or + // persistence IO must not occupy the heartbeat loop. + if gap_repair.as_ref().map_or(true, tokio::task::JoinHandle::is_finished) { + gap_repair = self.start_gap_repair(&tenure_cancel); + } } HeartbeatOutcome::SteppedDown { peer_term } => { self.step_down(&tenure_cancel, leader_term, StepDownReason::HigherTerm(peer_term)); diff --git a/lib/crowdb-kv/src/cluster/group_fetchgap.rs b/lib/crowdb-kv/src/cluster/group_fetchgap.rs index 35a5e6cf8..cbe3b850b 100644 --- a/lib/crowdb-kv/src/cluster/group_fetchgap.rs +++ b/lib/crowdb-kv/src/cluster/group_fetchgap.rs @@ -112,7 +112,16 @@ pub(crate) async fn run_fetchgap_driver( }; let result = tokio::time::timeout( rpc_timeout_clone, - transport.send_fetch_gap(&rpc_endpoint_clone, group_id, slot, term, leader_id), + transport.send_fetch_gap( + crate::rpc::ReplicaDestination { + endpoint: &rpc_endpoint_clone, + replica_id: leader_id, + }, + group_id, + slot, + term, + leader_id, + ), ) .await .map_err(|_| { diff --git a/lib/crowdb-kv/src/cluster/group_membership.rs b/lib/crowdb-kv/src/cluster/group_membership.rs index a74e16bd5..7996e7761 100644 --- a/lib/crowdb-kv/src/cluster/group_membership.rs +++ b/lib/crowdb-kv/src/cluster/group_membership.rs @@ -211,12 +211,21 @@ impl PxGroup { } pub fn inherit_local_state_from(&mut self, prior: &Self) { + let elect_bootstrap = prior.quorum() == 1 + && self.quorum() > 1 + && self.local_replica.role() == crate::cluster::local_replica::PxLocalReplicaRole::Follower + && self.local_replica.current_term_snapshot() == 0; // Inherited learners must also share admission state with requests // still finishing against the prior topology snapshot. self.owner_admission = Arc::clone(&prior.owner_admission); self.cas_transient_map = Arc::clone(&prior.cas_transient_map); self.cas_request_nonce = Arc::clone(&prior.cas_request_nonce); self.local_replica = PxLocalReplica::new_inheriting_election_state(prior.local_replica()); + // Preserve the caller's bootstrap demotion after the last state handoff. + // An elected tenure acquired meanwhile must always be inherited. + if elect_bootstrap && self.local_replica.current_term_snapshot() == 0 { + self.local_replica.become_follower(0); + } self.next_slot.store( self.next_slot .load(Ordering::Acquire) @@ -277,7 +286,7 @@ impl PxGroup { // Existing Real entry: update in place. if let Some(RemoteReplicaKind::Real(remote)) = self.remote_replicas.get_mut(idx) { let old_endpoint = remote.endpoint.clone(); - endpoint.clone_into(&mut remote.endpoint); + remote.set_configured_endpoint(endpoint); return Some(old_endpoint); } diff --git a/lib/crowdb-kv/src/cluster/group_routes.rs b/lib/crowdb-kv/src/cluster/group_routes.rs new file mode 100644 index 000000000..33641edbd --- /dev/null +++ b/lib/crowdb-kv/src/cluster/group_routes.rs @@ -0,0 +1,19 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Connection routes never change durable membership or voting configuration. + +use super::group::PxGroup; +use std::{collections::BTreeMap, net::SocketAddr}; + +impl PxGroup { + /// Apply a complete verified route view to existing remote identities. + /// Missing routes block outgoing calls rather than reusing an unverified IP. + pub fn set_recovery_routes(&self, routes: &BTreeMap) { + for remote in &self.remote_replicas { + if let Some(remote) = remote.as_real() { + remote.set_recovery_endpoint(routes.get(&remote.node_id).copied()); + } + } + } +} diff --git a/lib/crowdb-kv/src/cluster/group_tests.rs b/lib/crowdb-kv/src/cluster/group_tests.rs index 36d470122..7c402b8fa 100644 --- a/lib/crowdb-kv/src/cluster/group_tests.rs +++ b/lib/crowdb-kv/src/cluster/group_tests.rs @@ -14,6 +14,12 @@ use crate::paxos::PxNodeId; /// repair step, and peer-applied injection — to integration tests under /// `tests/` without permanently widening the production public API. impl PxGroup { + /// Whether a conditional mutation currently owns one key's admission. + #[must_use] + pub fn conditional_write_pending_for_tests(&self, key: &[u8]) -> bool { + self.cas_transient_map.get(key).is_some() + } + /// Run one heartbeat round without depending on the election timer. pub async fn heartbeat_round_for_tests(self: &std::sync::Arc) -> bool { matches!( @@ -40,6 +46,20 @@ impl PxGroup { &self.inflight.semaphore } + /// Hold the next gap repair to model slow prepare or persistence IO. + pub fn pause_repair_for_tests(&self) { + self.repair_gate.store(1, std::sync::atomic::Ordering::Release); + } + + #[must_use] + pub fn repair_is_paused_for_tests(&self) -> bool { + self.repair_gate.load(std::sync::atomic::Ordering::Acquire) == 2 + } + + pub fn resume_repair_for_tests(&self) { + self.repair_gate.store(3, std::sync::atomic::Ordering::Release); + } + /// Run one background-repair step, returning the slot that was filled /// (`Some`) or `None` when there was no gap to repair / repair did not /// choose. Wraps the internal [`Self::repair_once`]. diff --git a/lib/crowdb-kv/src/cluster/remote_replica.rs b/lib/crowdb-kv/src/cluster/remote_replica.rs index 0b470ec36..11f46343d 100644 --- a/lib/crowdb-kv/src/cluster/remote_replica.rs +++ b/lib/crowdb-kv/src/cluster/remote_replica.rs @@ -22,7 +22,7 @@ use crate::common::report::OperationReport; use crate::metrics::{Counter, LatencySummary, MetricPoint, MetricsRegistry}; use crate::paxos::roles::{PxAcceptReply, PxBallot, PxLogEntry, PxPrepareReply}; use crate::paxos::PxNodeId; -use crate::rpc::PxRpcTransport; +use crate::rpc::{PxRpcTransport, ReplicaDestination}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant}; @@ -32,6 +32,7 @@ use tracing::{debug, info}; pub struct PxRemoteReplica { pub(crate) node_id: PxNodeId, pub(crate) endpoint: String, + rpc_endpoint: arc_swap::ArcSwap, /// Per-RPC deadline for the crowdb-rpc unary calls (`Prepare`, `Accept`, /// `PreVote`, `RequestVote`, `Heartbeat`, `StepDown`, `FetchGap`). Snapshot of /// `PxElectionConfig::peer_rpc_timeout_ms`. @@ -81,10 +82,21 @@ impl ReplicaClient for PxRemoteReplica { membership_epoch: u64, ) -> Result { let transport = self.transport_or_err()?; + let endpoint = self.rpc_endpoint.load(); let started = Instant::now(); let result = tokio::time::timeout( self.rpc_timeout, - transport.send_prepare(&self.endpoint, slot, ballot, term, group_id, membership_epoch), + transport.send_prepare( + ReplicaDestination { + endpoint: &endpoint, + replica_id: self.node_id, + }, + slot, + ballot, + term, + group_id, + membership_epoch, + ), ) .await; self.finish_rpc(started, "prepare", result) @@ -97,10 +109,19 @@ impl ReplicaClient for PxRemoteReplica { membership_epoch: u64, ) -> Result { let transport = self.transport_or_err()?; + let endpoint = self.rpc_endpoint.load(); let started = Instant::now(); let result = tokio::time::timeout( self.rpc_timeout, - transport.send_accept(&self.endpoint, entry, group_id, membership_epoch), + transport.send_accept( + ReplicaDestination { + endpoint: &endpoint, + replica_id: self.node_id, + }, + entry, + group_id, + membership_epoch, + ), ) .await; self.finish_rpc(started, "accept", result) @@ -112,10 +133,18 @@ impl ReplicaClient for PxRemoteReplica { group_id: u64, ) -> Result { let transport = self.transport_or_err()?; + let endpoint = self.rpc_endpoint.load(); let started = Instant::now(); let result = tokio::time::timeout( self.rpc_timeout, - transport.send_pre_vote(&self.endpoint, req, group_id), + transport.send_pre_vote( + ReplicaDestination { + endpoint: &endpoint, + replica_id: self.node_id, + }, + req, + group_id, + ), ) .await; self.finish_rpc(started, "pre_vote", result) @@ -127,10 +156,18 @@ impl ReplicaClient for PxRemoteReplica { group_id: u64, ) -> Result { let transport = self.transport_or_err()?; + let endpoint = self.rpc_endpoint.load(); let started = Instant::now(); let result = tokio::time::timeout( self.rpc_timeout, - transport.send_request_vote(&self.endpoint, req, group_id), + transport.send_request_vote( + ReplicaDestination { + endpoint: &endpoint, + replica_id: self.node_id, + }, + req, + group_id, + ), ) .await; self.finish_rpc(started, "request_vote", result) @@ -142,10 +179,18 @@ impl ReplicaClient for PxRemoteReplica { group_id: u64, ) -> Result { let transport = self.transport_or_err()?; + let endpoint = self.rpc_endpoint.load(); let started = Instant::now(); let result = tokio::time::timeout( self.rpc_timeout, - transport.send_heartbeat(&self.endpoint, req, group_id), + transport.send_heartbeat( + ReplicaDestination { + endpoint: &endpoint, + replica_id: self.node_id, + }, + req, + group_id, + ), ) .await; self.finish_rpc(started, "heartbeat", result) @@ -157,10 +202,18 @@ impl ReplicaClient for PxRemoteReplica { group_id: u64, ) -> Result { let transport = self.transport_or_err()?; + let endpoint = self.rpc_endpoint.load(); let started = Instant::now(); let result = tokio::time::timeout( self.rpc_timeout, - transport.send_step_down(&self.endpoint, req, group_id), + transport.send_step_down( + ReplicaDestination { + endpoint: &endpoint, + replica_id: self.node_id, + }, + req, + group_id, + ), ) .await; self.finish_rpc(started, "step_down", result) @@ -172,6 +225,7 @@ impl PxRemoteReplica { pub fn new(node_id: PxNodeId, endpoint: String) -> Self { Self { node_id, + rpc_endpoint: arc_swap::ArcSwap::from_pointee(endpoint.clone()), endpoint, rpc_timeout: Duration::from_millis(PxElectionConfig::DEFAULT.peer_rpc_timeout_ms), voting: true, @@ -226,7 +280,18 @@ impl PxRemoteReplica { ballot_round: u64, ) -> Result<(), PxReplicaError> { let transport = self.transport_or_err_sync()?; - transport.send_chosen(&self.endpoint, group_id, slot, term, leader_id, ballot_round) + let endpoint = self.rpc_endpoint.load(); + transport.send_chosen( + ReplicaDestination { + endpoint: &endpoint, + replica_id: self.node_id, + }, + group_id, + slot, + term, + leader_id, + ballot_round, + ) } /// R63: fire-and-forget batch chosen notice covering `[start_slot, @@ -248,8 +313,12 @@ impl PxRemoteReplica { ballot_round: u64, ) -> Result<(), PxReplicaError> { let transport = self.transport_or_err_sync()?; + let endpoint = self.rpc_endpoint.load(); transport.send_batch_chosen( - &self.endpoint, + ReplicaDestination { + endpoint: &endpoint, + replica_id: self.node_id, + }, group_id, start_slot, end_slot, @@ -277,10 +346,20 @@ impl PxRemoteReplica { group_id: u64, ) -> Result { let transport = self.transport_or_err()?; + let endpoint = self.rpc_endpoint.load(); let started = Instant::now(); let result = tokio::time::timeout( self.rpc_timeout, - transport.send_fetch_gap(&self.endpoint, group_id, slot, term, leader_id), + transport.send_fetch_gap( + ReplicaDestination { + endpoint: &endpoint, + replica_id: self.node_id, + }, + group_id, + slot, + term, + leader_id, + ), ) .await; self.finish_rpc(started, "fetch_gap", result) @@ -372,8 +451,23 @@ impl PxRemoteReplica { /// Get the endpoint string (for spawned tasks that need to call /// the transport directly). - pub(crate) fn endpoint_str(&self) -> &str { - &self.endpoint + pub(crate) fn endpoint_str(&self) -> arc_swap::Guard> { + self.rpc_endpoint.load() + } + + pub(crate) fn set_recovery_endpoint(&self, endpoint: Option) { + let endpoint = endpoint.map_or_else(String::new, |address| address.to_string()); + if self.rpc_endpoint.load().as_str() != endpoint { + self.rpc_endpoint.store(Arc::new(endpoint)); + if let Some(transport) = self.rpc_transport.get() { + transport.clear_connections(); + } + } + } + + pub(crate) fn set_configured_endpoint(&mut self, endpoint: String) { + self.rpc_endpoint.store(Arc::new(endpoint.clone())); + self.endpoint = endpoint; } /// Register per-peer RPC latency summary and error counter with the @@ -433,7 +527,7 @@ impl PxRemoteReplica { .then_some(crowdb_protocol::mgmt::MetricsSnapshot { rpc_count, err_count }); RemoteStatus { id: self.node_id, - endpoint: self.endpoint.clone(), + endpoint: self.rpc_endpoint.load().to_string(), voting: self.voting, status, messages, diff --git a/lib/crowdb-kv/src/rpc.rs b/lib/crowdb-kv/src/rpc.rs index 0699bf13c..7cea2daff 100644 --- a/lib/crowdb-kv/src/rpc.rs +++ b/lib/crowdb-kv/src/rpc.rs @@ -25,7 +25,7 @@ pub(crate) mod snapshot_registry; pub(crate) use kv_rpc_service::{KvClientRpcForwarder, KvRpcService}; #[allow(unused_imports)] pub(crate) use px_rpc_service::PxRpcService; -pub use px_rpc_transport::PxRpcTransport; +pub use px_rpc_transport::{PxRpcTransport, ReplicaDestination}; // Re-export the flatbuffer KV client request/response types so the // server handler (`kv_rpc_service.rs`) can reference them via diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service.rs b/lib/crowdb-kv/src/rpc/px_rpc_service.rs index 62392d096..a04944799 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_service.rs +++ b/lib/crowdb-kv/src/rpc/px_rpc_service.rs @@ -60,6 +60,7 @@ pub struct PxRpcService { store: Arc, rt: Handle, snapshots: Arc, + cluster_id: Option, } fn spawn_request(rt: &Handle, operation: &'static str, store_id: u64, future: F) @@ -125,7 +126,12 @@ impl PxRpcService { registry.reap_expired(); } }); - Self { store, rt, snapshots } + Self { + store, + rt, + snapshots, + cluster_id: std::env::var("CROWDB_KV_CLUSTER_ID").ok(), + } } /// Register all consensus request handlers into the `RpcServer`. @@ -190,1311 +196,43 @@ impl PxRpcService { f: fn(&Self, ServerRequest, &Arc), ) -> impl Fn(ServerRequest) + Send + 'static { move |req| { - f(&this, req, &server); - } - } - - // ── Prepare ────────────────────────────────────────────────── - - #[allow(clippy::too_many_lines)] - fn handle_prepare(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EPromiseResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "prepare", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let slot = fb_req.slot(); - let round = fb_req.round(); - let leader_id = fb_req.leader_id(); - let term = fb_req.term(); - let membership_epoch = fb_req.membership_epoch(); - - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let responder_epoch = group.membership_epoch(); - - // Membership-epoch fence (same as px_service.rs L131-147). - if membership_epoch != responder_epoch { - let converged_epoch = group.adopt_membership_epoch(membership_epoch); - warn!( - group_id, - slot, - round, - leader_id, - proposer_epoch = membership_epoch, - responder_epoch, - converged_epoch, - "prepare rejected by membership-epoch fence; adopting higher epoch from proposer" - ); - let term = group.local_replica().current_term_snapshot(); - let ctrl = build_promise_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - None, - false, - 0, - 0, - term, - false, - responder_epoch, - true, - ); - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); + if !this.target_matches(&req) { return; } - - let ballot = PxBallot { round, leader_id }; - let replica = group.local_replica(); - let reply = - ::on_prepare(replica, slot, ballot, term, group_id).await; - let ctrl = match reply { - Ok(PxPrepareReply::Promised { slot, accepted }) => { - let mut builder = FlatBufferBuilder::new(); - let prev_accepted_off = accepted.as_ref().map(|entry| { - let payload = builder.create_vector(&entry.payload); - FBAcceptedValue::create( - &mut builder, - &FBAcceptedValueArgs { - slot: entry.slot, - round: entry.ballot.round, - leader_id: entry.ballot.leader_id, - term: entry.term, - payload: Some(payload), - }, - ) - }); - let term = replica.current_term_snapshot(); - finish_promise_response( - builder, - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - prev_accepted_off, - false, - 0, - 0, - term, - false, - responder_epoch, - false, - ) - } - Ok(PxPrepareReply::TermStale { slot, new_term }) => build_promise_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - None, - false, - 0, - 0, - new_term, - true, - responder_epoch, - false, - ), - Ok(PxPrepareReply::Rejected { - slot, - current_promised, - }) => { - warn!( - group_id, - slot, - current_round = current_promised.round, - current_leader_id = current_promised.leader_id, - "prepare rejected; next step: proposer should retry with a higher ballot" - ); - let term = replica.current_term_snapshot(); - build_promise_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - None, - true, - current_promised.round, - current_promised.leader_id, - term, - false, - responder_epoch, - false, - ) - } - Ok(PxPrepareReply::EpochMismatch { .. }) => { - unreachable!("on_prepare does not produce EpochMismatch") - } - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_promise_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - slot, - round, - leader_id, - None, - false, - 0, - 0, - replica.current_term_snapshot(), - false, - responder_epoch, - false, - ) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── Accept ─────────────────────────────────────────────────── - - #[allow(clippy::too_many_lines)] - fn handle_accept(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EAcceptedResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "accept", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let slot = fb_req.slot(); - let round = fb_req.round(); - let leader_id = fb_req.leader_id(); - let term = fb_req.term(); - let membership_epoch = fb_req.membership_epoch(); - - let Some(fb_value) = fb_req.value() else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "missing value", - ); - return; - }; - let payload_bytes: Vec = fb_value.payload().map(|v| v.bytes().to_vec()).unwrap_or_default(); - let entry = PxLogEntry { - slot, - ballot: PxBallot { round, leader_id }, - term, - payload: payload_bytes.into(), - }; - - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let responder_epoch = group.membership_epoch(); - - if membership_epoch != responder_epoch { - let converged_epoch = group.adopt_membership_epoch(membership_epoch); - warn!( - group_id, - slot, - round, - leader_id, - proposer_epoch = membership_epoch, - responder_epoch, - converged_epoch, - "accept rejected by membership-epoch fence; adopting higher epoch from proposer" - ); - let term = group.local_replica().current_term_snapshot(); - let ctrl = build_accepted_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - false, - 0, - 0, - term, - false, - responder_epoch, - true, - ); - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - return; - } - - let replica = group.local_replica(); - let reply = ::on_accept(replica, &entry, group_id).await; - let (rejected, rejected_round, rejected_leader_id, term_stale, reply_term) = match reply { - Ok(PxAcceptReply::Accepted { .. }) => (false, 0, 0, false, replica.current_term_snapshot()), - Ok(PxAcceptReply::Rejected { current_promised, .. }) => { - warn!( - group_id, - slot, - current_round = current_promised.round, - current_leader_id = current_promised.leader_id, - "accept rejected; next step: proposer should run prepare with a higher ballot" - ); - ( - true, - current_promised.round, - current_promised.leader_id, - false, - replica.current_term_snapshot(), - ) - } - Ok(PxAcceptReply::TermStale { new_term, .. }) => { - warn!( - group_id, - slot, new_term, "accept rejected by term fence; proposer should step down" - ); - (false, 0, 0, true, new_term) - } - Ok(PxAcceptReply::EpochMismatch { .. }) => { - unreachable!("on_accept does not produce EpochMismatch") - } - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - let ctrl = build_accepted_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - slot, - round, - leader_id, - false, - 0, - 0, - replica.current_term_snapshot(), - false, - responder_epoch, - false, - ); - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - return; - } - }; - let ctrl = build_accepted_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - slot, - round, - leader_id, - rejected, - rejected_round, - rejected_leader_id, - reply_term, - term_stale, - responder_epoch, - false, - ); - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── PreVote ────────────────────────────────────────────────── - - fn handle_pre_vote(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EPreVoteResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "pre_vote", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let payload = VoteRequestPayload { - term: fb_req.term(), - candidate_id: fb_req.candidate_id(), - accepted_log_tip_slot: fb_req.accepted_log_tip_slot(), - accepted_log_tip_term: fb_req.accepted_log_tip_term(), - }; - let replica = group.local_replica(); - let reply = ::on_pre_vote(replica, payload, group_id).await; - let ctrl = match reply { - Ok(r) => build_pre_vote_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - r.term, - r.granted, - r.contiguous_chosen, - r.last_chosen_term, - r.highest_seen_slot, - ), - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_pre_vote_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - group_id, - 0, - false, - 0, - 0, - 0, - ) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── RequestVote ────────────────────────────────────────────── - - fn handle_request_vote(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::ERequestVoteResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "request_vote", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let payload = VoteRequestPayload { - term: fb_req.term(), - candidate_id: fb_req.candidate_id(), - accepted_log_tip_slot: fb_req.accepted_log_tip_slot(), - accepted_log_tip_term: fb_req.accepted_log_tip_term(), - }; - let replica = group.local_replica(); - let reply = ::on_request_vote(replica, payload, group_id).await; - let ctrl = match reply { - Ok(r) => build_request_vote_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - r.term, - r.granted, - r.contiguous_chosen, - r.last_chosen_term, - r.highest_seen_slot, - ), - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_request_vote_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - group_id, - 0, - false, - 0, - 0, - 0, - ) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── Heartbeat ──────────────────────────────────────────────── - - fn handle_heartbeat(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EHeartbeatResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "heartbeat", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let payload = HeartbeatRequestPayload { - term: fb_req.term(), - leader_id: fb_req.leader_id(), - prev_log_slot: fb_req.prev_log_slot(), - prev_log_term: fb_req.prev_log_term(), - committed_safe_slot: fb_req.committed_safe_slot(), - lease_grant_until_ms_mono: fb_req.lease_grant_until_ms_mono(), - t_send_ms_mono: fb_req.t_send_ms_mono(), - }; - let replica = group.local_replica(); - let reply = ::on_heartbeat(replica, payload, group_id).await; - let ctrl = match reply { - Ok(r) => build_heartbeat_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - r.term, - r.success, - r.contiguous_chosen, - r.last_chosen_term, - r.contiguous_applied, - r.highest_seen_slot, - r.durable_snapshot_slot, - ), - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_heartbeat_response( - req_id, - create_nano, - code, - Some(&msg), - 1, - group_id, - 0, - false, - 0, - 0, - 0, - 0, - 0, - ) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── StepDown ───────────────────────────────────────────────── - - fn handle_step_down(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle_usize = req.conn_handle as usize; - let msg_type = FBMsgType::EStepDownResponse.0 as u16; - let store = Arc::clone(&self.store); - let server = Arc::clone(server); - spawn_request(&self.rt, "step_down", store.store_id, async move { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&store, group_id); - let Some(group) = store.get_group(group_id) else { - submit_error( - &server, - conn_handle_usize as *mut std::ffi::c_void, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; - }; - let reason = fb_req.reason().unwrap_or("").to_string(); - let payload = StepDownRequestPayload { - term: fb_req.term(), - target_leader_id: fb_req.target_leader_id(), - reason, - }; - let replica = group.local_replica(); - let reply = ::on_step_down(replica, &payload, group_id).await; - let ctrl = match reply { - Ok(r) => build_step_down_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - r.accepted, - r.current_term, - r.current_leader_id, - ), - Err(e) => { - let (code, msg) = px_error_to_ret_code(&e); - build_step_down_response(req_id, create_nano, code, Some(&msg), 1, group_id, false, 0, 0) - } - }; - submit_fb_response( - &server, - conn_handle_usize as *mut std::ffi::c_void, - ctrl, - msg_type, - req_id, - ); - // req dropped here, frame released - }); - } - - // ── ChosenNotification (fire-and-forget) ───────────────────── - - #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] - fn handle_chosen_notice(&self, req: ServerRequest, _server: &Arc) { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - debug!("chosen notice: invalid flatbuffer"); - return; - }; - let group_id = fb_req.group_id(); - let slot = fb_req.slot(); - let term = fb_req.term(); - let leader_id = fb_req.leader_id(); - let ballot_round = fb_req.ballot_round(); - - let Some(group) = self.store.get_group(group_id) else { - debug!(slot, term, "chosen notice dropped (group not found)"); - return; - }; - let chosen_ballot = PxBallot { - round: ballot_round, - leader_id, - }; - spawn_group_request( - &self.rt, - "chosen_notice", - self.store.store_id, - group_id, - group.local_replica().id, - async move { - let replica = group.local_replica(); - let accepted = replica.accepted_at(slot).await; - let ballot_matches = accepted.as_ref().is_some_and(|e| e.ballot == chosen_ballot); - if ballot_matches { - replica.learner.update_chosen_frontier(slot, term); - replica.wake_apply_loop(); - debug!( - slot, - term, leader_id, ballot_round, "chosen notification applied (ballot match)" - ); - } else { - replica.note_chosen(slot, term); - replica.record_gap(slot); - if let Some(ref entry) = accepted { - replica.incr_chosen_notice_stale(); - debug!( - slot, - term, - leader_id, - chosen_ballot_round = ballot_round, - accepted_ballot_round = entry.ballot.round, - accepted_ballot_leader = entry.ballot.leader_id, - "chosen notification stale ballot (gap recorded)" - ); - } else { - replica.incr_chosen_notice_missing(); - debug!( - slot, - term, leader_id, "chosen notification missing value (gap recorded)" - ); - } - } - }, - ); - } - - // ── BatchChosenNotification (fire-and-forget) ──────────────── - - #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] - fn handle_batch_chosen(&self, req: ServerRequest, _server: &Arc) { - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - debug!("batch chosen: invalid flatbuffer"); - return; - }; - let group_id = fb_req.group_id(); - let start_slot = fb_req.start_slot(); - let end_slot = fb_req.end_slot(); - let term = fb_req.term(); - let leader_id = fb_req.leader_id(); - let ballot_round = fb_req.ballot_round(); - - let Some(group) = self.store.get_group(group_id) else { - debug!("batch chosen dropped (group not found)"); - return; - }; - let chosen_ballot = PxBallot { - round: ballot_round, - leader_id, - }; - spawn_group_request( - &self.rt, - "batch_chosen", - self.store.store_id, - group_id, - group.local_replica().id, - async move { - let replica = group.local_replica(); - let mut advanced_count = 0u64; - for slot in start_slot..=end_slot { - let accepted = replica.accepted_at(slot).await; - if accepted.as_ref().is_some_and(|e| e.ballot == chosen_ballot) { - replica.learner.update_chosen_frontier(slot, term); - advanced_count += 1; - } else { - replica.note_chosen(slot, term); - replica.record_gap(slot); - } - } - replica.wake_apply_loop(); - debug!( - group_id, - start_slot, - end_slot, - term, - leader_id, - ballot_round, - advanced_count, - "batch chosen notification applied" - ); - }, - ); + f(&this, req, &server); + } } - // ── FetchGap ───────────────────────────────────────────────── - - #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] - fn handle_fetch_gap(&self, req: ServerRequest, server: &Arc) { - let span = request_span("fetch_gap", self.store.store_id); - let _entered = span.enter(); - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle = req.conn_handle; - let msg_type = FBMsgType::EFetchGapResponse.0 as u16; - - let Ok(fb_req) = flatbuffers::root::(req.control()) else { - submit_error( - server, - conn_handle, - req_id, - create_nano, - msg_type, - FBKvRetCode::InvalidArgument, - "invalid request flatbuffer", - ); - return; - }; - let group_id = fb_req.group_id(); - record_group_context(&self.store, group_id); - let slot = fb_req.slot(); - - let Some(group) = self.store.get_group(group_id) else { - submit_error( - server, - conn_handle, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "px group not found", - ); - return; + fn target_matches(&self, req: &ServerRequest) -> bool { + let consensus = [ + FBMsgType::EPrepareRequest, + FBMsgType::EAcceptRequest, + FBMsgType::EPreVoteRequest, + FBMsgType::ERequestVoteRequest, + FBMsgType::EHeartbeatRequest, + FBMsgType::EStepDownRequest, + FBMsgType::EChosenNotification, + FBMsgType::EBatchChosenNotification, + FBMsgType::EFetchGapRequest, + ]; + if !consensus + .iter() + .any(|kind| u16::try_from(kind.0).ok() == Some(req.msg_type)) + { + return true; + } + let Ok(target) = + flatbuffers::root::(req.control()) + else { + return true; }; - if let Some(resp) = group.handle_fetch_gap(slot) { - let payload_vec = resp.payload.clone(); - let ctrl = build_fetch_gap_response( - req_id, - create_nano, - FBKvRetCode::Success, - None, - 1, - group_id, - resp.slot, - resp.term, - resp.ballot_round, - resp.leader_id, - &payload_vec, - ); - submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); - } else { - debug!(slot, "fetch_gap no value (not yet chosen)"); - submit_error( - server, - conn_handle, - req_id, - create_nano, - msg_type, - FBKvRetCode::NotFound, - "slot not yet chosen", - ); + if target.target_cluster_id() != self.cluster_id.as_deref() { + return false; } - // req dropped here, frame released - } - - // ── Snapshot streaming ─────────────────────────────────────── - - #[allow(clippy::too_many_lines)] - fn handle_snapshot_begin(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle = req.conn_handle as usize; - let store = Arc::clone(&self.store); - let registry = Arc::clone(&self.snapshots); - let server = Arc::clone(server); - spawn_request(&self.rt, "snapshot_begin", store.store_id, async move { - let Ok(request) = flatbuffers::root::(req.control()) else { - submit_snapshot_begin( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::InvalidArgument, - Some("invalid snapshot Begin request"), - None, - 0, - ); - return; - }; - let group_id = request.group_id(); - record_group_context(&store, group_id); - let requested = request.max_chunk_bytes() as usize; - if requested == 0 { - submit_snapshot_begin( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::InvalidArgument, - Some("max_chunk_bytes must be nonzero"), - None, - group_id, - ); - return; - } - let Some(group) = store.get_group(group_id) else { - submit_snapshot_begin( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::NotFound, - Some("px group not found"), - None, - group_id, - ); - return; - }; - let membership_epoch = group.membership_epoch(); - let chunk_bytes = requested.min(store.snapshot_chunk_bytes).min(1024 * 1024); - if chunk_bytes == 0 { - submit_snapshot_begin( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::Internal, - Some("snapshot source chunk size is not configured"), - None, - group_id, - ); - return; - } - let permit = match registry.reserve() { - Ok(permit) => permit, - Err(error) => { - let (code, message) = snapshot_error(&error); - submit_snapshot_begin( - &server, - conn_handle, - req_id, - create_nano, - code, - Some(&message), - None, - group_id, - ); - return; - } - }; - let export_group = Arc::clone(&group); - let export_started = std::time::Instant::now(); - let exporter = match tokio::task::spawn_blocking(move || { - export_group - .local_replica() - .learner - .engine() - .snapshot_export_begin(chunk_bytes) + target.target_replica_id() == 0 + || self.store.get_group(target.group_id()).map_or(true, |group| { + group.local_replica().id == target.target_replica_id() }) - .await - { - Ok(Ok(exporter)) => exporter, - Ok(Err(error)) => { - submit_snapshot_begin( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::Internal, - Some(&error), - None, - group_id, - ); - return; - } - Err(error) => { - let message = format!("snapshot export task failed: {error}"); - submit_snapshot_begin( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::Internal, - Some(&message), - None, - group_id, - ); - return; - } - }; - registry.observe_export(export_started.elapsed()); - if group.membership_epoch() != membership_epoch { - submit_snapshot_begin( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::SnapshotTopologyChanged, - Some("membership changed while snapshot began"), - None, - group_id, - ); - return; - } - let at_slot = exporter.metadata().at_slot; - let term_at_slot = group - .local_replica() - .accepted_at(at_slot) - .await - .map_or(0, |entry| entry.term); - let registration = registry.begin(group_id, membership_epoch, exporter, permit); - submit_snapshot_begin( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::Success, - None, - Some((®istration, term_at_slot)), - group_id, - ); - }); - } - - fn handle_snapshot_read(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle = req.conn_handle as usize; - let store = Arc::clone(&self.store); - let registry = Arc::clone(&self.snapshots); - let server = Arc::clone(server); - spawn_request(&self.rt, "snapshot_read", store.store_id, async move { - let Ok(request) = flatbuffers::root::(req.control()) else { - submit_snapshot_read( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::InvalidArgument, - Some("invalid snapshot Read request"), - SnapshotSessionId { - boot_nonce: 0, - session_number: 0, - }, - 0, - None, - ); - return; - }; - let id = SnapshotSessionId { - boot_nonce: request.boot_nonce(), - session_number: request.session_number(), - }; - let offset = request.offset(); - let registration = match registry.registration(id) { - Ok(value) => value, - Err(error) => { - let (code, message) = snapshot_error(&error); - submit_snapshot_read( - &server, - conn_handle, - req_id, - create_nano, - code, - Some(&message), - id, - offset, - None, - ); - return; - } - }; - if store.get_group(registration.group_id).map_or(true, |group| { - group.membership_epoch() != registration.membership_epoch - }) { - registry.expire(id); - submit_snapshot_read( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::SnapshotTopologyChanged, - Some("snapshot membership epoch changed"), - id, - offset, - None, - ); - return; - } - match registry.read(id, offset).await { - Ok(read) => submit_snapshot_read( - &server, - conn_handle, - req_id, - create_nano, - FBKvRetCode::Success, - None, - id, - offset, - Some(read), - ), - Err(error) => { - let (code, message) = snapshot_error(&error); - submit_snapshot_read( - &server, - conn_handle, - req_id, - create_nano, - code, - Some(&message), - id, - offset, - None, - ); - } - } - }); - } - - fn handle_snapshot_finish(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle = req.conn_handle as usize; - let store = Arc::clone(&self.store); - let registry = Arc::clone(&self.snapshots); - let server = Arc::clone(server); - spawn_request(&self.rt, "snapshot_finish", store.store_id, async move { - let Ok(request) = flatbuffers::root::(req.control()) else { - submit_snapshot_close( - &server, - conn_handle, - req_id, - create_nano, - FBMsgType::ESnapshotFinishResponse.0 as u16, - FBKvRetCode::InvalidArgument, - Some("invalid snapshot Finish request"), - SnapshotSessionId { - boot_nonce: 0, - session_number: 0, - }, - ); - return; - }; - let id = SnapshotSessionId { - boot_nonce: request.boot_nonce(), - session_number: request.session_number(), - }; - if let Ok(registration) = registry.registration(id) { - if store.get_group(registration.group_id).map_or(true, |group| { - group.membership_epoch() != registration.membership_epoch - }) { - registry.expire(id); - submit_snapshot_close( - &server, - conn_handle, - req_id, - create_nano, - FBMsgType::ESnapshotFinishResponse.0 as u16, - FBKvRetCode::SnapshotTopologyChanged, - Some("snapshot membership epoch changed"), - id, - ); - return; - } - } - match registry.finish(id, request.final_offset()).await { - Ok(()) => submit_snapshot_close( - &server, - conn_handle, - req_id, - create_nano, - FBMsgType::ESnapshotFinishResponse.0 as u16, - FBKvRetCode::Success, - None, - id, - ), - Err(error) => { - let (code, message) = snapshot_error(&error); - submit_snapshot_close( - &server, - conn_handle, - req_id, - create_nano, - FBMsgType::ESnapshotFinishResponse.0 as u16, - code, - Some(&message), - id, - ); - } - } - }); - } - - fn handle_snapshot_abort(&self, req: ServerRequest, server: &Arc) { - let req_id = req.request_id; - let create_nano = req.rpc_create_nano; - let conn_handle = req.conn_handle as usize; - let registry = Arc::clone(&self.snapshots); - let server = Arc::clone(server); - spawn_request(&self.rt, "snapshot_abort", self.store.store_id, async move { - let Ok(request) = flatbuffers::root::(req.control()) else { - submit_snapshot_close( - &server, - conn_handle, - req_id, - create_nano, - FBMsgType::ESnapshotAbortResponse.0 as u16, - FBKvRetCode::InvalidArgument, - Some("invalid snapshot Abort request"), - SnapshotSessionId { - boot_nonce: 0, - session_number: 0, - }, - ); - return; - }; - let id = SnapshotSessionId { - boot_nonce: request.boot_nonce(), - session_number: request.session_number(), - }; - registry.abort(id).await; - submit_snapshot_close( - &server, - conn_handle, - req_id, - create_nano, - FBMsgType::ESnapshotAbortResponse.0 as u16, - FBKvRetCode::Success, - None, - id, - ); - }); } } @@ -1506,559 +244,16 @@ impl Drop for PxRpcService { // ── Helper functions ───────────────────────────────────────────── +mod accept; +mod election; +mod notifications; #[allow(clippy::too_many_arguments)] -fn submit_snapshot_begin( - server: &RpcServer, - conn_handle: usize, - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error: Option<&str>, - success: Option<(&SnapshotRegistration, u64)>, - group_id: u64, -) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = error.map(|message| builder.create_string(message)); - let ( - boot_nonce, - session_number, - engine_format, - at_slot, - term_at_slot, - membership_epoch, - chunk_bytes, - total_bytes, - final_crc32c, - ) = success.map_or((0, 0, 0, 0, 0, 0, 0, 0, 0), |(registration, term)| { - ( - registration.id.boot_nonce, - registration.id.session_number, - registration.metadata.format as u8, - registration.metadata.at_slot, - term, - registration.membership_epoch, - u32::try_from(registration.metadata.chunk_bytes).unwrap_or(u32::MAX), - registration.metadata.total_bytes, - registration.metadata.final_crc32c, - ) - }); - let response = FBSnapshotBeginResponse::create( - &mut builder, - &FBSnapshotBeginResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - group_id, - boot_nonce, - session_number, - engine_format, - at_slot, - term_at_slot, - membership_epoch, - chunk_bytes, - total_bytes, - final_crc32c, - }, - ); - builder.finish(response, None); - submit_fb_response( - server, - conn_handle as *mut std::ffi::c_void, - builder.collapse(), - FBMsgType::ESnapshotBeginResponse.0 as u16, - req_id, - ); -} - -#[allow(clippy::too_many_arguments)] -fn submit_snapshot_read( - server: &RpcServer, - conn_handle: usize, - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error: Option<&str>, - id: SnapshotSessionId, - offset: u64, - read: Option, -) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = error.map(|message| builder.create_string(message)); - let payload_crc32c = read.as_ref().map_or(0, |value| value.payload_crc32c); - let done = read.as_ref().is_some_and(|value| value.chunk.done); - let response = FBSnapshotReadResponse::create( - &mut builder, - &FBSnapshotReadResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - boot_nonce: id.boot_nonce, - session_number: id.session_number, - offset, - payload_crc32c, - done, - }, - ); - builder.finish(response, None); - let control = builder.collapse(); - let data = read.map(|value| Buffer::from_vec_offset(value.chunk.bytes, 0)); - let control = Buffer::from_vec_offset(control.0, control.1); - unsafe { - let _ = server.submit_response_buffer( - conn_handle as *mut std::ffi::c_void, - control, - data, - FBMsgType::ESnapshotReadResponse.0 as u16, - req_id, - ); - } -} - -#[allow(clippy::too_many_arguments)] -fn submit_snapshot_close( - server: &RpcServer, - conn_handle: usize, - req_id: u64, - create_nano: u64, - msg_type: u16, - ret_code: FBKvRetCode, - error: Option<&str>, - id: SnapshotSessionId, -) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = error.map(|message| builder.create_string(message)); - if msg_type == FBMsgType::ESnapshotFinishResponse.0 as u16 { - let response = FBSnapshotFinishResponse::create( - &mut builder, - &FBSnapshotFinishResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - boot_nonce: id.boot_nonce, - session_number: id.session_number, - }, - ); - builder.finish(response, None); - } else { - let response = FBSnapshotAbortResponse::create( - &mut builder, - &FBSnapshotAbortResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - boot_nonce: id.boot_nonce, - session_number: id.session_number, - }, - ); - builder.finish(response, None); - } - submit_fb_response( - server, - conn_handle as *mut std::ffi::c_void, - builder.collapse(), - msg_type, - req_id, - ); -} - -fn snapshot_error(error: &SnapshotSessionError) -> (FBKvRetCode, String) { - match error { - SnapshotSessionError::NotFound => ( - FBKvRetCode::SnapshotNotFound, - "snapshot session not found".to_string(), - ), - SnapshotSessionError::Expired => ( - FBKvRetCode::SnapshotExpired, - "snapshot session expired".to_string(), - ), - SnapshotSessionError::Backpressure => ( - FBKvRetCode::SnapshotBackpressure, - "snapshot session capacity exhausted".to_string(), - ), - SnapshotSessionError::InvalidOffset { expected } => ( - FBKvRetCode::SnapshotInvalidOffset, - format!("invalid snapshot offset; expected {expected}"), - ), - SnapshotSessionError::Export(message) => (FBKvRetCode::Internal, message.clone()), - } -} - -fn px_error_to_ret_code(e: &PxReplicaError) -> (FBKvRetCode, String) { - match e { - PxReplicaError::GroupNotFound(_) => (FBKvRetCode::NotFound, e.to_string()), - PxReplicaError::ShuttingDown => (FBKvRetCode::Unavailable, e.to_string()), - PxReplicaError::Transport(_) | PxReplicaError::Timeout(_) | PxReplicaError::Backpressure(_) => { - (FBKvRetCode::Unavailable, e.to_string()) - } - PxReplicaError::Internal(_) => (FBKvRetCode::Internal, e.to_string()), - } -} - -fn submit_error( - server: &Arc, - conn_handle: *mut std::ffi::c_void, - req_id: u64, - create_nano: u64, - msg_type: u16, - code: FBKvRetCode, - msg: &str, -) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = builder.create_string(msg); - let args = FBPromiseResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code: code, - error_msg: Some(error_msg), - version: 0, - slot: 0, - round: 0, - leader_id: 0, - previously_accepted: None, - rejected: false, - rejected_round: 0, - rejected_leader_id: 0, - term: 0, - term_stale: false, - membership_epoch: 0, - epoch_mismatch: false, - }; - let resp = FBPromiseResponse::create(&mut builder, &args); - builder.finish(resp, None); - let ctrl = builder.collapse(); - submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); -} - -// ── Zero-copy response submit helper ────────────────────────────── - -/// Submit a response from a collapsed `FlatBufferBuilder` (zero-copy). -/// `ctrl` is the `(Vec, head)` tuple from `builder.collapse()` — the -/// finished flatbuffer data is at `ctrl.0[ctrl.1..]`. The Vec allocation -/// is wrapped as an external C++ Buffer (no copy); C++ uses it directly -/// for the `OutFrame` and frees it when the write completes. -fn submit_fb_response( - server: &RpcServer, - conn_handle: *mut std::ffi::c_void, - ctrl: (Vec, usize), - msg_type: u16, - req_id: u64, -) { - let buf = Buffer::from_vec_offset(ctrl.0, ctrl.1); - if buf.is_null_handle() { - // Empty control — submit with raw bytes path. - unsafe { - let _ = server.submit_response(conn_handle, &[], None, msg_type, req_id); - } - return; - } - unsafe { - let _ = server.submit_response_buffer(conn_handle, buf, None, msg_type, req_id); - } -} - -// ── Response builders ──────────────────────────────────────────── - -/// Build a `FBPromiseResponse` with a pre-existing `FBAcceptedValue` -/// offset (for the `Promised` reply with `previously_accepted`). -/// The caller must have already created the offset in `builder`. -#[allow(clippy::too_many_arguments)] -fn finish_promise_response( - mut builder: FlatBufferBuilder, - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error_msg: Option<&str>, - version: u32, - slot: u64, - round: u64, - leader_id: u64, - previously_accepted: Option>, - rejected: bool, - rejected_round: u64, - rejected_leader_id: u64, - term: u64, - term_stale: bool, - membership_epoch: u64, - epoch_mismatch: bool, -) -> (Vec, usize) { - let error_msg = error_msg.map(|m| builder.create_string(m)); - let args = FBPromiseResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - version, - slot, - round, - leader_id, - previously_accepted, - rejected, - rejected_round, - rejected_leader_id, - term, - term_stale, - membership_epoch, - epoch_mismatch, - }; - let resp = FBPromiseResponse::create(&mut builder, &args); - builder.finish(resp, None); - builder.collapse() -} - -#[allow(clippy::too_many_arguments)] -fn build_promise_response( - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error_msg: Option<&str>, - version: u32, - slot: u64, - round: u64, - leader_id: u64, - previously_accepted: Option>, - rejected: bool, - rejected_round: u64, - rejected_leader_id: u64, - term: u64, - term_stale: bool, - membership_epoch: u64, - epoch_mismatch: bool, -) -> (Vec, usize) { - let builder = FlatBufferBuilder::new(); - finish_promise_response( - builder, - req_id, - create_nano, - ret_code, - error_msg, - version, - slot, - round, - leader_id, - previously_accepted, - rejected, - rejected_round, - rejected_leader_id, - term, - term_stale, - membership_epoch, - epoch_mismatch, - ) -} - -#[allow(clippy::too_many_arguments)] -fn build_accepted_response( - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error_msg: Option<&str>, - version: u32, - slot: u64, - round: u64, - leader_id: u64, - rejected: bool, - rejected_round: u64, - rejected_leader_id: u64, - term: u64, - term_stale: bool, - membership_epoch: u64, - epoch_mismatch: bool, -) -> (Vec, usize) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = error_msg.map(|m| builder.create_string(m)); - let args = FBAcceptedResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - version, - slot, - round, - leader_id, - rejected, - rejected_round, - rejected_leader_id, - term, - term_stale, - membership_epoch, - epoch_mismatch, - }; - let resp = FBAcceptedResponse::create(&mut builder, &args); - builder.finish(resp, None); - builder.collapse() -} - -#[allow(clippy::too_many_arguments)] -fn build_pre_vote_response( - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error_msg: Option<&str>, - version: u32, - group_id: u64, - term: u64, - granted: bool, - contiguous_chosen: u64, - last_chosen_term: u64, - highest_seen_slot: u64, -) -> (Vec, usize) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = error_msg.map(|m| builder.create_string(m)); - let args = FBPreVoteResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - version, - group_id, - term, - granted, - contiguous_chosen, - last_chosen_term, - highest_seen_slot, - }; - let resp = FBPreVoteResponse::create(&mut builder, &args); - builder.finish(resp, None); - builder.collapse() -} - -#[allow(clippy::too_many_arguments)] -fn build_request_vote_response( - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error_msg: Option<&str>, - version: u32, - group_id: u64, - term: u64, - granted: bool, - contiguous_chosen: u64, - last_chosen_term: u64, - highest_seen_slot: u64, -) -> (Vec, usize) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = error_msg.map(|m| builder.create_string(m)); - let args = FBRequestVoteResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - version, - group_id, - term, - granted, - contiguous_chosen, - last_chosen_term, - highest_seen_slot, - }; - let resp = FBRequestVoteResponse::create(&mut builder, &args); - builder.finish(resp, None); - builder.collapse() -} - -#[allow(clippy::too_many_arguments)] -fn build_heartbeat_response( - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error_msg: Option<&str>, - version: u32, - group_id: u64, - term: u64, - success: bool, - contiguous_chosen: u64, - last_chosen_term: u64, - contiguous_applied: u64, - highest_seen_slot: u64, - durable_snapshot_slot: u64, -) -> (Vec, usize) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = error_msg.map(|m| builder.create_string(m)); - let args = FBHeartbeatResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - version, - group_id, - term, - success, - contiguous_chosen, - last_chosen_term, - contiguous_applied, - highest_seen_slot, - durable_snapshot_slot, - }; - let resp = FBHeartbeatResponse::create(&mut builder, &args); - builder.finish(resp, None); - builder.collapse() -} - -#[allow(clippy::too_many_arguments)] -fn build_step_down_response( - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error_msg: Option<&str>, - version: u32, - group_id: u64, - accepted: bool, - current_term: u64, - current_leader_id: u64, -) -> (Vec, usize) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = error_msg.map(|m| builder.create_string(m)); - let args = FBStepDownResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - version, - group_id, - accepted, - current_term, - current_leader_id, - }; - let resp = FBStepDownResponse::create(&mut builder, &args); - builder.finish(resp, None); - builder.collapse() -} - -#[allow(clippy::too_many_arguments)] -fn build_fetch_gap_response( - req_id: u64, - create_nano: u64, - ret_code: FBKvRetCode, - error_msg: Option<&str>, - version: u32, - group_id: u64, - slot: u64, - term: u64, - ballot_round: u64, - leader_id: u64, - payload: &[u8], -) -> (Vec, usize) { - let mut builder = FlatBufferBuilder::new(); - let error_msg = error_msg.map(|m| builder.create_string(m)); - let payload_vec = builder.create_vector(payload); - let args = FBFetchGapResponseArgs { - id: req_id, - rpc_create_nano: create_nano, - ret_code, - error_msg, - version, - group_id, - slot, - term, - ballot_round, - leader_id, - payload: Some(payload_vec), - }; - let resp = FBFetchGapResponse::create(&mut builder, &args); - builder.finish(resp, None); - builder.collapse() -} +mod prepare; +mod responses; +mod snapshots; +use responses::{ + build_accepted_response, build_fetch_gap_response, build_heartbeat_response, build_pre_vote_response, + build_promise_response, build_request_vote_response, build_step_down_response, finish_promise_response, + px_error_to_ret_code, snapshot_error, submit_error, submit_fb_response, submit_snapshot_begin, + submit_snapshot_close, submit_snapshot_read, +}; diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/accept.rs b/lib/crowdb-kv/src/rpc/px_rpc_service/accept.rs new file mode 100644 index 000000000..49a7e9750 --- /dev/null +++ b/lib/crowdb-kv/src/rpc/px_rpc_service/accept.rs @@ -0,0 +1,202 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + build_accepted_response, px_error_to_ret_code, record_group_context, spawn_request, submit_error, + submit_fb_response, warn, Arc, FBAcceptRequest, FBKvRetCode, FBMsgType, PxAcceptReply, PxBallot, + PxLocalReplica, PxLogEntry, PxRpcService, ReplicaHandler, RpcServer, ServerRequest, +}; + +impl PxRpcService { + // ── Accept ─────────────────────────────────────────────────── + + #[allow(clippy::too_many_lines)] + pub(super) fn handle_accept(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EAcceptedResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "accept", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let slot = fb_req.slot(); + let round = fb_req.round(); + let leader_id = fb_req.leader_id(); + let term = fb_req.term(); + let membership_epoch = fb_req.membership_epoch(); + + let Some(fb_value) = fb_req.value() else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "missing value", + ); + return; + }; + let payload_bytes: Vec = fb_value.payload().map(|v| v.bytes().to_vec()).unwrap_or_default(); + let entry = PxLogEntry { + slot, + ballot: PxBallot { round, leader_id }, + term, + payload: payload_bytes.into(), + }; + + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let responder_epoch = group.membership_epoch(); + + if membership_epoch != responder_epoch { + let converged_epoch = group.adopt_membership_epoch(membership_epoch); + warn!( + group_id, + slot, + round, + leader_id, + proposer_epoch = membership_epoch, + responder_epoch, + converged_epoch, + "accept rejected by membership-epoch fence; adopting higher epoch from proposer" + ); + let term = group.local_replica().current_term_snapshot(); + let ctrl = build_accepted_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + false, + 0, + 0, + term, + false, + responder_epoch, + true, + ); + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + return; + } + + let replica = group.local_replica(); + let reply = ::on_accept(replica, &entry, group_id).await; + let (rejected, rejected_round, rejected_leader_id, term_stale, reply_term) = match reply { + Ok(PxAcceptReply::Accepted { .. }) => (false, 0, 0, false, replica.current_term_snapshot()), + Ok(PxAcceptReply::Rejected { current_promised, .. }) => { + warn!( + group_id, + slot, + current_round = current_promised.round, + current_leader_id = current_promised.leader_id, + "accept rejected; next step: proposer should run prepare with a higher ballot" + ); + ( + true, + current_promised.round, + current_promised.leader_id, + false, + replica.current_term_snapshot(), + ) + } + Ok(PxAcceptReply::TermStale { new_term, .. }) => { + warn!( + group_id, + slot, new_term, "accept rejected by term fence; proposer should step down" + ); + (false, 0, 0, true, new_term) + } + Ok(PxAcceptReply::EpochMismatch { .. }) => { + unreachable!("on_accept does not produce EpochMismatch") + } + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + let ctrl = build_accepted_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + slot, + round, + leader_id, + false, + 0, + 0, + replica.current_term_snapshot(), + false, + responder_epoch, + false, + ); + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + return; + } + }; + let ctrl = build_accepted_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + rejected, + rejected_round, + rejected_leader_id, + reply_term, + term_stale, + responder_epoch, + false, + ); + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } +} diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/election.rs b/lib/crowdb-kv/src/rpc/px_rpc_service/election.rs new file mode 100644 index 000000000..2f22a699f --- /dev/null +++ b/lib/crowdb-kv/src/rpc/px_rpc_service/election.rs @@ -0,0 +1,349 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + build_heartbeat_response, build_pre_vote_response, build_request_vote_response, build_step_down_response, + px_error_to_ret_code, record_group_context, spawn_request, submit_error, submit_fb_response, Arc, + FBHeartbeatRequest, FBKvRetCode, FBMsgType, FBPreVoteRequest, FBRequestVoteRequest, FBStepDownRequest, + HeartbeatRequestPayload, PxLocalReplica, PxRpcService, ReplicaHandler, RpcServer, ServerRequest, + StepDownRequestPayload, VoteRequestPayload, +}; + +impl PxRpcService { + // ── PreVote ────────────────────────────────────────────────── + + pub(super) fn handle_pre_vote(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EPreVoteResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "pre_vote", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let payload = VoteRequestPayload { + term: fb_req.term(), + candidate_id: fb_req.candidate_id(), + accepted_log_tip_slot: fb_req.accepted_log_tip_slot(), + accepted_log_tip_term: fb_req.accepted_log_tip_term(), + }; + let replica = group.local_replica(); + let reply = ::on_pre_vote(replica, payload, group_id).await; + let ctrl = match reply { + Ok(r) => build_pre_vote_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + r.term, + r.granted, + r.contiguous_chosen, + r.last_chosen_term, + r.highest_seen_slot, + ), + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_pre_vote_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + group_id, + 0, + false, + 0, + 0, + 0, + ) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } + + // ── RequestVote ────────────────────────────────────────────── + + pub(super) fn handle_request_vote(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::ERequestVoteResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "request_vote", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let payload = VoteRequestPayload { + term: fb_req.term(), + candidate_id: fb_req.candidate_id(), + accepted_log_tip_slot: fb_req.accepted_log_tip_slot(), + accepted_log_tip_term: fb_req.accepted_log_tip_term(), + }; + let replica = group.local_replica(); + let reply = ::on_request_vote(replica, payload, group_id).await; + let ctrl = match reply { + Ok(r) => build_request_vote_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + r.term, + r.granted, + r.contiguous_chosen, + r.last_chosen_term, + r.highest_seen_slot, + ), + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_request_vote_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + group_id, + 0, + false, + 0, + 0, + 0, + ) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } + + // ── Heartbeat ──────────────────────────────────────────────── + + pub(super) fn handle_heartbeat(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EHeartbeatResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "heartbeat", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let payload = HeartbeatRequestPayload { + term: fb_req.term(), + leader_id: fb_req.leader_id(), + prev_log_slot: fb_req.prev_log_slot(), + prev_log_term: fb_req.prev_log_term(), + committed_safe_slot: fb_req.committed_safe_slot(), + lease_grant_until_ms_mono: fb_req.lease_grant_until_ms_mono(), + t_send_ms_mono: fb_req.t_send_ms_mono(), + }; + let replica = group.local_replica(); + let reply = ::on_heartbeat(replica, payload, group_id).await; + let ctrl = match reply { + Ok(r) => build_heartbeat_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + r.term, + r.success, + r.contiguous_chosen, + r.last_chosen_term, + r.contiguous_applied, + r.highest_seen_slot, + r.durable_snapshot_slot, + ), + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_heartbeat_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + group_id, + 0, + false, + 0, + 0, + 0, + 0, + 0, + ) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } + + // ── StepDown ───────────────────────────────────────────────── + + pub(super) fn handle_step_down(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EStepDownResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "step_down", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let reason = fb_req.reason().unwrap_or("").to_string(); + let payload = StepDownRequestPayload { + term: fb_req.term(), + target_leader_id: fb_req.target_leader_id(), + reason, + }; + let replica = group.local_replica(); + let reply = ::on_step_down(replica, &payload, group_id).await; + let ctrl = match reply { + Ok(r) => build_step_down_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + r.accepted, + r.current_term, + r.current_leader_id, + ), + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_step_down_response(req_id, create_nano, code, Some(&msg), 1, group_id, false, 0, 0) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } +} diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/notifications.rs b/lib/crowdb-kv/src/rpc/px_rpc_service/notifications.rs new file mode 100644 index 000000000..f64dee916 --- /dev/null +++ b/lib/crowdb-kv/src/rpc/px_rpc_service/notifications.rs @@ -0,0 +1,202 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + build_fetch_gap_response, debug, record_group_context, request_span, spawn_group_request, submit_error, + submit_fb_response, Arc, FBBatchChosenNotification, FBChosenNotification, FBFetchGapRequest, FBKvRetCode, + FBMsgType, PxBallot, PxRpcService, RpcServer, ServerRequest, +}; + +impl PxRpcService { + // ── ChosenNotification (fire-and-forget) ───────────────────── + + #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] + pub(super) fn handle_chosen_notice(&self, req: ServerRequest, _server: &Arc) { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + debug!("chosen notice: invalid flatbuffer"); + return; + }; + let group_id = fb_req.group_id(); + let slot = fb_req.slot(); + let term = fb_req.term(); + let leader_id = fb_req.leader_id(); + let ballot_round = fb_req.ballot_round(); + + let Some(group) = self.store.get_group(group_id) else { + debug!(slot, term, "chosen notice dropped (group not found)"); + return; + }; + let chosen_ballot = PxBallot { + round: ballot_round, + leader_id, + }; + spawn_group_request( + &self.rt, + "chosen_notice", + self.store.store_id, + group_id, + group.local_replica().id, + async move { + let replica = group.local_replica(); + let accepted = replica.accepted_at(slot).await; + let ballot_matches = accepted.as_ref().is_some_and(|e| e.ballot == chosen_ballot); + if ballot_matches { + replica.learner.update_chosen_frontier(slot, term); + replica.wake_apply_loop(); + debug!( + slot, + term, leader_id, ballot_round, "chosen notification applied (ballot match)" + ); + } else { + replica.note_chosen(slot, term); + replica.record_gap(slot); + if let Some(ref entry) = accepted { + replica.incr_chosen_notice_stale(); + debug!( + slot, + term, + leader_id, + chosen_ballot_round = ballot_round, + accepted_ballot_round = entry.ballot.round, + accepted_ballot_leader = entry.ballot.leader_id, + "chosen notification stale ballot (gap recorded)" + ); + } else { + replica.incr_chosen_notice_missing(); + debug!( + slot, + term, leader_id, "chosen notification missing value (gap recorded)" + ); + } + } + }, + ); + } + + // ── BatchChosenNotification (fire-and-forget) ──────────────── + + #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] + pub(super) fn handle_batch_chosen(&self, req: ServerRequest, _server: &Arc) { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + debug!("batch chosen: invalid flatbuffer"); + return; + }; + let group_id = fb_req.group_id(); + let start_slot = fb_req.start_slot(); + let end_slot = fb_req.end_slot(); + let term = fb_req.term(); + let leader_id = fb_req.leader_id(); + let ballot_round = fb_req.ballot_round(); + + let Some(group) = self.store.get_group(group_id) else { + debug!("batch chosen dropped (group not found)"); + return; + }; + let chosen_ballot = PxBallot { + round: ballot_round, + leader_id, + }; + spawn_group_request( + &self.rt, + "batch_chosen", + self.store.store_id, + group_id, + group.local_replica().id, + async move { + let replica = group.local_replica(); + let mut advanced_count = 0u64; + for slot in start_slot..=end_slot { + let accepted = replica.accepted_at(slot).await; + if accepted.as_ref().is_some_and(|e| e.ballot == chosen_ballot) { + replica.learner.update_chosen_frontier(slot, term); + advanced_count += 1; + } else { + replica.note_chosen(slot, term); + replica.record_gap(slot); + } + } + replica.wake_apply_loop(); + debug!( + group_id, + start_slot, + end_slot, + term, + leader_id, + ballot_round, + advanced_count, + "batch chosen notification applied" + ); + }, + ); + } + + // ── FetchGap ───────────────────────────────────────────────── + + #[allow(clippy::needless_pass_by_value, reason = "make_handler uniform signature")] + pub(super) fn handle_fetch_gap(&self, req: ServerRequest, server: &Arc) { + let span = request_span("fetch_gap", self.store.store_id); + let _entered = span.enter(); + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle = req.conn_handle; + let msg_type = FBMsgType::EFetchGapResponse.0 as u16; + + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + server, + conn_handle, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&self.store, group_id); + let slot = fb_req.slot(); + + let Some(group) = self.store.get_group(group_id) else { + submit_error( + server, + conn_handle, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + if let Some(resp) = group.handle_fetch_gap(slot) { + let payload_vec = resp.payload.clone(); + let ctrl = build_fetch_gap_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + group_id, + resp.slot, + resp.term, + resp.ballot_round, + resp.leader_id, + &payload_vec, + ); + submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); + } else { + debug!(slot, "fetch_gap no value (not yet chosen)"); + submit_error( + server, + conn_handle, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "slot not yet chosen", + ); + } + // req dropped here, frame released + } +} diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/prepare.rs b/lib/crowdb-kv/src/rpc/px_rpc_service/prepare.rs new file mode 100644 index 000000000..9ccd6c504 --- /dev/null +++ b/lib/crowdb-kv/src/rpc/px_rpc_service/prepare.rs @@ -0,0 +1,224 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + build_promise_response, finish_promise_response, px_error_to_ret_code, record_group_context, + spawn_request, submit_error, submit_fb_response, warn, Arc, FBAcceptedValue, FBAcceptedValueArgs, + FBKvRetCode, FBMsgType, FBPrepareRequest, FlatBufferBuilder, PxBallot, PxLocalReplica, PxPrepareReply, + PxRpcService, ReplicaHandler, RpcServer, ServerRequest, +}; + +impl PxRpcService { + // ── Prepare ────────────────────────────────────────────────── + + #[allow(clippy::too_many_lines)] + pub(super) fn handle_prepare(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle_usize = req.conn_handle as usize; + let msg_type = FBMsgType::EPromiseResponse.0 as u16; + let store = Arc::clone(&self.store); + let server = Arc::clone(server); + spawn_request(&self.rt, "prepare", store.store_id, async move { + let Ok(fb_req) = flatbuffers::root::(req.control()) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::InvalidArgument, + "invalid request flatbuffer", + ); + return; + }; + let group_id = fb_req.group_id(); + record_group_context(&store, group_id); + let slot = fb_req.slot(); + let round = fb_req.round(); + let leader_id = fb_req.leader_id(); + let term = fb_req.term(); + let membership_epoch = fb_req.membership_epoch(); + + let Some(group) = store.get_group(group_id) else { + submit_error( + &server, + conn_handle_usize as *mut std::ffi::c_void, + req_id, + create_nano, + msg_type, + FBKvRetCode::NotFound, + "px group not found", + ); + return; + }; + let responder_epoch = group.membership_epoch(); + + // Membership-epoch fence (same as px_service.rs L131-147). + if membership_epoch != responder_epoch { + let converged_epoch = group.adopt_membership_epoch(membership_epoch); + warn!( + group_id, + slot, + round, + leader_id, + proposer_epoch = membership_epoch, + responder_epoch, + converged_epoch, + "prepare rejected by membership-epoch fence; adopting higher epoch from proposer" + ); + let term = group.local_replica().current_term_snapshot(); + let ctrl = build_promise_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + None, + false, + 0, + 0, + term, + false, + responder_epoch, + true, + ); + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + return; + } + + let ballot = PxBallot { round, leader_id }; + let replica = group.local_replica(); + let reply = + ::on_prepare(replica, slot, ballot, term, group_id).await; + let ctrl = match reply { + Ok(PxPrepareReply::Promised { slot, accepted }) => { + let mut builder = FlatBufferBuilder::new(); + let prev_accepted_off = accepted.as_ref().map(|entry| { + let payload = builder.create_vector(&entry.payload); + FBAcceptedValue::create( + &mut builder, + &FBAcceptedValueArgs { + slot: entry.slot, + round: entry.ballot.round, + leader_id: entry.ballot.leader_id, + term: entry.term, + payload: Some(payload), + }, + ) + }); + let term = replica.current_term_snapshot(); + finish_promise_response( + builder, + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + prev_accepted_off, + false, + 0, + 0, + term, + false, + responder_epoch, + false, + ) + } + Ok(PxPrepareReply::TermStale { slot, new_term }) => build_promise_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + None, + false, + 0, + 0, + new_term, + true, + responder_epoch, + false, + ), + Ok(PxPrepareReply::Rejected { + slot, + current_promised, + }) => { + warn!( + group_id, + slot, + current_round = current_promised.round, + current_leader_id = current_promised.leader_id, + "prepare rejected; next step: proposer should retry with a higher ballot" + ); + let term = replica.current_term_snapshot(); + build_promise_response( + req_id, + create_nano, + FBKvRetCode::Success, + None, + 1, + slot, + round, + leader_id, + None, + true, + current_promised.round, + current_promised.leader_id, + term, + false, + responder_epoch, + false, + ) + } + Ok(PxPrepareReply::EpochMismatch { .. }) => { + unreachable!("on_prepare does not produce EpochMismatch") + } + Err(e) => { + let (code, msg) = px_error_to_ret_code(&e); + build_promise_response( + req_id, + create_nano, + code, + Some(&msg), + 1, + slot, + round, + leader_id, + None, + false, + 0, + 0, + replica.current_term_snapshot(), + false, + responder_epoch, + false, + ) + } + }; + submit_fb_response( + &server, + conn_handle_usize as *mut std::ffi::c_void, + ctrl, + msg_type, + req_id, + ); + // req dropped here, frame released + }); + } +} diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/responses.rs b/lib/crowdb-kv/src/rpc/px_rpc_service/responses.rs new file mode 100644 index 000000000..eb4b366bb --- /dev/null +++ b/lib/crowdb-kv/src/rpc/px_rpc_service/responses.rs @@ -0,0 +1,570 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + Arc, Buffer, FBAcceptedResponse, FBAcceptedResponseArgs, FBAcceptedValue, FBFetchGapResponse, + FBFetchGapResponseArgs, FBHeartbeatResponse, FBHeartbeatResponseArgs, FBKvRetCode, FBMsgType, + FBPreVoteResponse, FBPreVoteResponseArgs, FBPromiseResponse, FBPromiseResponseArgs, + FBRequestVoteResponse, FBRequestVoteResponseArgs, FBSnapshotAbortResponse, FBSnapshotAbortResponseArgs, + FBSnapshotBeginResponse, FBSnapshotBeginResponseArgs, FBSnapshotFinishResponse, + FBSnapshotFinishResponseArgs, FBSnapshotReadResponse, FBSnapshotReadResponseArgs, FBStepDownResponse, + FBStepDownResponseArgs, FlatBufferBuilder, PxReplicaError, RpcServer, SnapshotRegistration, + SnapshotSessionError, SnapshotSessionId, +}; + +#[allow(clippy::too_many_arguments)] +pub(super) fn submit_snapshot_begin( + server: &RpcServer, + conn_handle: usize, + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error: Option<&str>, + success: Option<(&SnapshotRegistration, u64)>, + group_id: u64, +) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = error.map(|message| builder.create_string(message)); + let ( + boot_nonce, + session_number, + engine_format, + at_slot, + term_at_slot, + membership_epoch, + chunk_bytes, + total_bytes, + final_crc32c, + ) = success.map_or((0, 0, 0, 0, 0, 0, 0, 0, 0), |(registration, term)| { + ( + registration.id.boot_nonce, + registration.id.session_number, + registration.metadata.format as u8, + registration.metadata.at_slot, + term, + registration.membership_epoch, + u32::try_from(registration.metadata.chunk_bytes).unwrap_or(u32::MAX), + registration.metadata.total_bytes, + registration.metadata.final_crc32c, + ) + }); + let response = FBSnapshotBeginResponse::create( + &mut builder, + &FBSnapshotBeginResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + group_id, + boot_nonce, + session_number, + engine_format, + at_slot, + term_at_slot, + membership_epoch, + chunk_bytes, + total_bytes, + final_crc32c, + }, + ); + builder.finish(response, None); + submit_fb_response( + server, + conn_handle as *mut std::ffi::c_void, + builder.collapse(), + FBMsgType::ESnapshotBeginResponse.0 as u16, + req_id, + ); +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn submit_snapshot_read( + server: &RpcServer, + conn_handle: usize, + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error: Option<&str>, + id: SnapshotSessionId, + offset: u64, + read: Option, +) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = error.map(|message| builder.create_string(message)); + let payload_crc32c = read.as_ref().map_or(0, |value| value.payload_crc32c); + let done = read.as_ref().is_some_and(|value| value.chunk.done); + let response = FBSnapshotReadResponse::create( + &mut builder, + &FBSnapshotReadResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + boot_nonce: id.boot_nonce, + session_number: id.session_number, + offset, + payload_crc32c, + done, + }, + ); + builder.finish(response, None); + let control = builder.collapse(); + let data = read.map(|value| Buffer::from_vec_offset(value.chunk.bytes, 0)); + let control = Buffer::from_vec_offset(control.0, control.1); + unsafe { + let _ = server.submit_response_buffer( + conn_handle as *mut std::ffi::c_void, + control, + data, + FBMsgType::ESnapshotReadResponse.0 as u16, + req_id, + ); + } +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn submit_snapshot_close( + server: &RpcServer, + conn_handle: usize, + req_id: u64, + create_nano: u64, + msg_type: u16, + ret_code: FBKvRetCode, + error: Option<&str>, + id: SnapshotSessionId, +) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = error.map(|message| builder.create_string(message)); + if msg_type == FBMsgType::ESnapshotFinishResponse.0 as u16 { + let response = FBSnapshotFinishResponse::create( + &mut builder, + &FBSnapshotFinishResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + boot_nonce: id.boot_nonce, + session_number: id.session_number, + }, + ); + builder.finish(response, None); + } else { + let response = FBSnapshotAbortResponse::create( + &mut builder, + &FBSnapshotAbortResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + boot_nonce: id.boot_nonce, + session_number: id.session_number, + }, + ); + builder.finish(response, None); + } + submit_fb_response( + server, + conn_handle as *mut std::ffi::c_void, + builder.collapse(), + msg_type, + req_id, + ); +} + +pub(super) fn snapshot_error(error: &SnapshotSessionError) -> (FBKvRetCode, String) { + match error { + SnapshotSessionError::NotFound => ( + FBKvRetCode::SnapshotNotFound, + "snapshot session not found".to_string(), + ), + SnapshotSessionError::Expired => ( + FBKvRetCode::SnapshotExpired, + "snapshot session expired".to_string(), + ), + SnapshotSessionError::Backpressure => ( + FBKvRetCode::SnapshotBackpressure, + "snapshot session capacity exhausted".to_string(), + ), + SnapshotSessionError::InvalidOffset { expected } => ( + FBKvRetCode::SnapshotInvalidOffset, + format!("invalid snapshot offset; expected {expected}"), + ), + SnapshotSessionError::Export(message) => (FBKvRetCode::Internal, message.clone()), + } +} + +pub(super) fn px_error_to_ret_code(e: &PxReplicaError) -> (FBKvRetCode, String) { + match e { + PxReplicaError::GroupNotFound(_) => (FBKvRetCode::NotFound, e.to_string()), + PxReplicaError::ShuttingDown => (FBKvRetCode::Unavailable, e.to_string()), + PxReplicaError::Transport(_) | PxReplicaError::Timeout(_) | PxReplicaError::Backpressure(_) => { + (FBKvRetCode::Unavailable, e.to_string()) + } + PxReplicaError::Internal(_) => (FBKvRetCode::Internal, e.to_string()), + } +} + +pub(super) fn submit_error( + server: &Arc, + conn_handle: *mut std::ffi::c_void, + req_id: u64, + create_nano: u64, + msg_type: u16, + code: FBKvRetCode, + msg: &str, +) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = builder.create_string(msg); + let args = FBPromiseResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code: code, + error_msg: Some(error_msg), + version: 0, + slot: 0, + round: 0, + leader_id: 0, + previously_accepted: None, + rejected: false, + rejected_round: 0, + rejected_leader_id: 0, + term: 0, + term_stale: false, + membership_epoch: 0, + epoch_mismatch: false, + }; + let resp = FBPromiseResponse::create(&mut builder, &args); + builder.finish(resp, None); + let ctrl = builder.collapse(); + submit_fb_response(server, conn_handle, ctrl, msg_type, req_id); +} + +// ── Zero-copy response submit helper ────────────────────────────── + +/// Submit a response from a collapsed `FlatBufferBuilder` (zero-copy). +/// `ctrl` is the `(Vec, head)` tuple from `builder.collapse()` — the +/// finished flatbuffer data is at `ctrl.0[ctrl.1..]`. The Vec allocation +/// is wrapped as an external C++ Buffer (no copy); C++ uses it directly +/// for the `OutFrame` and frees it when the write completes. +pub(super) fn submit_fb_response( + server: &RpcServer, + conn_handle: *mut std::ffi::c_void, + ctrl: (Vec, usize), + msg_type: u16, + req_id: u64, +) { + let buf = Buffer::from_vec_offset(ctrl.0, ctrl.1); + if buf.is_null_handle() { + // Empty control — submit with raw bytes path. + unsafe { + let _ = server.submit_response(conn_handle, &[], None, msg_type, req_id); + } + return; + } + unsafe { + let _ = server.submit_response_buffer(conn_handle, buf, None, msg_type, req_id); + } +} + +// ── Response builders ──────────────────────────────────────────── + +/// Build a `FBPromiseResponse` with a pre-existing `FBAcceptedValue` +/// offset (for the `Promised` reply with `previously_accepted`). +/// The caller must have already created the offset in `builder`. +#[allow(clippy::too_many_arguments)] +pub(super) fn finish_promise_response( + mut builder: FlatBufferBuilder, + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error_msg: Option<&str>, + version: u32, + slot: u64, + round: u64, + leader_id: u64, + previously_accepted: Option>, + rejected: bool, + rejected_round: u64, + rejected_leader_id: u64, + term: u64, + term_stale: bool, + membership_epoch: u64, + epoch_mismatch: bool, +) -> (Vec, usize) { + let error_msg = error_msg.map(|m| builder.create_string(m)); + let args = FBPromiseResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + version, + slot, + round, + leader_id, + previously_accepted, + rejected, + rejected_round, + rejected_leader_id, + term, + term_stale, + membership_epoch, + epoch_mismatch, + }; + let resp = FBPromiseResponse::create(&mut builder, &args); + builder.finish(resp, None); + builder.collapse() +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn build_promise_response( + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error_msg: Option<&str>, + version: u32, + slot: u64, + round: u64, + leader_id: u64, + previously_accepted: Option>, + rejected: bool, + rejected_round: u64, + rejected_leader_id: u64, + term: u64, + term_stale: bool, + membership_epoch: u64, + epoch_mismatch: bool, +) -> (Vec, usize) { + let builder = FlatBufferBuilder::new(); + finish_promise_response( + builder, + req_id, + create_nano, + ret_code, + error_msg, + version, + slot, + round, + leader_id, + previously_accepted, + rejected, + rejected_round, + rejected_leader_id, + term, + term_stale, + membership_epoch, + epoch_mismatch, + ) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn build_accepted_response( + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error_msg: Option<&str>, + version: u32, + slot: u64, + round: u64, + leader_id: u64, + rejected: bool, + rejected_round: u64, + rejected_leader_id: u64, + term: u64, + term_stale: bool, + membership_epoch: u64, + epoch_mismatch: bool, +) -> (Vec, usize) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = error_msg.map(|m| builder.create_string(m)); + let args = FBAcceptedResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + version, + slot, + round, + leader_id, + rejected, + rejected_round, + rejected_leader_id, + term, + term_stale, + membership_epoch, + epoch_mismatch, + }; + let resp = FBAcceptedResponse::create(&mut builder, &args); + builder.finish(resp, None); + builder.collapse() +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn build_pre_vote_response( + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error_msg: Option<&str>, + version: u32, + group_id: u64, + term: u64, + granted: bool, + contiguous_chosen: u64, + last_chosen_term: u64, + highest_seen_slot: u64, +) -> (Vec, usize) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = error_msg.map(|m| builder.create_string(m)); + let args = FBPreVoteResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + version, + group_id, + term, + granted, + contiguous_chosen, + last_chosen_term, + highest_seen_slot, + }; + let resp = FBPreVoteResponse::create(&mut builder, &args); + builder.finish(resp, None); + builder.collapse() +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn build_request_vote_response( + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error_msg: Option<&str>, + version: u32, + group_id: u64, + term: u64, + granted: bool, + contiguous_chosen: u64, + last_chosen_term: u64, + highest_seen_slot: u64, +) -> (Vec, usize) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = error_msg.map(|m| builder.create_string(m)); + let args = FBRequestVoteResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + version, + group_id, + term, + granted, + contiguous_chosen, + last_chosen_term, + highest_seen_slot, + }; + let resp = FBRequestVoteResponse::create(&mut builder, &args); + builder.finish(resp, None); + builder.collapse() +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn build_heartbeat_response( + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error_msg: Option<&str>, + version: u32, + group_id: u64, + term: u64, + success: bool, + contiguous_chosen: u64, + last_chosen_term: u64, + contiguous_applied: u64, + highest_seen_slot: u64, + durable_snapshot_slot: u64, +) -> (Vec, usize) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = error_msg.map(|m| builder.create_string(m)); + let args = FBHeartbeatResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + version, + group_id, + term, + success, + contiguous_chosen, + last_chosen_term, + contiguous_applied, + highest_seen_slot, + durable_snapshot_slot, + }; + let resp = FBHeartbeatResponse::create(&mut builder, &args); + builder.finish(resp, None); + builder.collapse() +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn build_step_down_response( + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error_msg: Option<&str>, + version: u32, + group_id: u64, + accepted: bool, + current_term: u64, + current_leader_id: u64, +) -> (Vec, usize) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = error_msg.map(|m| builder.create_string(m)); + let args = FBStepDownResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + version, + group_id, + accepted, + current_term, + current_leader_id, + }; + let resp = FBStepDownResponse::create(&mut builder, &args); + builder.finish(resp, None); + builder.collapse() +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn build_fetch_gap_response( + req_id: u64, + create_nano: u64, + ret_code: FBKvRetCode, + error_msg: Option<&str>, + version: u32, + group_id: u64, + slot: u64, + term: u64, + ballot_round: u64, + leader_id: u64, + payload: &[u8], +) -> (Vec, usize) { + let mut builder = FlatBufferBuilder::new(); + let error_msg = error_msg.map(|m| builder.create_string(m)); + let payload_vec = builder.create_vector(payload); + let args = FBFetchGapResponseArgs { + id: req_id, + rpc_create_nano: create_nano, + ret_code, + error_msg, + version, + group_id, + slot, + term, + ballot_round, + leader_id, + payload: Some(payload_vec), + }; + let resp = FBFetchGapResponse::create(&mut builder, &args); + builder.finish(resp, None); + builder.collapse() +} diff --git a/lib/crowdb-kv/src/rpc/px_rpc_service/snapshots.rs b/lib/crowdb-kv/src/rpc/px_rpc_service/snapshots.rs new file mode 100644 index 000000000..14d42e839 --- /dev/null +++ b/lib/crowdb-kv/src/rpc/px_rpc_service/snapshots.rs @@ -0,0 +1,380 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + record_group_context, snapshot_error, spawn_request, submit_snapshot_begin, submit_snapshot_close, + submit_snapshot_read, Arc, FBKvRetCode, FBMsgType, FBSnapshotAbortRequest, FBSnapshotBeginRequest, + FBSnapshotFinishRequest, FBSnapshotReadRequest, PxRpcService, RpcServer, ServerRequest, + SnapshotSessionId, +}; + +impl PxRpcService { + // ── Snapshot streaming ─────────────────────────────────────── + + #[allow(clippy::too_many_lines)] + pub(super) fn handle_snapshot_begin(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle = req.conn_handle as usize; + let store = Arc::clone(&self.store); + let registry = Arc::clone(&self.snapshots); + let server = Arc::clone(server); + spawn_request(&self.rt, "snapshot_begin", store.store_id, async move { + let Ok(request) = flatbuffers::root::(req.control()) else { + submit_snapshot_begin( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::InvalidArgument, + Some("invalid snapshot Begin request"), + None, + 0, + ); + return; + }; + let group_id = request.group_id(); + record_group_context(&store, group_id); + let requested = request.max_chunk_bytes() as usize; + if requested == 0 { + submit_snapshot_begin( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::InvalidArgument, + Some("max_chunk_bytes must be nonzero"), + None, + group_id, + ); + return; + } + let Some(group) = store.get_group(group_id) else { + submit_snapshot_begin( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::NotFound, + Some("px group not found"), + None, + group_id, + ); + return; + }; + let membership_epoch = group.membership_epoch(); + let chunk_bytes = requested.min(store.snapshot_chunk_bytes).min(1024 * 1024); + if chunk_bytes == 0 { + submit_snapshot_begin( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::Internal, + Some("snapshot source chunk size is not configured"), + None, + group_id, + ); + return; + } + let permit = match registry.reserve() { + Ok(permit) => permit, + Err(error) => { + let (code, message) = snapshot_error(&error); + submit_snapshot_begin( + &server, + conn_handle, + req_id, + create_nano, + code, + Some(&message), + None, + group_id, + ); + return; + } + }; + let export_group = Arc::clone(&group); + let export_started = std::time::Instant::now(); + let exporter = match tokio::task::spawn_blocking(move || { + export_group + .local_replica() + .learner + .engine() + .snapshot_export_begin(chunk_bytes) + }) + .await + { + Ok(Ok(exporter)) => exporter, + Ok(Err(error)) => { + submit_snapshot_begin( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::Internal, + Some(&error), + None, + group_id, + ); + return; + } + Err(error) => { + let message = format!("snapshot export task failed: {error}"); + submit_snapshot_begin( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::Internal, + Some(&message), + None, + group_id, + ); + return; + } + }; + registry.observe_export(export_started.elapsed()); + if group.membership_epoch() != membership_epoch { + submit_snapshot_begin( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::SnapshotTopologyChanged, + Some("membership changed while snapshot began"), + None, + group_id, + ); + return; + } + let at_slot = exporter.metadata().at_slot; + let term_at_slot = group + .local_replica() + .accepted_at(at_slot) + .await + .map_or(0, |entry| entry.term); + let registration = registry.begin(group_id, membership_epoch, exporter, permit); + submit_snapshot_begin( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::Success, + None, + Some((®istration, term_at_slot)), + group_id, + ); + }); + } + + pub(super) fn handle_snapshot_read(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle = req.conn_handle as usize; + let store = Arc::clone(&self.store); + let registry = Arc::clone(&self.snapshots); + let server = Arc::clone(server); + spawn_request(&self.rt, "snapshot_read", store.store_id, async move { + let Ok(request) = flatbuffers::root::(req.control()) else { + submit_snapshot_read( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::InvalidArgument, + Some("invalid snapshot Read request"), + SnapshotSessionId { + boot_nonce: 0, + session_number: 0, + }, + 0, + None, + ); + return; + }; + let id = SnapshotSessionId { + boot_nonce: request.boot_nonce(), + session_number: request.session_number(), + }; + let offset = request.offset(); + let registration = match registry.registration(id) { + Ok(value) => value, + Err(error) => { + let (code, message) = snapshot_error(&error); + submit_snapshot_read( + &server, + conn_handle, + req_id, + create_nano, + code, + Some(&message), + id, + offset, + None, + ); + return; + } + }; + if store.get_group(registration.group_id).map_or(true, |group| { + group.membership_epoch() != registration.membership_epoch + }) { + registry.expire(id); + submit_snapshot_read( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::SnapshotTopologyChanged, + Some("snapshot membership epoch changed"), + id, + offset, + None, + ); + return; + } + match registry.read(id, offset).await { + Ok(read) => submit_snapshot_read( + &server, + conn_handle, + req_id, + create_nano, + FBKvRetCode::Success, + None, + id, + offset, + Some(read), + ), + Err(error) => { + let (code, message) = snapshot_error(&error); + submit_snapshot_read( + &server, + conn_handle, + req_id, + create_nano, + code, + Some(&message), + id, + offset, + None, + ); + } + } + }); + } + + pub(super) fn handle_snapshot_finish(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle = req.conn_handle as usize; + let store = Arc::clone(&self.store); + let registry = Arc::clone(&self.snapshots); + let server = Arc::clone(server); + spawn_request(&self.rt, "snapshot_finish", store.store_id, async move { + let Ok(request) = flatbuffers::root::(req.control()) else { + submit_snapshot_close( + &server, + conn_handle, + req_id, + create_nano, + FBMsgType::ESnapshotFinishResponse.0 as u16, + FBKvRetCode::InvalidArgument, + Some("invalid snapshot Finish request"), + SnapshotSessionId { + boot_nonce: 0, + session_number: 0, + }, + ); + return; + }; + let id = SnapshotSessionId { + boot_nonce: request.boot_nonce(), + session_number: request.session_number(), + }; + if let Ok(registration) = registry.registration(id) { + if store.get_group(registration.group_id).map_or(true, |group| { + group.membership_epoch() != registration.membership_epoch + }) { + registry.expire(id); + submit_snapshot_close( + &server, + conn_handle, + req_id, + create_nano, + FBMsgType::ESnapshotFinishResponse.0 as u16, + FBKvRetCode::SnapshotTopologyChanged, + Some("snapshot membership epoch changed"), + id, + ); + return; + } + } + match registry.finish(id, request.final_offset()).await { + Ok(()) => submit_snapshot_close( + &server, + conn_handle, + req_id, + create_nano, + FBMsgType::ESnapshotFinishResponse.0 as u16, + FBKvRetCode::Success, + None, + id, + ), + Err(error) => { + let (code, message) = snapshot_error(&error); + submit_snapshot_close( + &server, + conn_handle, + req_id, + create_nano, + FBMsgType::ESnapshotFinishResponse.0 as u16, + code, + Some(&message), + id, + ); + } + } + }); + } + + pub(super) fn handle_snapshot_abort(&self, req: ServerRequest, server: &Arc) { + let req_id = req.request_id; + let create_nano = req.rpc_create_nano; + let conn_handle = req.conn_handle as usize; + let registry = Arc::clone(&self.snapshots); + let server = Arc::clone(server); + spawn_request(&self.rt, "snapshot_abort", self.store.store_id, async move { + let Ok(request) = flatbuffers::root::(req.control()) else { + submit_snapshot_close( + &server, + conn_handle, + req_id, + create_nano, + FBMsgType::ESnapshotAbortResponse.0 as u16, + FBKvRetCode::InvalidArgument, + Some("invalid snapshot Abort request"), + SnapshotSessionId { + boot_nonce: 0, + session_number: 0, + }, + ); + return; + }; + let id = SnapshotSessionId { + boot_nonce: request.boot_nonce(), + session_number: request.session_number(), + }; + registry.abort(id).await; + submit_snapshot_close( + &server, + conn_handle, + req_id, + create_nano, + FBMsgType::ESnapshotAbortResponse.0 as u16, + FBKvRetCode::Success, + None, + id, + ); + }); + } +} diff --git a/lib/crowdb-kv/src/rpc/px_rpc_transport.rs b/lib/crowdb-kv/src/rpc/px_rpc_transport.rs index 7e07a3b1c..1a74e93b6 100644 --- a/lib/crowdb-kv/src/rpc/px_rpc_transport.rs +++ b/lib/crowdb-kv/src/rpc/px_rpc_transport.rs @@ -47,6 +47,7 @@ pub struct PxRpcTransport { rpc: Arc, connections: ConnectionPoolIndex, next_req_id: AtomicU64, + cluster_id: Option, } impl std::fmt::Debug for PxRpcTransport { @@ -94,6 +95,7 @@ impl PxRpcTransport { rpc, connections: ConnectionPoolIndex::new(pool_size, None), next_req_id: AtomicU64::new(1), + cluster_id: std::env::var("CROWDB_KV_CLUSTER_ID").ok(), } } @@ -142,16 +144,19 @@ impl PxRpcTransport { /// Send a `Prepare` request via crowdb-rpc. pub async fn send_prepare( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, slot: u64, ballot: PxBallot, term: u64, group_id: u64, membership_epoch: u64, ) -> Result { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBPrepareRequestArgs { id: req_id, rpc_create_nano: 0, @@ -162,6 +167,8 @@ impl PxRpcTransport { term, group_id, membership_epoch, + target_replica_id, + target_cluster_id, }; let req = FBPrepareRequest::create(&mut builder, &args); builder.finish(req, None); @@ -215,11 +222,13 @@ impl PxRpcTransport { /// Send a unary `Accept` request via crowdb-rpc. pub async fn send_accept( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, entry: &PxLogEntry, group_id: u64, membership_epoch: u64, ) -> Result { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); @@ -234,6 +243,7 @@ impl PxRpcTransport { payload: Some(payload), }, ); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBAcceptRequestArgs { id: req_id, rpc_create_nano: 0, @@ -245,6 +255,8 @@ impl PxRpcTransport { value: Some(value), group_id, membership_epoch, + target_replica_id, + target_cluster_id, }; let req = FBAcceptRequest::create(&mut builder, &args); builder.finish(req, None); @@ -290,13 +302,16 @@ impl PxRpcTransport { /// Send a `PreVote` request via crowdb-rpc. pub async fn send_pre_vote( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, req: VoteRequestPayload, group_id: u64, ) -> Result { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBPreVoteRequestArgs { id: req_id, rpc_create_nano: 0, @@ -306,6 +321,8 @@ impl PxRpcTransport { candidate_id: req.candidate_id, accepted_log_tip_slot: req.accepted_log_tip_slot, accepted_log_tip_term: req.accepted_log_tip_term, + target_replica_id, + target_cluster_id, }; let fb_req = FBPreVoteRequest::create(&mut builder, &args); builder.finish(fb_req, None); @@ -338,13 +355,16 @@ impl PxRpcTransport { /// Send a `RequestVote` request via crowdb-rpc. pub async fn send_request_vote( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, req: VoteRequestPayload, group_id: u64, ) -> Result { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBRequestVoteRequestArgs { id: req_id, rpc_create_nano: 0, @@ -354,6 +374,8 @@ impl PxRpcTransport { candidate_id: req.candidate_id, accepted_log_tip_slot: req.accepted_log_tip_slot, accepted_log_tip_term: req.accepted_log_tip_term, + target_replica_id, + target_cluster_id, }; let fb_req = FBRequestVoteRequest::create(&mut builder, &args); builder.finish(fb_req, None); @@ -386,13 +408,16 @@ impl PxRpcTransport { /// Send a `Heartbeat` request via crowdb-rpc. pub async fn send_heartbeat( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, req: HeartbeatRequestPayload, group_id: u64, ) -> Result { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBHeartbeatRequestArgs { id: req_id, rpc_create_nano: 0, @@ -405,6 +430,8 @@ impl PxRpcTransport { committed_safe_slot: req.committed_safe_slot, lease_grant_until_ms_mono: req.lease_grant_until_ms_mono, t_send_ms_mono: req.t_send_ms_mono, + target_replica_id, + target_cluster_id, }; let fb_req = FBHeartbeatRequest::create(&mut builder, &args); builder.finish(fb_req, None); @@ -439,14 +466,17 @@ impl PxRpcTransport { /// Send a `StepDown` request via crowdb-rpc. pub async fn send_step_down( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, req: &StepDownRequestPayload, group_id: u64, ) -> Result { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); let reason = builder.create_string(&req.reason); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBStepDownRequestArgs { id: req_id, rpc_create_nano: 0, @@ -455,6 +485,8 @@ impl PxRpcTransport { term: req.term, target_leader_id: req.target_leader_id, reason: Some(reason), + target_replica_id, + target_cluster_id, }; let fb_req = FBStepDownRequest::create(&mut builder, &args); builder.finish(fb_req, None); @@ -485,15 +517,18 @@ impl PxRpcTransport { /// Send a `FetchGap` request via crowdb-rpc. pub async fn send_fetch_gap( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, group_id: u64, slot: u64, term: u64, leader_id: u64, ) -> Result { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBFetchGapRequestArgs { id: req_id, rpc_create_nano: 0, @@ -502,6 +537,8 @@ impl PxRpcTransport { slot, term, leader_id, + target_replica_id, + target_cluster_id, }; let fb_req = FBFetchGapRequest::create(&mut builder, &args); builder.finish(fb_req, None); @@ -537,16 +574,19 @@ impl PxRpcTransport { /// reply is expected — the frame is sent with no completion callback. pub fn send_chosen( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, group_id: u64, slot: u64, term: u64, leader_id: u64, ballot_round: u64, ) -> Result<(), PxReplicaError> { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBChosenNotificationArgs { id: req_id, rpc_create_nano: 0, @@ -556,6 +596,8 @@ impl PxRpcTransport { term, leader_id, ballot_round, + target_replica_id, + target_cluster_id, }; let fb_req = FBChosenNotification::create(&mut builder, &args); builder.finish(fb_req, None); @@ -579,7 +621,7 @@ impl PxRpcTransport { #[allow(clippy::too_many_arguments)] pub fn send_batch_chosen( &self, - rpc_endpoint: &str, + destination: ReplicaDestination<'_>, group_id: u64, start_slot: u64, end_slot: u64, @@ -587,9 +629,12 @@ impl PxRpcTransport { leader_id: u64, ballot_round: u64, ) -> Result<(), PxReplicaError> { + let rpc_endpoint = destination.endpoint; + let target_replica_id = destination.replica_id; let req_id = self.next_id(); let conn = self.conn_for(rpc_endpoint)?; let mut builder = FlatBufferBuilder::new(); + let target_cluster_id = self.cluster_id.as_deref().map(|id| builder.create_string(id)); let args = FBBatchChosenNotificationArgs { id: req_id, rpc_create_nano: 0, @@ -600,6 +645,8 @@ impl PxRpcTransport { term, leader_id, ballot_round, + target_replica_id, + target_cluster_id, }; let fb_req = FBBatchChosenNotification::create(&mut builder, &args); builder.finish(fb_req, None); @@ -928,81 +975,10 @@ pub(crate) enum SnapshotTransferError { Protocol(String), } -// ── Error mapping ──────────────────────────────────────────────── - -fn rpc_error_to_px(e: RpcError) -> PxReplicaError { - match e { - RpcError::Timeout => PxReplicaError::Timeout("crowdb-rpc deadline expired".into()), - RpcError::SendQueueFull => PxReplicaError::Backpressure("crowdb-rpc send queues are full".into()), - RpcError::ConnectionClosed | RpcError::ConnectionError | RpcError::AllDown => { - PxReplicaError::Transport(format!("crowdb-rpc error: {e:?}")) - } - RpcError::Ok | RpcError::RegistrationFailed | RpcError::InvalidArg | RpcError::Unknown(_) => { - PxReplicaError::Internal(format!("crowdb-rpc error: {e:?}")) - } - } -} - -fn rpc_error_invalidates_connection(e: RpcError) -> bool { - matches!(e, RpcError::ConnectionClosed | RpcError::ConnectionError) -} - -fn check_ret_code(code: FBKvRetCode, msg: Option<&str>) -> Result<(), PxReplicaError> { - match code { - FBKvRetCode::Success => Ok(()), - FBKvRetCode::NotFound => Err(PxReplicaError::GroupNotFound(0)), - FBKvRetCode::Unavailable => Err(PxReplicaError::ShuttingDown), - FBKvRetCode::Internal | FBKvRetCode::InvalidArgument => { - Err(PxReplicaError::Internal(msg.unwrap_or("internal error").into())) - } - _ => Err(PxReplicaError::Internal(format!("unknown ret_code: {code:?}"))), - } -} - -fn check_snapshot_ret_code(code: FBKvRetCode, msg: Option<&str>) -> Result<(), SnapshotTransferError> { - let message = msg.unwrap_or("remote snapshot failure").to_string(); - match code { - FBKvRetCode::Success => Ok(()), - FBKvRetCode::SnapshotNotFound | FBKvRetCode::NotFound => { - Err(SnapshotTransferError::NotFound(message)) - } - FBKvRetCode::SnapshotExpired => Err(SnapshotTransferError::Expired(message)), - FBKvRetCode::SnapshotInvalidOffset => Err(SnapshotTransferError::InvalidOffset(message)), - FBKvRetCode::SnapshotBackpressure => Err(SnapshotTransferError::Backpressure(message)), - FBKvRetCode::SnapshotTopologyChanged => Err(SnapshotTransferError::TopologyChanged(message)), - FBKvRetCode::SnapshotIntegrity => Err(SnapshotTransferError::Integrity(message)), - FBKvRetCode::Unavailable => Err(SnapshotTransferError::Transport(PxReplicaError::ShuttingDown)), - FBKvRetCode::Internal | FBKvRetCode::InvalidArgument => Err(SnapshotTransferError::Protocol(message)), - _ => Err(SnapshotTransferError::Protocol(format!( - "unknown ret_code: {code:?}" - ))), - } -} - -// ── Endpoint parsing ───────────────────────────────────────────── - -/// Normalize a service-registry endpoint: prepend `http://` if no -/// scheme, rewrite `0.0.0.0` to `127.0.0.1`. -fn normalize_endpoint(endpoint: &str) -> String { - let with_scheme = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { - endpoint.to_string() - } else { - format!("http://{endpoint}") - }; - with_scheme.replacen("://0.0.0.0:", "://127.0.0.1:", 1) -} +mod connection; +use connection::{ + check_ret_code, check_snapshot_ret_code, normalize_endpoint, parse_endpoint, + rpc_error_invalidates_connection, rpc_error_to_px, +}; -/// Parse `http://host:port` into `(host, port)`. -fn parse_endpoint(endpoint: &str) -> Result<(String, i32), String> { - let without_scheme = endpoint - .strip_prefix("http://") - .or_else(|| endpoint.strip_prefix("https://")) - .unwrap_or(endpoint); - let (host, port_str) = without_scheme - .rsplit_once(':') - .ok_or_else(|| format!("invalid endpoint: {endpoint}"))?; - let port: i32 = port_str - .parse() - .map_err(|_| format!("invalid port in endpoint: {endpoint}"))?; - Ok((host.to_string(), port)) -} +pub use connection::ReplicaDestination; diff --git a/lib/crowdb-kv/src/rpc/px_rpc_transport/connection.rs b/lib/crowdb-kv/src/rpc/px_rpc_transport/connection.rs new file mode 100644 index 000000000..29bbba2a6 --- /dev/null +++ b/lib/crowdb-kv/src/rpc/px_rpc_transport/connection.rs @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{FBKvRetCode, PxReplicaError, PxRpcTransport, RpcError, SnapshotTransferError}; + +impl PxRpcTransport { + pub(crate) fn clear_connections(&self) { + self.connections.clear(); + } +} + +// ── Error mapping ──────────────────────────────────────────────── + +pub(super) fn rpc_error_to_px(e: RpcError) -> PxReplicaError { + match e { + RpcError::Timeout => PxReplicaError::Timeout("crowdb-rpc deadline expired".into()), + RpcError::SendQueueFull => PxReplicaError::Backpressure("crowdb-rpc send queues are full".into()), + RpcError::ConnectionClosed | RpcError::ConnectionError | RpcError::AllDown => { + PxReplicaError::Transport(format!("crowdb-rpc error: {e:?}")) + } + RpcError::Ok | RpcError::RegistrationFailed | RpcError::InvalidArg | RpcError::Unknown(_) => { + PxReplicaError::Internal(format!("crowdb-rpc error: {e:?}")) + } + } +} + +pub(super) fn rpc_error_invalidates_connection(e: RpcError) -> bool { + matches!(e, RpcError::ConnectionClosed | RpcError::ConnectionError) +} + +pub(super) fn check_ret_code(code: FBKvRetCode, msg: Option<&str>) -> Result<(), PxReplicaError> { + match code { + FBKvRetCode::Success => Ok(()), + FBKvRetCode::NotFound => Err(PxReplicaError::GroupNotFound(0)), + FBKvRetCode::Unavailable => Err(PxReplicaError::ShuttingDown), + FBKvRetCode::Internal | FBKvRetCode::InvalidArgument => { + Err(PxReplicaError::Internal(msg.unwrap_or("internal error").into())) + } + _ => Err(PxReplicaError::Internal(format!("unknown ret_code: {code:?}"))), + } +} + +pub(super) fn check_snapshot_ret_code( + code: FBKvRetCode, + msg: Option<&str>, +) -> Result<(), SnapshotTransferError> { + let message = msg.unwrap_or("remote snapshot failure").to_string(); + match code { + FBKvRetCode::Success => Ok(()), + FBKvRetCode::SnapshotNotFound | FBKvRetCode::NotFound => { + Err(SnapshotTransferError::NotFound(message)) + } + FBKvRetCode::SnapshotExpired => Err(SnapshotTransferError::Expired(message)), + FBKvRetCode::SnapshotInvalidOffset => Err(SnapshotTransferError::InvalidOffset(message)), + FBKvRetCode::SnapshotBackpressure => Err(SnapshotTransferError::Backpressure(message)), + FBKvRetCode::SnapshotTopologyChanged => Err(SnapshotTransferError::TopologyChanged(message)), + FBKvRetCode::SnapshotIntegrity => Err(SnapshotTransferError::Integrity(message)), + FBKvRetCode::Unavailable => Err(SnapshotTransferError::Transport(PxReplicaError::ShuttingDown)), + FBKvRetCode::Internal | FBKvRetCode::InvalidArgument => Err(SnapshotTransferError::Protocol(message)), + _ => Err(SnapshotTransferError::Protocol(format!( + "unknown ret_code: {code:?}" + ))), + } +} + +// ── Endpoint parsing ───────────────────────────────────────────── + +/// Normalize a service-registry endpoint: prepend `http://` if no +/// scheme, rewrite `0.0.0.0` to `127.0.0.1`. +pub(super) fn normalize_endpoint(endpoint: &str) -> String { + let with_scheme = if endpoint.starts_with("http://") || endpoint.starts_with("https://") { + endpoint.to_string() + } else { + format!("http://{endpoint}") + }; + with_scheme.replacen("://0.0.0.0:", "://127.0.0.1:", 1) +} + +/// Parse `http://host:port` into `(host, port)`. +pub(super) fn parse_endpoint(endpoint: &str) -> Result<(String, i32), String> { + let without_scheme = endpoint + .strip_prefix("http://") + .or_else(|| endpoint.strip_prefix("https://")) + .unwrap_or(endpoint); + let (host, port_str) = without_scheme + .rsplit_once(':') + .ok_or_else(|| format!("invalid endpoint: {endpoint}"))?; + let port: i32 = port_str + .parse() + .map_err(|_| format!("invalid port in endpoint: {endpoint}"))?; + Ok((host.to_string(), port)) +} + +/// Logical destination remains explicit even if its IP is reassigned. +#[derive(Clone, Copy)] +pub struct ReplicaDestination<'a> { + pub endpoint: &'a str, + pub replica_id: u64, +} diff --git a/lib/crowdb-kv/tests/common/test_client.rs b/lib/crowdb-kv/tests/common/test_client.rs index 02c1bbde8..656fa268f 100644 --- a/lib/crowdb-kv/tests/common/test_client.rs +++ b/lib/crowdb-kv/tests/common/test_client.rs @@ -224,7 +224,10 @@ impl TestPxClient { let reply = self .transport .send_prepare( - &self.endpoint, + crowdb_kv::rpc::ReplicaDestination { + endpoint: &self.endpoint, + replica_id: 0, + }, req.slot, PxBallot::new(req.round, req.leader_id), req.term, diff --git a/lib/crowdb-kv/tests/group_test/election_test.rs b/lib/crowdb-kv/tests/group_test/election_test.rs index efa3190bc..8a8f5011d 100644 --- a/lib/crowdb-kv/tests/group_test/election_test.rs +++ b/lib/crowdb-kv/tests/group_test/election_test.rs @@ -414,3 +414,76 @@ fn xorshift_random_between_ms_stays_in_range() { assert!((30..=60).contains(&v), "v={v} out of [30, 60]"); } } + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn stalled_gap_repair_cannot_stop_the_leader_heartbeat_ticker() { + use crowdb_kv::paxos::roles::{Learner, PxBallot, PxLogEntry}; + let cfg = PxElectionConfig::for_tests(); + let group = Arc::new(PxGroup::new( + 99, + PxLocalReplica::new(31, PxLocalReplicaRole::Follower), + )); + let cancel = group.tenure_cancel(); + let handle = spawn(Arc::downgrade(&group), cfg, cancel.clone()); + advance_until_role(&group, &cfg, PxLocalReplicaRole::Leader).await; + for _ in 0..100 { + if group.leader_read_ready() { + break; + } + tokio::task::yield_now().await; + } + assert!(group.leader_read_ready()); + group.pause_repair_for_tests(); + group + .local_replica() + .learner + .learn( + PxLogEntry { + slot: 2, + ballot: PxBallot::new(1, 31), + term: group.local_replica().current_term_snapshot(), + payload: bytes::Bytes::new(), + }, + &[], + ) + .await; + for _ in 0..100 { + tokio::time::advance(Duration::from_millis(cfg.heartbeat_interval_ms + 1)).await; + tokio::task::yield_now().await; + if group.repair_is_paused_for_tests() { + break; + } + } + assert!( + group.repair_is_paused_for_tests(), + "repair must be in flight before measuring heartbeats" + ); + let before = group.local_replica().lease_state_snapshot().lease_read_until; + for _ in 0..10 { + tokio::time::advance(Duration::from_millis(cfg.heartbeat_interval_ms + 1)).await; + tokio::task::yield_now().await; + } + let after = group.local_replica().lease_state_snapshot().lease_read_until; + assert!( + after > before, + "gap repair held heartbeats: before={before:?}, after={after:?}" + ); + assert_eq!(group.local_replica().contiguous_chosen(), 0); + group.resume_repair_for_tests(); + for _ in 0..100 { + if group.local_replica().contiguous_chosen() == 2 { + break; + } + tokio::task::yield_now().await; + } + assert_eq!( + group.local_replica().contiguous_chosen(), + 2, + "the same repair must finish after IO resumes" + ); + cancel.cancel(); + tokio::time::timeout(Duration::from_secs(1), handle) + .await + .unwrap() + .unwrap(); +} diff --git a/lib/crowdb-kv/tests/group_test/owner_fence_test.rs b/lib/crowdb-kv/tests/group_test/owner_fence_test.rs index 9b6c689df..9f9e5177e 100644 --- a/lib/crowdb-kv/tests/group_test/owner_fence_test.rs +++ b/lib/crowdb-kv/tests/group_test/owner_fence_test.rs @@ -349,3 +349,66 @@ owner_fence_tests!( new_leader_tenure_does_not_reuse_a_closed_old_admission, unresolved_old_proposal_blocks_a_topology_replacement_from_claiming_ownership, ); + +#[tokio::test] +async fn uncertain_owner_proposal_reenters_election_before_reopening_reads() { + let cluster = crate::common::cluster::start_cluster_no_leader(&[1, 2, 3]).await; + let leader = tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if let Some(node) = cluster.elected_leader() { + let group = node.get_group(1).unwrap(); + if group.leader_read_ready() { + break group; + } + } + tokio::task::yield_now().await; + } + }) + .await + .expect("initial leader recovery"); + let term = leader.local_replica().current_term_snapshot(); + let fence = Bytes::from_static(b"/diskdb/ownership-fence/1/1/1"); + // Losing an admitted proposal's result must close this tenure, even + // while the peer connections and heartbeat quorum remain healthy. + drop(leader.hold_owner_proposal_for_tests(fence.clone())); + assert!(matches!( + leader + .propose_cas(encode(&fence, b"owner"), fence, 0, 21, 1) + .await, + ProposeResult::OutcomeUnknown + )); + assert!(!leader.leader_read_ready()); + let recovered = tokio::time::timeout(std::time::Duration::from_secs(3), async { + loop { + if let Some(node) = cluster.elected_leader() { + let group = node.get_group(1).unwrap(); + if group.local_replica().current_term_snapshot() > term && group.leader_read_ready() { + break group; + } + } + tokio::task::yield_now().await; + } + }) + .await + .expect("an uncertain tenure must yield and recover despite healthy heartbeats"); + let key = Bytes::from_static(b"after-owner-recovery"); + assert!(matches!( + recovered + .propose_cas(encode(&key, b"confirmed"), key.clone(), 0, 22, 1) + .await, + ProposeResult::Chosen { .. } + )); + assert_eq!( + recovered + .local_replica() + .learner + .engine() + .get_versioned(&key) + .await + .unwrap() + .unwrap() + .1, + Bytes::from_static(b"confirmed") + ); + cluster.shutdown().await; +} diff --git a/lib/crowdb-kv/tests/group_test/paxos_error_test.rs b/lib/crowdb-kv/tests/group_test/paxos_error_test.rs index 5b8a33a9c..6a591e82f 100644 --- a/lib/crowdb-kv/tests/group_test/paxos_error_test.rs +++ b/lib/crowdb-kv/tests/group_test/paxos_error_test.rs @@ -111,6 +111,8 @@ async fn malformed_accept_request_is_rejected_by_rpc_boundary() { term: 0, group_id: 1, membership_epoch: 0, + target_replica_id: 0, + target_cluster_id: None, }; let req = FBPrepareRequest::create(&mut builder, &args); builder.finish(req, None); @@ -144,6 +146,8 @@ async fn malformed_accept_request_is_rejected_by_rpc_boundary() { value: None, group_id: 1, membership_epoch: 0, + target_replica_id: 0, + target_cluster_id: None, }; let req = FBAcceptRequest::create(&mut builder, &args); builder.finish(req, None); diff --git a/lib/crowdb-kv/tests/group_test/proposer_test.rs b/lib/crowdb-kv/tests/group_test/proposer_test.rs index 5e3ea9804..98d0c29ce 100644 --- a/lib/crowdb-kv/tests/group_test/proposer_test.rs +++ b/lib/crowdb-kv/tests/group_test/proposer_test.rs @@ -127,3 +127,47 @@ async fn propose_queues_when_policy_is_queue() { other => panic!("expected Chosen after queue drain, got {other:?}"), } } + +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn gap_backlog_drains_without_waiting_one_heartbeat_per_slot() { + use crowdb_kv::cluster::group_election::{spawn, LeaderElection}; + use crowdb_kv::common::config::PxElectionConfig; + use std::{sync::Arc, time::Duration}; + + let cfg = PxElectionConfig::for_tests(); + let group = Arc::new(single_leader_group()); + let cancel = group.tenure_cancel(); + let handle = spawn(Arc::downgrade(&group), cfg, cancel.clone()); + for _ in 0..100 { + tokio::task::yield_now().await; + } + assert!(group.leader_read_ready()); + group + .local_replica() + .learner + .learn( + PxLogEntry { + slot: 65, + ballot: PxBallot::new(0, 1), + term: group.local_replica().current_term_snapshot(), + payload: bytes::Bytes::new(), + }, + &[], + ) + .await; + assert_eq!(group.local_replica().contiguous_chosen(), 0); + + // Dispatch one repair pass, then keep the clock fixed. Recovery work + // must drain the existing backlog independently of heartbeat cadence. + tokio::time::advance(Duration::from_millis(cfg.heartbeat_interval_ms + 1)).await; + for _ in 0..10_000 { + if group.local_replica().contiguous_applied() == 65 { + break; + } + tokio::task::yield_now().await; + } + let frontier = group.local_replica().contiguous_applied(); + cancel.cancel(); + handle.await.unwrap(); + assert_eq!(frontier, 65, "repair left a backlog waiting for heartbeat ticks"); +} diff --git a/lib/crowdb-kv/tests/rpc_migration_test.rs b/lib/crowdb-kv/tests/rpc_migration_test.rs index a57e8fcd8..d54782206 100644 --- a/lib/crowdb-kv/tests/rpc_migration_test.rs +++ b/lib/crowdb-kv/tests/rpc_migration_test.rs @@ -86,6 +86,54 @@ impl CrowdbRpcCluster { } } +#[tokio::test] +async fn recovered_routes_preserve_membership_and_reject_reused_peer_addresses() { + let cluster = start_crowdb_rpc_cluster().await; + let group = cluster.leader.get_group(1).unwrap(); + let before: Vec<_> = group + .remote_replica_info() + .into_iter() + .map(|(id, endpoint, voting)| (id, endpoint.to_owned(), voting)) + .collect(); + let quorum = group.quorum(); + let wrong = cluster.leader.listen_addr().unwrap(); + group.set_recovery_routes(&std::collections::BTreeMap::from([(2, wrong)])); + let remote = group.get_remote_replica(2).unwrap(); + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(200), + remote.send_prepare(91, PxBallot::new(5, 1), 0, 1, 0) + ) + .await + .is_err(), + "a live endpoint hosting another replica must not contribute a promise" + ); + group.set_recovery_routes(&std::collections::BTreeMap::new()); + assert!( + remote + .send_prepare(92, PxBallot::new(5, 1), 0, 1, 0) + .await + .is_err(), + "missing proof cannot fall back to the original IP" + ); + group.set_recovery_routes(&std::collections::BTreeMap::from([( + 2, + cluster.follower.listen_addr().unwrap(), + )])); + assert!(remote + .send_prepare(93, PxBallot::new(5, 1), 0, 1, 0) + .await + .is_ok()); + let after: Vec<_> = group + .remote_replica_info() + .into_iter() + .map(|(id, endpoint, voting)| (id, endpoint.to_owned(), voting)) + .collect(); + assert_eq!(before, after); + assert_eq!(group.quorum(), quorum); + cluster.shutdown().await; +} + #[tokio::test] async fn crowdb_rpc_prepare_accept_roundtrip() { let cluster = start_crowdb_rpc_cluster().await; diff --git a/lib/crowdb-protocol/Cargo.toml b/lib/crowdb-protocol/Cargo.toml index 46cb891e7..837495797 100644 --- a/lib/crowdb-protocol/Cargo.toml +++ b/lib/crowdb-protocol/Cargo.toml @@ -32,5 +32,6 @@ sha2 = "0.10" utoipa = { version = "5", optional = true } fs2 = "0.4" thiserror = { workspace = true } +url = "2" [build-dependencies] diff --git a/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame.h b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame.h index 01c88ff37..cbbc1bf69 100644 --- a/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame.h +++ b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame.h @@ -3,289 +3,7 @@ #pragma once -#include "crowdb-common/crc32c.h" - -#include -#include -#include -#include -#include -#include - -namespace crowdb::protocol -{ - -inline constexpr size_t kFrameHeaderPrefixBytes = 14; -inline constexpr size_t kFrameFooterBytes = 20; -inline constexpr size_t kMaxFrameBytes = 64 * 1024; -inline constexpr size_t kMaxFramePayloadBytes = kMaxFrameBytes - kFrameHeaderPrefixBytes - kFrameFooterBytes; - -enum class FrameMagic : uint16_t { - RepoSmallV1 = 0x0101, - RepoLargeV1 = 0x0201, - StreamV1 = 0x0301, - BtreePageV1 = 0x0401, - PageIndexV1 = 0x0501, -}; - -struct FrameChunkId -{ - uint64_t high = 0; - uint64_t low = 0; - - [[nodiscard]] bool operator==(const FrameChunkId &) const = default; -}; - -struct FrameHeaderPrefix -{ - FrameMagic magic = FrameMagic::RepoSmallV1; - uint16_t payload_offset = 0; - uint16_t payload_size = 0; - uint64_t write_time_ms = 0; -}; - -enum class FrameError : uint8_t { - Ok, - Incomplete, - UnknownMagic, - InvalidPayloadOffset, - FrameTooLarge, - ChunkIdMismatch, - ChecksumMismatch, -}; - -struct ParsedFrame -{ - FrameHeaderPrefix header; - std::span payload; - FrameChunkId chunk_id; - size_t physical_length = 0; -}; - -[[nodiscard]] inline bool valid_magic(uint16_t value) -{ - return value >= static_cast(FrameMagic::RepoSmallV1) && - value <= static_cast(FrameMagic::PageIndexV1) && (value & 0xffU) == 1; -} - -[[nodiscard]] inline uint16_t read_u16_le(const uint8_t *data) -{ - return static_cast(data[0]) | (static_cast(data[1]) << 8U); -} - -[[nodiscard]] inline uint32_t read_u32_le(const uint8_t *data) -{ - return static_cast(data[0]) | (static_cast(data[1]) << 8U) | - (static_cast(data[2]) << 16U) | (static_cast(data[3]) << 24U); -} - -[[nodiscard]] inline uint64_t read_u64_le(const uint8_t *data) -{ - uint64_t value = 0; - for (uint32_t i = 0; i != 8; ++i) { - value |= static_cast(data[i]) << (i * 8U); - } - return value; -} - -[[nodiscard]] inline uint64_t read_u64_be(const uint8_t *data) -{ - uint64_t value = 0; - for (uint32_t i = 0; i != 8; ++i) { - value = (value << 8U) | data[i]; - } - return value; -} - -inline void append_u16_le(std::vector *out, uint16_t value) -{ - out->push_back(static_cast(value)); - out->push_back(static_cast(value >> 8U)); -} - -inline void append_u32_le(std::vector *out, uint32_t value) -{ - for (uint32_t i = 0; i != 4; ++i) { - out->push_back(static_cast(value >> (i * 8U))); - } -} - -inline void append_u64_le(std::vector *out, uint64_t value) -{ - for (uint32_t i = 0; i != 8; ++i) { - out->push_back(static_cast(value >> (i * 8U))); - } -} - -inline void append_u64_be(std::vector *out, uint64_t value) -{ - for (int shift = 56; shift >= 0; shift -= 8) { - out->push_back(static_cast(value >> shift)); - } -} - -[[nodiscard]] inline FrameError encode_frame(FrameMagic magic, FrameChunkId chunk_id, std::span payload, - uint64_t write_time_ms, std::vector *out) -{ - if (out == nullptr || payload.size() > kMaxFramePayloadBytes) { - return FrameError::FrameTooLarge; - } - out->clear(); - out->reserve(kFrameHeaderPrefixBytes + payload.size() + kFrameFooterBytes); - append_u16_le(out, static_cast(magic)); - append_u16_le(out, static_cast(kFrameHeaderPrefixBytes)); - append_u16_le(out, static_cast(payload.size())); - append_u64_le(out, write_time_ms); - out->insert(out->end(), payload.begin(), payload.end()); - append_u64_be(out, chunk_id.high); - append_u64_be(out, chunk_id.low); - const uint32_t checksum = crowdb::common::crc32c(out->data(), out->size()); - out->insert(out->begin() + static_cast(kFrameHeaderPrefixBytes + payload.size()), - {static_cast(checksum), static_cast(checksum >> 8U), - static_cast(checksum >> 16U), static_cast(checksum >> 24U)}); - return FrameError::Ok; -} - -[[nodiscard]] inline size_t framed_physical_length(size_t payload_size) -{ - const size_t full_frames = payload_size / kMaxFramePayloadBytes; - const size_t tail = payload_size % kMaxFramePayloadBytes; - if (full_frames > - (SIZE_MAX - (tail == 0 ? 0 : tail + kFrameHeaderPrefixBytes + kFrameFooterBytes)) / kMaxFrameBytes) { - return 0; - } - return (full_frames * kMaxFrameBytes) + (tail == 0 ? 0 : tail + kFrameHeaderPrefixBytes + kFrameFooterBytes); -} - -[[nodiscard]] inline FrameError encode_frames(FrameMagic magic, FrameChunkId chunk_id, std::span payload, - uint64_t write_time_ms, std::vector *out) -{ - if (out == nullptr || (framed_physical_length(payload.size()) == 0 && !payload.empty())) { - return FrameError::FrameTooLarge; - } - out->clear(); - out->reserve(framed_physical_length(payload.size())); - for (size_t offset = 0; offset < payload.size(); offset += kMaxFramePayloadBytes) { - std::vector frame; - const size_t length = std::min(kMaxFramePayloadBytes, payload.size() - offset); - const FrameError error = encode_frame(magic, chunk_id, payload.subspan(offset, length), write_time_ms, &frame); - if (error != FrameError::Ok) { - return error; - } - out->insert(out->end(), frame.begin(), frame.end()); - } - return FrameError::Ok; -} - -[[nodiscard]] inline FrameError parse_frame(std::span bytes, FrameChunkId expected_chunk_id, - ParsedFrame *out) -{ - if (bytes.size() < kFrameHeaderPrefixBytes) { - return FrameError::Incomplete; - } - const uint16_t magic_value = read_u16_le(bytes.data()); - if (!valid_magic(magic_value)) { - return FrameError::UnknownMagic; - } - const uint16_t payload_offset = read_u16_le(bytes.data() + 2); - const uint16_t payload_size = read_u16_le(bytes.data() + 4); - if (payload_offset < kFrameHeaderPrefixBytes) { - return FrameError::InvalidPayloadOffset; - } - const size_t physical_length = static_cast(payload_offset) + payload_size + kFrameFooterBytes; - if (physical_length > kMaxFrameBytes) { - return FrameError::FrameTooLarge; - } - if (bytes.size() < physical_length) { - return FrameError::Incomplete; - } - const size_t footer_start = physical_length - kFrameFooterBytes; - std::vector checksum_input; - checksum_input.reserve(footer_start + 16); - checksum_input.insert(checksum_input.end(), bytes.begin(), bytes.begin() + static_cast(footer_start)); - checksum_input.insert(checksum_input.end(), bytes.begin() + static_cast(footer_start + 4), - bytes.begin() + static_cast(physical_length)); - if (crowdb::common::crc32c(checksum_input.data(), checksum_input.size()) != - read_u32_le(bytes.data() + footer_start)) { - return FrameError::ChecksumMismatch; - } - const FrameChunkId chunk_id{ - .high = read_u64_be(bytes.data() + footer_start + 4), - .low = read_u64_be(bytes.data() + footer_start + 12), - }; - if (chunk_id != expected_chunk_id) { - return FrameError::ChunkIdMismatch; - } - if (out != nullptr) { - *out = ParsedFrame{ - .header = - { - .magic = static_cast(magic_value), - .payload_offset = payload_offset, - .payload_size = payload_size, - .write_time_ms = read_u64_le(bytes.data() + 6), - }, - .payload = bytes.subspan(payload_offset, payload_size), - .chunk_id = chunk_id, - .physical_length = physical_length, - }; - } - return FrameError::Ok; -} - -[[nodiscard]] inline FrameError parse_frames(std::span bytes, FrameChunkId expected_chunk_id, - FrameMagic expected_magic, size_t logical_length, - std::vector *payload) -{ - if (payload == nullptr || framed_physical_length(logical_length) != bytes.size()) { - return FrameError::Incomplete; - } - payload->clear(); - payload->reserve(logical_length); - size_t offset = 0; - while (offset < bytes.size()) { - ParsedFrame frame; - const FrameError error = parse_frame(bytes.subspan(offset), expected_chunk_id, &frame); - if (error != FrameError::Ok) { - return error; - } - if (frame.header.magic != expected_magic || frame.physical_length > bytes.size() - offset) { - return FrameError::UnknownMagic; - } - payload->insert(payload->end(), frame.payload.begin(), frame.payload.end()); - offset += frame.physical_length; - } - return payload->size() == logical_length ? FrameError::Ok : FrameError::Incomplete; -} - -/// Validates a concatenated sequence whose first frame already identifies its -/// public kind. DiskIO uses this boundary check without needing caller-private -/// location metadata: each footer supplies the expected chunk ID for parsing. -[[nodiscard]] inline FrameError validate_frame_sequence(std::span bytes) -{ - size_t offset = 0; - while (offset < bytes.size()) { - const auto remaining = bytes.subspan(offset); - if (remaining.size() < kFrameHeaderPrefixBytes) { - return FrameError::Incomplete; - } - const size_t physical_length = static_cast(read_u16_le(remaining.data() + 2)) + - read_u16_le(remaining.data() + 4) + kFrameFooterBytes; - if (physical_length > remaining.size() || physical_length < kFrameHeaderPrefixBytes + kFrameFooterBytes) { - return FrameError::Incomplete; - } - const size_t footer_start = physical_length - kFrameFooterBytes; - ParsedFrame frame; - const FrameError error = parse_frame(remaining.first(physical_length), - {.high = read_u64_be(remaining.data() + footer_start + 4), - .low = read_u64_be(remaining.data() + footer_start + 12)}, - &frame); - if (error != FrameError::Ok) { - return error; - } - offset += frame.physical_length; - } - return FrameError::Ok; -} - -} // namespace crowdb::protocol +#include "crowdb-protocol/frame/base.h" +#include "crowdb-protocol/frame/encoding.h" +#include "crowdb-protocol/frame/group.h" +#include "crowdb-protocol/frame/parsing.h" diff --git a/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/base.h b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/base.h new file mode 100644 index 000000000..10dbe61dd --- /dev/null +++ b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/base.h @@ -0,0 +1,162 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-common/crc32c.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace crowdb::protocol +{ + +inline constexpr size_t kFrameHeaderPrefixBytes = 14; +inline constexpr size_t kFrameFooterBytes = 20; +inline constexpr size_t kMaxFrameBytes = 64 * 1024; + +enum class FrameMagic : uint16_t { + RepoSmallV2 = 0x0102, + RepoLargeV2 = 0x0202, + StreamV2 = 0x0302, + BtreePageV2 = 0x0402, + PageIndexV2 = 0x0502, +}; + +struct FrameChunkId +{ + uint64_t high = 0; + uint64_t low = 0; + + [[nodiscard]] bool operator==(const FrameChunkId &) const = default; +}; + +struct FrameHeaderPrefix +{ + FrameMagic magic = FrameMagic::RepoSmallV2; + uint16_t payload_offset = 0; + uint16_t payload_size = 0; + uint64_t write_time_ms = 0; +}; + +enum class FrameError : uint8_t { + Ok, + Incomplete, + UnknownMagic, + InvalidPayloadOffset, + FrameTooLarge, + ChunkIdMismatch, + ChecksumMismatch, + InvalidRole, + InvalidGroup, + LengthOverflow, +}; + +inline constexpr size_t kFrameV2HeaderBytes = 15; +inline constexpr size_t kFrameGroupHeaderBytes = 27; +inline constexpr size_t kMaxFrameV2PayloadBytes = kMaxFrameBytes - kFrameV2HeaderBytes - kFrameFooterBytes; +inline constexpr size_t kMaxFrameGroupPayloadBytes = kMaxFrameBytes - kFrameGroupHeaderBytes - kFrameFooterBytes; + +enum class FrameRole : uint8_t { Ordinary, Head, Follower }; + +struct FrameGroupFields +{ + FrameRole role = FrameRole::Ordinary; + uint64_t value = 0; + uint32_t count = 0; +}; + +struct ParsedFrame +{ + FrameHeaderPrefix header; + FrameGroupFields group; + std::span payload; + FrameChunkId chunk_id; + size_t physical_length = 0; +}; + +[[nodiscard]] inline std::optional frame_magic(uint16_t value) +{ + switch (value) { + case 0x0102: + return FrameMagic::RepoSmallV2; + case 0x0202: + return FrameMagic::RepoLargeV2; + case 0x0302: + return FrameMagic::StreamV2; + case 0x0402: + return FrameMagic::BtreePageV2; + case 0x0502: + return FrameMagic::PageIndexV2; + default: + return std::nullopt; + } +} + +[[nodiscard]] inline bool valid_magic(uint16_t value) +{ + return frame_magic(value).has_value(); +} + +[[nodiscard]] inline uint16_t read_u16_le(const uint8_t *data) +{ + return static_cast(data[0]) | (static_cast(data[1]) << 8U); +} + +[[nodiscard]] inline uint32_t read_u32_le(const uint8_t *data) +{ + return static_cast(data[0]) | (static_cast(data[1]) << 8U) | + (static_cast(data[2]) << 16U) | (static_cast(data[3]) << 24U); +} + +[[nodiscard]] inline uint64_t read_u64_le(const uint8_t *data) +{ + uint64_t value = 0; + for (uint32_t i = 0; i != 8; ++i) { + value |= static_cast(data[i]) << (i * 8U); + } + return value; +} + +[[nodiscard]] inline uint64_t read_u64_be(const uint8_t *data) +{ + uint64_t value = 0; + for (uint32_t i = 0; i != 8; ++i) { + value = (value << 8U) | data[i]; + } + return value; +} + +inline void append_u16_le(std::vector *out, uint16_t value) +{ + out->push_back(static_cast(value)); + out->push_back(static_cast(value >> 8U)); +} + +inline void append_u32_le(std::vector *out, uint32_t value) +{ + for (uint32_t i = 0; i != 4; ++i) { + out->push_back(static_cast(value >> (i * 8U))); + } +} + +inline void append_u64_le(std::vector *out, uint64_t value) +{ + for (uint32_t i = 0; i != 8; ++i) { + out->push_back(static_cast(value >> (i * 8U))); + } +} + +inline void append_u64_be(std::vector *out, uint64_t value) +{ + for (int shift = 56; shift >= 0; shift -= 8) { + out->push_back(static_cast(value >> shift)); + } +} + +} // namespace crowdb::protocol diff --git a/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/encoding.h b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/encoding.h new file mode 100644 index 000000000..d29ee7c6b --- /dev/null +++ b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/encoding.h @@ -0,0 +1,73 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-protocol/frame/base.h" + +namespace crowdb::protocol +{ + +[[nodiscard]] inline FrameError encode_frame(FrameMagic magic, FrameChunkId chunk_id, std::span payload, + uint64_t write_time_ms, std::vector *out) +{ + if (magic == FrameMagic::StreamV2) { + return FrameError::InvalidRole; + } + const size_t header_bytes = kFrameV2HeaderBytes; + if (out == nullptr || !valid_magic(static_cast(magic)) || + payload.size() > kMaxFrameBytes - header_bytes - kFrameFooterBytes) { + return FrameError::FrameTooLarge; + } + out->clear(); + out->reserve(header_bytes + payload.size() + kFrameFooterBytes); + append_u16_le(out, static_cast(magic)); + append_u16_le(out, static_cast(header_bytes)); + append_u16_le(out, static_cast(payload.size())); + append_u64_le(out, write_time_ms); + out->push_back(0); + out->insert(out->end(), payload.begin(), payload.end()); + const uint32_t prefix_crc = crowdb::common::crc32c(out->data(), out->size()); + append_u64_be(out, chunk_id.high); + append_u64_be(out, chunk_id.low); + const uint32_t checksum = prefix_crc; + out->insert(out->begin() + static_cast(header_bytes + payload.size()), + {static_cast(checksum), static_cast(checksum >> 8U), + static_cast(checksum >> 16U), static_cast(checksum >> 24U)}); + return FrameError::Ok; +} + +[[nodiscard]] inline size_t framed_physical_length(size_t payload_size, size_t header_bytes = kFrameV2HeaderBytes) +{ + const size_t capacity = kMaxFrameBytes - header_bytes - kFrameFooterBytes; + const size_t full_frames = payload_size / capacity; + const size_t tail = payload_size % capacity; + if (full_frames > (SIZE_MAX - (tail == 0 ? 0 : tail + header_bytes + kFrameFooterBytes)) / kMaxFrameBytes) { + return 0; + } + return (full_frames * kMaxFrameBytes) + (tail == 0 ? 0 : tail + header_bytes + kFrameFooterBytes); +} + +[[nodiscard]] inline FrameError encode_frames(FrameMagic magic, FrameChunkId chunk_id, std::span payload, + uint64_t write_time_ms, std::vector *out) +{ + const size_t header_bytes = kFrameV2HeaderBytes; + const size_t capacity = kMaxFrameBytes - header_bytes - kFrameFooterBytes; + if (out == nullptr || (framed_physical_length(payload.size(), header_bytes) == 0 && !payload.empty())) { + return FrameError::FrameTooLarge; + } + out->clear(); + out->reserve(framed_physical_length(payload.size(), header_bytes)); + for (size_t offset = 0; offset < payload.size(); offset += capacity) { + std::vector frame; + const size_t length = std::min(capacity, payload.size() - offset); + const FrameError error = encode_frame(magic, chunk_id, payload.subspan(offset, length), write_time_ms, &frame); + if (error != FrameError::Ok) { + return error; + } + out->insert(out->end(), frame.begin(), frame.end()); + } + return FrameError::Ok; +} + +} // namespace crowdb::protocol diff --git a/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/group.h b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/group.h new file mode 100644 index 000000000..f66205fff --- /dev/null +++ b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/group.h @@ -0,0 +1,187 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-protocol/frame/base.h" +#include "crowdb-protocol/frame/encoding.h" +#include "crowdb-protocol/frame/parsing.h" + +#include + +namespace crowdb::protocol +{ + +struct ParsedAppendGroup +{ + uint64_t logical_length = 0; + uint64_t physical_length = 0; + uint32_t frame_count = 0; +}; + +/// Constant-sized validation state for bounded readers and tail scanners. +class AppendGroupValidator +{ + public: + [[nodiscard]] FrameError start(uint64_t offset, const ParsedFrame &head) + { + if (head.header.magic != FrameMagic::StreamV2 || head.group.role != FrameRole::Head || head.group.value == 0 || + head.group.count == 0 || + head.group.count != (head.group.value / kMaxFrameGroupPayloadBytes) + + (head.group.value % kMaxFrameGroupPayloadBytes != 0 ? 1 : 0)) { + return FrameError::InvalidGroup; + } + const size_t physical = framed_physical_length(static_cast(head.group.value), kFrameGroupHeaderBytes); + if (head.group.value > std::numeric_limits::max() || physical == 0 || + physical > std::numeric_limits::max() - offset) { + return FrameError::LengthOverflow; + } + head_offset_ = offset; + next_offset_ = offset; + length_ = head.group.value; + count_ = head.group.count; + observed_ = 0; + write_time_ = head.header.write_time_ms; + chunk_ = head.chunk_id; + return push(offset, head); + } + + [[nodiscard]] FrameError push(uint64_t offset, const ParsedFrame &frame) + { + if (observed_ >= count_ || offset != next_offset_ || frame.header.magic != FrameMagic::StreamV2 || + frame.chunk_id != chunk_ || frame.header.write_time_ms != write_time_ || + frame.header.payload_offset != kFrameGroupHeaderBytes || + frame.physical_length != kFrameGroupHeaderBytes + frame.payload.size() + kFrameFooterBytes) { + return FrameError::InvalidGroup; + } + const FrameRole role = observed_ == 0 ? FrameRole::Head : FrameRole::Follower; + const uint64_t value = observed_ == 0 ? length_ : head_offset_; + const uint32_t number = observed_ == 0 ? count_ : observed_; + const uint64_t remaining = length_ - (static_cast(observed_) * kMaxFrameGroupPayloadBytes); + if (frame.group.role != role || frame.group.value != value || frame.group.count != number || + frame.payload.size() != std::min(remaining, kMaxFrameGroupPayloadBytes)) { + return FrameError::InvalidGroup; + } + if (frame.physical_length > std::numeric_limits::max() - next_offset_) { + return FrameError::LengthOverflow; + } + next_offset_ += frame.physical_length; + ++observed_; + return FrameError::Ok; + } + + [[nodiscard]] bool complete() const + { + return count_ != 0 && observed_ == count_; + } + + [[nodiscard]] FrameError finish(ParsedAppendGroup *out) const + { + if (!complete()) { + return FrameError::InvalidGroup; + } + if (out != nullptr) { + *out = {.logical_length = length_, .physical_length = next_offset_ - head_offset_, .frame_count = count_}; + } + return FrameError::Ok; + } + + private: + uint64_t head_offset_ = 0; + uint64_t next_offset_ = 0; + uint64_t length_ = 0; + uint64_t write_time_ = 0; + uint32_t count_ = 0; + uint32_t observed_ = 0; + FrameChunkId chunk_; +}; + +[[nodiscard]] inline FrameError encode_group_frame(FrameChunkId chunk, std::span payload, + uint64_t write_time, FrameGroupFields group, + std::vector *out) +{ + if (out == nullptr || payload.empty() || payload.size() > kMaxFrameGroupPayloadBytes || + (group.role != FrameRole::Head && group.role != FrameRole::Follower) || group.count == 0 || + (group.role == FrameRole::Head && group.value == 0)) { + return FrameError::InvalidGroup; + } + out->clear(); + out->reserve(kFrameGroupHeaderBytes + payload.size() + kFrameFooterBytes); + append_u16_le(out, static_cast(FrameMagic::StreamV2)); + append_u16_le(out, kFrameGroupHeaderBytes); + append_u16_le(out, static_cast(payload.size())); + append_u64_le(out, write_time); + out->push_back(static_cast(group.role)); + append_u64_le(out, group.value); + append_u32_le(out, group.count); + out->insert(out->end(), payload.begin(), payload.end()); + const uint32_t crc = crowdb::common::crc32c(out->data(), out->size()); + append_u32_le(out, crc); + append_u64_be(out, chunk.high); + append_u64_be(out, chunk.low); + return FrameError::Ok; +} + +[[nodiscard]] inline FrameError encode_append_group(FrameChunkId chunk, uint64_t head_offset, + std::span payload, uint64_t write_time, + std::vector *out) +{ + const size_t physical = framed_physical_length(payload.size(), kFrameGroupHeaderBytes); + const size_t count = + (payload.size() / kMaxFrameGroupPayloadBytes) + (payload.size() % kMaxFrameGroupPayloadBytes != 0 ? 1 : 0); + if (out == nullptr || payload.empty() || physical == 0 || count > std::numeric_limits::max() || + physical > std::numeric_limits::max() - head_offset) { + return FrameError::InvalidGroup; + } + out->clear(); + out->reserve(physical); + size_t offset = 0; + for (size_t ordinal = 0; ordinal < count; ++ordinal) { + const size_t length = std::min(kMaxFrameGroupPayloadBytes, payload.size() - offset); + const FrameGroupFields group{.role = ordinal == 0 ? FrameRole::Head : FrameRole::Follower, + .value = ordinal == 0 ? payload.size() : head_offset, + .count = static_cast(ordinal == 0 ? count : ordinal)}; + std::vector frame; + const FrameError error = encode_group_frame(chunk, payload.subspan(offset, length), write_time, group, &frame); + if (error != FrameError::Ok) { + return error; + } + out->insert(out->end(), frame.begin(), frame.end()); + offset += length; + } + return FrameError::Ok; +} + +[[nodiscard]] inline FrameError parse_append_group(std::span bytes, FrameChunkId chunk, + uint64_t head_offset, ParsedAppendGroup *out) +{ + ParsedFrame head; + FrameError error = parse_frame(bytes, chunk, &head); + if (error != FrameError::Ok) { + return error; + } + AppendGroupValidator validator; + error = validator.start(head_offset, head); + if (error != FrameError::Ok) { + return error; + } + size_t cursor = head.physical_length; + while (!validator.complete()) { + ParsedFrame frame; + error = parse_frame(bytes.subspan(cursor), chunk, &frame); + if (error != FrameError::Ok) { + return error; + } + if (cursor > std::numeric_limits::max() - head_offset) { + return FrameError::LengthOverflow; + } + error = validator.push(head_offset + cursor, frame); + if (error != FrameError::Ok) { + return error; + } + cursor += frame.physical_length; + } + return validator.finish(out); +} + +} // namespace crowdb::protocol diff --git a/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/parsing.h b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/parsing.h new file mode 100644 index 000000000..504aa50cd --- /dev/null +++ b/lib/crowdb-protocol/cpp/include/crowdb-protocol/frame/parsing.h @@ -0,0 +1,151 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-protocol/frame/base.h" +#include "crowdb-protocol/frame/encoding.h" + +namespace crowdb::protocol +{ + +[[nodiscard]] inline FrameError parse_frame_role(std::span bytes, FrameMagic magic, + size_t payload_offset, FrameGroupFields *out) +{ + if (payload_offset == kFrameV2HeaderBytes && bytes[14] == 0 && magic != FrameMagic::StreamV2) { + *out = {}; + return FrameError::Ok; + } + if (payload_offset != kFrameGroupHeaderBytes || magic != FrameMagic::StreamV2 || + (bytes[14] != 1 && bytes[14] != 2)) { + return FrameError::InvalidRole; + } + *out = {.role = static_cast(bytes[14]), + .value = read_u64_le(bytes.data() + 15), + .count = read_u32_le(bytes.data() + 23)}; + if (out->count == 0 || (out->role == FrameRole::Head && out->value == 0)) { + return FrameError::InvalidGroup; + } + return FrameError::Ok; +} + +[[nodiscard]] inline FrameError parse_frame(std::span bytes, FrameChunkId expected_chunk_id, + ParsedFrame *out) +{ + if (bytes.size() < kFrameHeaderPrefixBytes) { + return FrameError::Incomplete; + } + const uint16_t magic_value = read_u16_le(bytes.data()); + const auto parsed_magic = frame_magic(magic_value); + if (!parsed_magic.has_value()) { + return FrameError::UnknownMagic; + } + const uint16_t payload_offset = read_u16_le(bytes.data() + 2); + const uint16_t payload_size = read_u16_le(bytes.data() + 4); + if (payload_offset < kFrameHeaderPrefixBytes) { + return FrameError::InvalidPayloadOffset; + } + if (payload_offset != kFrameV2HeaderBytes && payload_offset != kFrameGroupHeaderBytes) { + return FrameError::InvalidPayloadOffset; + } + const size_t physical_length = static_cast(payload_offset) + payload_size + kFrameFooterBytes; + if (physical_length > kMaxFrameBytes) { + return FrameError::FrameTooLarge; + } + if (bytes.size() < physical_length) { + return FrameError::Incomplete; + } + const size_t footer_start = physical_length - kFrameFooterBytes; + const auto magic = *parsed_magic; + FrameGroupFields group; + const FrameError role_error = parse_frame_role(bytes, magic, payload_offset, &group); + if (role_error != FrameError::Ok) { + return role_error; + } + const uint32_t crc = crowdb::common::crc32c(bytes.data(), footer_start); + const uint32_t expected_crc = read_u32_le(bytes.data() + footer_start); + if (crc != expected_crc) { + return FrameError::ChecksumMismatch; + } + const FrameChunkId chunk_id{ + .high = read_u64_be(bytes.data() + footer_start + 4), + .low = read_u64_be(bytes.data() + footer_start + 12), + }; + if (chunk_id != expected_chunk_id) { + return FrameError::ChunkIdMismatch; + } + if (out != nullptr) { + *out = ParsedFrame{ + .header = + { + .magic = *parsed_magic, + .payload_offset = payload_offset, + .payload_size = payload_size, + .write_time_ms = read_u64_le(bytes.data() + 6), + }, + .group = group, + .payload = bytes.subspan(payload_offset, payload_size), + .chunk_id = chunk_id, + .physical_length = physical_length, + }; + } + return FrameError::Ok; +} + +[[nodiscard]] inline FrameError parse_frames(std::span bytes, FrameChunkId expected_chunk_id, + FrameMagic expected_magic, size_t logical_length, + std::vector *payload) +{ + if (payload == nullptr || framed_physical_length(logical_length, kFrameV2HeaderBytes) != bytes.size()) { + return FrameError::Incomplete; + } + payload->clear(); + payload->reserve(logical_length); + size_t offset = 0; + while (offset < bytes.size()) { + ParsedFrame frame; + const FrameError error = parse_frame(bytes.subspan(offset), expected_chunk_id, &frame); + if (error != FrameError::Ok) { + return error; + } + if (frame.header.magic != expected_magic || frame.physical_length > bytes.size() - offset) { + return FrameError::UnknownMagic; + } + payload->insert(payload->end(), frame.payload.begin(), frame.payload.end()); + offset += frame.physical_length; + } + return payload->size() == logical_length ? FrameError::Ok : FrameError::Incomplete; +} + +/// Validates a concatenated sequence whose first frame already identifies its +/// public kind. DiskIO uses this boundary check without needing caller-private +/// location metadata. This checks framing/CRC only; the footer supplies its claimed +/// identity. Content readers must pass their external expected chunk ID to parse_frame. +[[nodiscard]] inline FrameError validate_frame_sequence(std::span bytes) +{ + size_t offset = 0; + while (offset < bytes.size()) { + const auto remaining = bytes.subspan(offset); + if (remaining.size() < kFrameHeaderPrefixBytes) { + return FrameError::Incomplete; + } + const size_t physical_length = static_cast(read_u16_le(remaining.data() + 2)) + + read_u16_le(remaining.data() + 4) + kFrameFooterBytes; + if (physical_length > remaining.size() || physical_length < kFrameHeaderPrefixBytes + kFrameFooterBytes) { + return FrameError::Incomplete; + } + const size_t footer_start = physical_length - kFrameFooterBytes; + ParsedFrame frame; + const FrameError error = parse_frame(remaining.first(physical_length), + {.high = read_u64_be(remaining.data() + footer_start + 4), + .low = read_u64_be(remaining.data() + footer_start + 12)}, + &frame); + if (error != FrameError::Ok) { + return error; + } + offset += frame.physical_length; + } + return FrameError::Ok; +} + +} // namespace crowdb::protocol diff --git a/lib/crowdb-protocol/cpp/include/crowdb-protocol/timing.h b/lib/crowdb-protocol/cpp/include/crowdb-protocol/timing.h index 6b8712c11..743e8c143 100644 --- a/lib/crowdb-protocol/cpp/include/crowdb-protocol/timing.h +++ b/lib/crowdb-protocol/cpp/include/crowdb-protocol/timing.h @@ -8,10 +8,10 @@ namespace crowdb::protocol { -// System-wide stale-write and owner self-fence policy. Deployments tune these -// values as one policy shared by DiskIO and ChunkDB. -inline constexpr uint64_t kDefaultMaxWriteRequestAgeMs = 30'000; -inline constexpr uint64_t kDefaultMaxClockSkewMs = 1'000; -inline constexpr uint64_t kDefaultSelfFenceMarginMs = 1'000; +// Shared write-admission ceilings used by DiskIO and tail recovery. +// A server may enforce shorter windows but cannot extend these bounds. +inline constexpr uint64_t kMaxWriteRequestAgeMs = 30'000; +inline constexpr uint64_t kMaxWriteClockSkewMs = 1'000; +inline constexpr uint64_t kDefaultSelfFenceMarginMs = 1'000; } // namespace crowdb::protocol diff --git a/lib/crowdb-protocol/src/chunk_kv.rs b/lib/crowdb-protocol/src/chunk_kv.rs index c366d65c9..d5fac362d 100644 --- a/lib/crowdb-protocol/src/chunk_kv.rs +++ b/lib/crowdb-protocol/src/chunk_kv.rs @@ -56,7 +56,7 @@ pub struct TailOverlayArtifact { pub source_partition_id: Id128, pub source_epoch: u64, pub source_stream_name: StreamName, - pub source_stream_manifest_generation: u64, + pub source_stream_generation: u64, pub replay_offset: u64, pub cutover_offset: u64, pub base_root_manifest_generation: u64, @@ -362,6 +362,7 @@ pub struct HostedPartition { #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct ChunkKvInstanceObservation { pub instance_id: u64, + pub process_epoch: u64, pub rpc_endpoint: String, pub last_heartbeat_ms: u64, pub capacity_bytes: u64, @@ -547,8 +548,10 @@ pub struct ServingAssignment { } #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct ServingGrant { pub instance_id: u64, + pub process_epoch: u64, pub lease_sequence: u64, pub catalog_generation: u64, pub issued_at_ms: u64, @@ -911,6 +914,7 @@ impl ServingGrant { /// Returns an error for malformed or ambiguous authority. pub fn validate(&self) -> Result<(), ChunkKvProtocolError> { if self.instance_id == 0 + || self.process_epoch == 0 || self.lease_sequence == 0 || self.catalog_generation == 0 || self.expires_at_ms <= self.issued_at_ms @@ -971,7 +975,7 @@ fn valid_tail_overlay(overlay: &TailOverlayArtifact) -> bool { overlay.source_partition_id != Id128::default() && overlay.source_epoch != 0 && overlay.source_stream_name != StreamName::default() - && overlay.source_stream_manifest_generation != 0 + && overlay.source_stream_generation != 0 && overlay.base_root_manifest_generation != 0 && overlay.replay_offset <= overlay.cutover_offset && overlay.base_applied_seq <= overlay.cutover_seq @@ -1042,7 +1046,7 @@ fn same_transfer_overlay_base(left: &TailOverlayArtifact, right: &TailOverlayArt left.source_partition_id == right.source_partition_id && left.source_epoch == right.source_epoch && left.source_stream_name == right.source_stream_name - && left.source_stream_manifest_generation == right.source_stream_manifest_generation + && left.source_stream_generation == right.source_stream_generation && left.replay_offset == right.replay_offset && left.base_root_manifest_generation == right.base_root_manifest_generation && left.base_tree_manifest == right.base_tree_manifest diff --git a/lib/crowdb-protocol/src/chunk_kv/split.rs b/lib/crowdb-protocol/src/chunk_kv/split.rs index 949e404e6..e755b0830 100644 --- a/lib/crowdb-protocol/src/chunk_kv/split.rs +++ b/lib/crowdb-protocol/src/chunk_kv/split.rs @@ -29,6 +29,7 @@ pub struct SplitChildAssignment { } #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct SplitReadinessProof { pub cutover_seq: u64, pub parent_next_epoch: u64, @@ -40,13 +41,11 @@ pub struct SplitReadinessProof { pub child_applied_seq: u64, pub child_tree_manifest: u64, pub child_root_manifest_generation: u64, - /// Historical dependency for legacy replacement-parent records only. - #[serde(default)] - pub retained_parent_tail_overlay: Option, pub child_tail_overlay: TailOverlayArtifact, } #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct SplitTransition { pub transition_id: Id128, pub parent_id: Id128, @@ -59,11 +58,9 @@ pub struct SplitTransition { pub parent_next_epoch: u64, pub split_key: Vec, pub child: SplitChildAssignment, - #[serde(default)] pub planned_at_ms: u64, pub phase: SplitPhase, /// Durable local handoff evidence, independent of final readiness. - #[serde(default)] pub handoff_proof: Option, pub readiness_proof: Option, pub failure: Option, @@ -112,6 +109,9 @@ impl SplitTransition { && self.parent_next_epoch > self.parent_epoch && valid_artifact(&self.parent_artifact) && valid_artifact(&self.retained_parent_artifact) + && self.retained_parent_artifact.tree_id == self.parent_artifact.tree_id + && self.retained_parent_artifact.stream_name == self.parent_artifact.stream_name + && self.retained_parent_artifact.tail_overlay.is_none() && valid_split_child(&self.child); let exact_ranges = self.split_key > self.parent_range.start && self @@ -155,8 +155,6 @@ impl SplitTransition { Some("zero child tree manifest") } else if proof.child_root_manifest_generation == 0 { Some("zero child root manifest generation") - } else if !self.valid_retained_parent(proof) { - Some("retained-parent recovery artifact") } else if !self.valid_split_overlay(&proof.child_tail_overlay, &self.child, proof.cutover_seq) { Some("child tail overlay") } else { @@ -183,26 +181,6 @@ impl SplitTransition { Ok(()) } - fn valid_retained_parent(&self, proof: &SplitReadinessProof) -> bool { - if self.retained_parent_artifact.tree_id == self.parent_artifact.tree_id - && self.retained_parent_artifact.stream_name == self.parent_artifact.stream_name - { - return self.retained_parent_artifact.tail_overlay.is_none() - && proof.retained_parent_tail_overlay.is_none(); - } - proof - .retained_parent_tail_overlay - .as_ref() - .is_some_and(|overlay| { - valid_tail_overlay(overlay) - && self.retained_parent_artifact.tail_overlay.as_ref() == Some(overlay) - && overlay.source_partition_id == self.parent_id - && overlay.source_epoch == self.parent_epoch - && overlay.source_stream_name == self.parent_artifact.stream_name - && overlay.cutover_seq == proof.cutover_seq - }) - } - fn validate_handoff(&self) -> Result<(), ChunkKvProtocolError> { if self.handoff_proof.as_ref().is_some_and(|proof| { !valid_tail_overlay(proof) @@ -235,7 +213,7 @@ fn preserves_handoff_base(handoff: &TailOverlayArtifact, ready: &TailOverlayArti handoff.source_partition_id == ready.source_partition_id && handoff.source_epoch == ready.source_epoch && handoff.source_stream_name == ready.source_stream_name - && handoff.source_stream_manifest_generation == ready.source_stream_manifest_generation + && handoff.source_stream_generation == ready.source_stream_generation && handoff.replay_offset == ready.replay_offset && handoff.base_root_manifest_generation == ready.base_root_manifest_generation && handoff.base_tree_manifest == ready.base_tree_manifest diff --git a/lib/crowdb-protocol/src/chunk_stream.rs b/lib/crowdb-protocol/src/chunk_stream.rs index bcc8e81fc..1025bba05 100644 --- a/lib/crowdb-protocol/src/chunk_stream.rs +++ b/lib/crowdb-protocol/src/chunk_stream.rs @@ -9,7 +9,13 @@ use serde::{Deserialize, Serialize}; use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; -/// Durable purpose of a logical stream; absent legacy values are not inferred. +mod chunk_record; +mod ownership; + +pub use chunk_record::*; +pub use ownership::{StreamLeaseRecordError, StreamProcessIdentity, StreamProcessLease, StreamWriterOwner}; + +/// Durable purpose of a logical stream. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] pub enum StreamPurpose { Wal, @@ -118,7 +124,7 @@ mod tests { assert!(first < second); assert_eq!(first.to_string().len(), 32); assert_eq!( - super::StreamBinding::creating(first, None, super::StreamPurpose::Stream).metadata_group_id, + super::StreamRecord::creating(first, None, super::StreamPurpose::Stream).metadata_group_id, super::DEFAULT_STREAM_METADATA_GROUP_ID ); } @@ -142,7 +148,7 @@ impl std::fmt::Display for StreamName { } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub enum StreamBindingState { +pub enum StreamRecordState { #[default] Creating, Active, @@ -150,25 +156,49 @@ pub enum StreamBindingState { } #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct StreamBinding { +#[serde(deny_unknown_fields)] +pub struct StreamRecord { pub purpose: StreamPurpose, pub stream_name: StreamName, pub metadata_group_id: u64, - pub binding_generation: u64, - pub state: StreamBindingState, - pub owner_kind: Option, + pub record_version: u64, + pub state: StreamRecordState, + pub consumer_kind: Option, + pub consumer_id: Option, + pub created_at_ms: u64, } -impl StreamBinding { - /// Creates an inactive binding in the default nonzero metadata group. +impl StreamRecord { + #[must_use] + pub fn creation_time_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(1, |duration| { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX).max(1) + }) + } + + /// Stable identity comparison for idempotent creation; creation time is retained. #[must_use] - pub fn creating(stream_name: StreamName, owner_kind: Option, purpose: StreamPurpose) -> Self { + pub fn same_identity(&self, other: &Self) -> bool { + self.purpose == other.purpose + && self.stream_name == other.stream_name + && self.metadata_group_id == other.metadata_group_id + && self.consumer_kind == other.consumer_kind + && self.consumer_id == other.consumer_id + } + + /// Creates an inactive identity record in the default nonzero metadata group. + #[must_use] + pub fn creating(stream_name: StreamName, consumer_kind: Option, purpose: StreamPurpose) -> Self { Self { stream_name, metadata_group_id: DEFAULT_STREAM_METADATA_GROUP_ID, - binding_generation: 1, - state: StreamBindingState::Creating, - owner_kind, + record_version: 1, + state: StreamRecordState::Creating, + consumer_kind, + consumer_id: None, + created_at_ms: Self::creation_time_ms(), purpose, } } @@ -182,42 +212,3 @@ pub struct ActiveChunkDescriptor { pub acknowledged_cursor: u64, pub capacity: u64, } - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct StreamExtentPageFence { - pub page_index: u64, - pub first_logical: u64, - pub end_logical: u64, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct StreamManifest { - pub purpose: StreamPurpose, - pub stream_name: StreamName, - pub metadata_group_id: u64, - pub writer_epoch: u64, - pub generation: u64, - pub trim_offset: u64, - pub sealed_tail: u64, - pub active: Option, - pub extent_pages: Vec, - pub previous_generation: Option, - pub closed: bool, -} - -/// Parallel logical-to-physical extent arrays. Entry `i` maps -/// `[logical_offsets[i], logical_offsets[i + 1])` into `chunk_ids[i]` -/// starting at `physical_offsets[i]`. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct StreamExtentPage { - pub stream_name: StreamName, - pub writer_epoch: u64, - pub generation: u64, - pub page_index: u64, - pub chunk_ids: Vec, - pub logical_offsets: Vec, - /// Physical start of each complete public Stream frame. - pub physical_offsets: Vec, - /// Complete physical frame length, including its public header and footer. - pub frame_lengths: Vec, -} diff --git a/lib/crowdb-protocol/src/chunk_stream/chunk_record.rs b/lib/crowdb-protocol/src/chunk_stream/chunk_record.rs new file mode 100644 index 000000000..510c9125f --- /dev/null +++ b/lib/crowdb-protocol/src/chunk_stream/chunk_record.rs @@ -0,0 +1,82 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Ordered per-chunk Stream metadata and complete-append mappings. + +use super::{StreamName, StreamPurpose}; +use crate::common::ChunkId; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +mod codec; +mod mappings; +pub use codec::{decode_stream_chunk_record, encode_stream_chunk_record}; +pub use mappings::{StreamMapping, StreamMappingIndex, StreamMappings}; + +pub const MAX_STREAM_CHUNK_BYTES: u64 = 256 * 1024 * 1024; +pub const MAX_STREAM_CHUNK_RECORD_BYTES: usize = 24 * 1024 * 1024; + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum StreamChunkRecordError { + #[error("invalid Stream chunk record: {0}")] + Invalid(&'static str), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum StreamChunkState { + Candidate, + Active, + Sealed, + Deleting, +} + +/// Logical ranges are assigned only when a candidate becomes attached. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct StreamChunkRecord { + pub stream_id: StreamName, + pub purpose: StreamPurpose, + pub chunk_id: ChunkId, + pub state: StreamChunkState, + pub logical_start: Option, + pub logical_end: Option, + pub sequence: u64, + pub writer_epoch: u64, + pub generation: u64, + pub owner_session: [u8; 16], + pub trim_offset: u64, + pub closed: bool, + pub mappings: StreamMappings, +} + +impl StreamChunkRecord { + /// # Errors + /// Rejects malformed scope, state, coverage and physical geometry. + pub fn validate(&self) -> Result { + use StreamChunkRecordError::Invalid; + if !self.purpose.matches(self.chunk_id) || self.writer_epoch == 0 || self.owner_session == [0; 16] { + return Err(Invalid("identity or purpose")); + } + if self.state == StreamChunkState::Candidate { + if self.logical_start.is_some() + || self.logical_end.is_some() + || self.closed + || self.generation != 0 + || self.mappings.count() != 0 + || self.trim_offset != 0 + { + return Err(Invalid("candidate has attached progress")); + } + return self.mappings.index(0, 0); + } + let start = self.logical_start.ok_or(Invalid("missing logical start"))?; + let end = self.logical_end.ok_or(Invalid("missing logical end"))?; + if start > end + || self.generation == 0 + || self.trim_offset > end + || (self.closed && self.state == StreamChunkState::Active) + { + return Err(Invalid("logical bounds or state")); + } + self.mappings.index(start, end) + } +} diff --git a/lib/crowdb-protocol/src/chunk_stream/chunk_record/codec.rs b/lib/crowdb-protocol/src/chunk_stream/chunk_record/codec.rs new file mode 100644 index 000000000..38acedc19 --- /dev/null +++ b/lib/crowdb-protocol/src/chunk_stream/chunk_record/codec.rs @@ -0,0 +1,59 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{StreamChunkRecord, StreamChunkRecordError, MAX_STREAM_CHUNK_RECORD_BYTES}; +use bincode::Options; +use StreamChunkRecordError::Invalid; +const HEADER: &[u8; 8] = b"CSCR\x01\x00\x00\x00"; + +/// # Errors +/// Rejects malformed mappings/identity and oversized encoded records. +pub fn encode_stream_chunk_record(record: &StreamChunkRecord) -> Result, StreamChunkRecordError> { + record.validate()?; + let body = bincode::serialize(record).map_err(|_| Invalid("record encoding"))?; + if body.len() > MAX_STREAM_CHUNK_RECORD_BYTES - 16 { + return Err(Invalid("record byte budget")); + } + let mut bytes = Vec::with_capacity(body.len() + 16); + bytes.extend_from_slice(HEADER); + bytes.extend_from_slice( + &u32::try_from(body.len()) + .map_err(|_| Invalid("record length"))? + .to_le_bytes(), + ); + bytes.extend_from_slice(&body); + let checksum = crowdb_common::ec_isal::crc32c_update(0, &bytes); + bytes.extend_from_slice(&checksum.to_le_bytes()); + Ok(bytes) +} + +/// # Errors +/// Rejects unsupported schemas, checksum/length errors and invalid mappings. +pub fn decode_stream_chunk_record(bytes: &[u8]) -> Result { + if bytes.len() < 16 + || bytes.len() > MAX_STREAM_CHUNK_RECORD_BYTES + || bytes.get(..8) != Some(HEADER.as_slice()) + { + return Err(Invalid("record envelope")); + } + let length = u32::from_le_bytes(bytes[8..12].try_into().map_err(|_| Invalid("record length"))?) as usize; + if length != bytes.len() - 16 { + return Err(Invalid("record length")); + } + let checksum = u32::from_le_bytes( + bytes[bytes.len() - 4..] + .try_into() + .map_err(|_| Invalid("record checksum"))?, + ); + if checksum != crowdb_common::ec_isal::crc32c_update(0, &bytes[..bytes.len() - 4]) { + return Err(Invalid("record checksum")); + } + let record: StreamChunkRecord = bincode::DefaultOptions::new() + .with_fixint_encoding() + .with_limit(length as u64) + .reject_trailing_bytes() + .deserialize(&bytes[12..bytes.len() - 4]) + .map_err(|_| Invalid("record decoding"))?; + record.validate()?; + Ok(record) +} diff --git a/lib/crowdb-protocol/src/chunk_stream/chunk_record/mappings.rs b/lib/crowdb-protocol/src/chunk_stream/chunk_record/mappings.rs new file mode 100644 index 000000000..c13419de9 --- /dev/null +++ b/lib/crowdb-protocol/src/chunk_stream/chunk_record/mappings.rs @@ -0,0 +1,228 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Compact append lengths with bounded, in-memory random-seek samples. + +use super::{StreamChunkRecordError, MAX_STREAM_CHUNK_BYTES, MAX_STREAM_CHUNK_RECORD_BYTES}; +use crate::frame::FrameLayout; +use serde::{Deserialize, Serialize}; +use StreamChunkRecordError::Invalid; + +mod block; +use block::{decode_block, encode_block}; +const BLOCK_ENTRIES: usize = 256; +const MAX_ENTRIES: u64 = MAX_STREAM_CHUNK_BYTES / 48; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct StreamMappings { + count: u32, + #[serde(with = "bounded_bytes")] + encoded: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StreamMapping { + pub logical_start: u64, + pub logical_end: u64, + pub physical_start: u64, +} + +#[derive(Clone, Copy, Debug)] +struct SeekSample { + logical: u64, + physical: u64, + encoded: usize, +} + +#[derive(Clone, Debug, Default)] +pub struct StreamMappingIndex { + samples: Vec, + logical_end: u64, + physical_end: u64, +} + +impl StreamMappings { + /// # Errors + /// Rejects empty groups, overflow or geometry beyond one physical chunk. + pub fn new(lengths: &[u32]) -> Result { + if lengths.len() as u64 > MAX_ENTRIES { + return Err(Invalid("mapping count")); + } + let mut encoded = Vec::new(); + for block in lengths.chunks(BLOCK_ENTRIES) { + encode_block(block, &mut encoded)?; + } + let result = Self { + count: u32::try_from(lengths.len()).map_err(|_| Invalid("mapping count"))?, + encoded, + }; + let end = lengths + .iter() + .try_fold(0_u64, |total, length| total.checked_add(u64::from(*length))) + .ok_or(Invalid("logical overflow"))?; + result.index(0, end)?; + Ok(result) + } + + #[must_use] + pub const fn count(&self) -> u32 { + self.count + } + + #[must_use] + pub fn encoded_len(&self) -> usize { + self.encoded.len() + } + + /// Expand only one chunk's append lengths while creating a checkpoint. + /// # Errors + /// Rejects malformed counts/blocks before reserving unbounded memory. + pub fn lengths(&self) -> Result, StreamChunkRecordError> { + if u64::from(self.count) > MAX_ENTRIES { + return Err(Invalid("mapping count")); + } + let mut output = Vec::with_capacity(self.count as usize); + let mut cursor = 0; + while output.len() < self.count as usize { + let block = decode_block(&self.encoded, &mut cursor)?; + if block.len() != (self.count as usize - output.len()).min(BLOCK_ENTRIES) { + return Err(Invalid("noncanonical mapping block count")); + } + output.extend(block); + } + if cursor != self.encoded.len() { + return Err(Invalid("mapping trailing bytes")); + } + Ok(output) + } + + /// Validate the entire record once, creating samples without expanding all offsets. + /// # Errors + /// Rejects corrupt encoding, excessive work, discontinuous bounds or physical overflow. + pub fn index(&self, start: u64, end: u64) -> Result { + if u64::from(self.count) > MAX_ENTRIES || self.encoded.len() > MAX_STREAM_CHUNK_RECORD_BYTES { + return Err(Invalid("mapping budget")); + } + let mut index = StreamMappingIndex { + logical_end: start, + ..Default::default() + }; + let mut encoded = 0; + let mut count = 0; + while count < self.count { + index.samples.push(SeekSample { + logical: index.logical_end, + physical: index.physical_end, + encoded, + }); + let lengths = decode_block(&self.encoded, &mut encoded)?; + if lengths.len() != ((self.count - count) as usize).min(BLOCK_ENTRIES) { + return Err(Invalid("noncanonical mapping block count")); + } + count = count + .checked_add(u32::try_from(lengths.len()).map_err(|_| Invalid("mapping block count"))?) + .ok_or(Invalid("mapping count overflow"))?; + if count > self.count { + return Err(Invalid("mapping count differs")); + } + for length in lengths { + index.logical_end = index + .logical_end + .checked_add(u64::from(length)) + .ok_or(Invalid("logical overflow"))?; + index.physical_end = advance_physical(index.physical_end, length)?; + } + } + if encoded != self.encoded.len() || index.logical_end != end { + return Err(Invalid("mapping coverage or trailing bytes")); + } + Ok(index) + } +} + +impl StreamMappingIndex { + #[must_use] + pub const fn physical_end(&self) -> u64 { + self.physical_end + } + + #[must_use] + pub fn resident_bytes(&self) -> usize { + self.samples.capacity() * std::mem::size_of::() + } + + /// Decode at most 256 complete-append lengths for an arbitrary logical seek. + /// # Errors + /// Rejects a different/corrupt mapping buffer or an out-of-range offset. + pub fn locate( + &self, + mappings: &StreamMappings, + offset: u64, + ) -> Result { + if offset >= self.logical_end { + return Err(Invalid("offset beyond mappings")); + } + let index = self + .samples + .partition_point(|sample| sample.logical <= offset) + .checked_sub(1) + .ok_or(Invalid("offset before mappings"))?; + let mut sample = self.samples[index]; + for length in decode_block(&mappings.encoded, &mut sample.encoded)? { + let end = sample + .logical + .checked_add(u64::from(length)) + .ok_or(Invalid("logical overflow"))?; + if offset < end { + return Ok(StreamMapping { + logical_start: sample.logical, + logical_end: end, + physical_start: sample.physical, + }); + } + sample.logical = end; + sample.physical = advance_physical(sample.physical, length)?; + } + Err(Invalid("seek sample coverage")) + } +} + +fn advance_physical(physical: u64, length: u32) -> Result { + if length == 0 { + return Err(Invalid("empty append")); + } + let length = FrameLayout::StreamGroupV2 + .physical_length(u64::from(length)) + .map_err(|_| Invalid("frame geometry"))?; + physical + .checked_add(length) + .filter(|end| *end <= MAX_STREAM_CHUNK_BYTES) + .ok_or(Invalid("physical chunk overflow")) +} + +mod bounded_bytes { + use super::MAX_STREAM_CHUNK_RECORD_BYTES; + use serde::{ + de::{Error, Visitor}, + Deserializer, Serializer, + }; + pub(super) fn serialize(value: &[u8], serializer: S) -> Result { + serializer.serialize_bytes(value) + } + pub(super) fn deserialize<'de, D: Deserializer<'de>>(decoder: D) -> Result, D::Error> { + struct Bounded; + impl Visitor<'_> for Bounded { + type Value = Vec; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("bounded Stream mappings") + } + fn visit_bytes(self, value: &[u8]) -> Result { + if value.len() > MAX_STREAM_CHUNK_RECORD_BYTES { + return Err(E::custom("mapping bytes exceed chunk record budget")); + } + Ok(value.to_vec()) + } + } + decoder.deserialize_bytes(Bounded) + } +} diff --git a/lib/crowdb-protocol/src/chunk_stream/chunk_record/mappings/block.rs b/lib/crowdb-protocol/src/chunk_stream/chunk_record/mappings/block.rs new file mode 100644 index 000000000..4092cd90e --- /dev/null +++ b/lib/crowdb-protocol/src/chunk_stream/chunk_record/mappings/block.rs @@ -0,0 +1,99 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{Invalid, StreamChunkRecordError, BLOCK_ENTRIES}; + +pub(super) fn encode_block(lengths: &[u32], bytes: &mut Vec) -> Result<(), StreamChunkRecordError> { + if lengths.is_empty() || lengths.len() > BLOCK_ENTRIES || lengths.contains(&0) { + return Err(Invalid("mapping block lengths")); + } + bytes.extend_from_slice( + &u16::try_from(lengths.len()) + .map_err(|_| Invalid("mapping block count"))? + .to_le_bytes(), + ); + if lengths.iter().all(|length| *length == lengths[0]) { + bytes.push(0); + bytes.extend_from_slice(&lengths[0].to_le_bytes()); + } else if lengths + .iter() + .map(|length| ((32 - length.leading_zeros()) as usize).div_ceil(7)) + .sum::() + < lengths.len() * 4 + { + bytes.push(1); + for length in lengths { + let mut value = *length; + while value >= 128 { + bytes.push(value.to_le_bytes()[0] | 128); + value >>= 7; + } + bytes.push(value.to_le_bytes()[0]); + } + } else { + bytes.push(2); + for length in lengths { + bytes.extend_from_slice(&length.to_le_bytes()); + } + } + Ok(()) +} + +pub(super) fn decode_block(bytes: &[u8], cursor: &mut usize) -> Result, StreamChunkRecordError> { + let header = bytes + .get(*cursor..cursor.checked_add(3).ok_or(Invalid("mapping cursor overflow"))?) + .ok_or(Invalid("truncated mapping block"))?; + *cursor += 3; + let count = usize::from(u16::from_le_bytes([header[0], header[1]])); + if count == 0 || count > BLOCK_ENTRIES { + return Err(Invalid("mapping block count")); + } + let mut lengths = Vec::with_capacity(count); + match header[2] { + 0 => lengths.resize(count, read_u32(bytes, cursor)?), + 1 => { + for _ in 0..count { + lengths.push(read_varint(bytes, cursor)?); + } + } + 2 => { + for _ in 0..count { + lengths.push(read_u32(bytes, cursor)?); + } + } + _ => return Err(Invalid("mapping encoding")), + } + if lengths.contains(&0) { + return Err(Invalid("empty append")); + } + Ok(lengths) +} + +fn read_u32(bytes: &[u8], cursor: &mut usize) -> Result { + let value = bytes + .get(*cursor..cursor.checked_add(4).ok_or(Invalid("mapping cursor overflow"))?) + .ok_or(Invalid("truncated mapping length"))?; + *cursor += 4; + Ok(u32::from_le_bytes( + value.try_into().map_err(|_| Invalid("mapping length"))?, + )) +} + +fn read_varint(bytes: &[u8], cursor: &mut usize) -> Result { + let mut result = 0; + for ordinal in 0..5 { + let value = *bytes.get(*cursor).ok_or(Invalid("truncated mapping varint"))?; + *cursor += 1; + if ordinal == 4 && value > 15 { + return Err(Invalid("mapping varint overflow")); + } + result |= u32::from(value & 127) << (ordinal * 7); + if value < 128 { + if ordinal > 0 && value == 0 { + return Err(Invalid("noncanonical mapping varint")); + } + return Ok(result); + } + } + Err(Invalid("mapping varint overflow")) +} diff --git a/lib/crowdb-protocol/src/chunk_stream/ownership.rs b/lib/crowdb-protocol/src/chunk_stream/ownership.rs new file mode 100644 index 000000000..e6680f010 --- /dev/null +++ b/lib/crowdb-protocol/src/chunk_stream/ownership.rs @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Group-0 process identities and lease records for Stream owners. + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use super::StreamPurpose; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StreamProcessIdentity { + pub purpose: StreamPurpose, + pub instance_id: u64, + pub process_epoch: u64, + pub boot_nonce: [u8; 16], +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StreamProcessLease { + pub identity: StreamProcessIdentity, + pub expires_at_ms: u64, + pub prior_exclusion_deadline_ms: u64, + pub grant_sequence: u64, + pub grant_expires_at_ms: u64, +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum StreamLeaseRecordError { + #[error("invalid Stream process lease identity or progress")] + InvalidRecord, + #[error("cannot generate a Stream process boot identity")] + EntropyUnavailable, +} + +impl StreamProcessLease { + /// Generate one fixed boot nonce before capturing its predecessor. + /// # Errors + /// Returns an error when operating-system entropy is unavailable. + pub fn generate_boot_nonce() -> Result<[u8; 16], StreamLeaseRecordError> { + let mut nonce = [0; 16]; + getrandom::getrandom(&mut nonce).map_err(|_| StreamLeaseRecordError::EntropyUnavailable)?; + if nonce == [0; 16] { + return Err(StreamLeaseRecordError::EntropyUnavailable); + } + Ok(nonce) + } + + /// # Errors + /// Rejects absent identity, expiry or inconsistent grant fencing fields. + pub fn validate(&self) -> Result<(), StreamLeaseRecordError> { + let identity = &self.identity; + if identity.instance_id == 0 + || identity.process_epoch == 0 + || identity.boot_nonce == [0; 16] + || self.expires_at_ms == 0 + || (identity.process_epoch == 1) != (self.prior_exclusion_deadline_ms == 0) + || (self.grant_sequence == 0) != (self.grant_expires_at_ms == 0) + || (identity.purpose == StreamPurpose::Stream && self.grant_sequence != 0) + { + return Err(StreamLeaseRecordError::InvalidRecord); + } + Ok(()) + } +} + +/// One stream's owner assignment. Renewal lives in the referenced process row; +/// this record changes only on acquire or drained release. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StreamWriterOwner { + pub stream_name: super::StreamName, + pub writer_epoch: u64, + pub process: StreamProcessIdentity, + pub assignment_nonce: [u8; 16], + pub released: bool, +} + +impl StreamWriterOwner { + /// # Errors + /// Rejects missing writer, process or assignment identity. + pub fn validate(&self) -> Result<(), StreamLeaseRecordError> { + if self.writer_epoch == 0 + || self.process.instance_id == 0 + || self.process.process_epoch == 0 + || self.process.boot_nonce == [0; 16] + || self.assignment_nonce == [0; 16] + { + return Err(StreamLeaseRecordError::InvalidRecord); + } + Ok(()) + } +} diff --git a/lib/crowdb-protocol/src/fb_wrappers/chunkdb.rs b/lib/crowdb-protocol/src/fb_wrappers/chunkdb.rs index 45495c6ba..b0af161da 100644 --- a/lib/crowdb-protocol/src/fb_wrappers/chunkdb.rs +++ b/lib/crowdb-protocol/src/fb_wrappers/chunkdb.rs @@ -3,7 +3,7 @@ #![allow(clippy::must_use_candidate)] //! Zero-copy `FBRef` wrappers for chunkdb response types -//! (design-crowdb-rpc.md §6, R116). +//! for the current `ChunkDB` service contract. //! //! Each wrapper holds a `&[u8]` reference to the control buffer, //! parses the root on construction, and exposes typed accessor methods @@ -13,7 +13,7 @@ use crate::chunkdb_fb::{ FBAdvanceChunkWriteResponse, FBAllocateChunkResponse, FBAppendChunkResponse, FBChunk, FBChunkdbRetCode, FBDeleteChunkRangeResponse, FBDeleteChunkResponse, FBListChunksResponse, FBQueryChunkResponse, - FBSealChunkResponse, FBUpdateChunkStripResponse, + FBSealChunkResponse, }; use flatbuffers::Vector; @@ -308,45 +308,6 @@ impl<'a> FBDeleteChunkRangeResponseRef<'a> { } } -// ── FBUpdateChunkStripResponseRef ──────────────────────────────── - -/// Zero-copy view over an `FBUpdateChunkStripResponse` control buffer. -pub struct FBUpdateChunkStripResponseRef<'a> { - root: Option>, -} - -impl<'a> FBUpdateChunkStripResponseRef<'a> { - pub fn new(buf: &'a [u8]) -> Self { - Self { - root: parse_root::(buf), - } - } - pub fn valid(&self) -> bool { - self.root.is_some() - } - pub fn ret_code(&self) -> FBChunkdbRetCode { - self.root.map_or(FBChunkdbRetCode::Internal, |r| r.ret_code()) - } - pub fn error_msg(&self) -> Option<&'a str> { - self.root.and_then(|r| r.error_msg()) - } - pub fn request_id(&self) -> Option { - self.root.map(|r| r.id()) - } - pub fn ok(&self) -> bool { - self.root.is_some_and(|r| is_ok(r.ret_code())) - } - pub fn range_start(&self) -> u32 { - self.root.map_or(0, |r| r.range_start()) - } - pub fn range_end(&self) -> u32 { - self.root.map_or(0, |r| r.range_end()) - } - pub fn chunk(&self) -> Option> { - self.root.and_then(|r| r.chunk()) - } -} - // ── FBListChunksResponseRef ────────────────────────────────────── /// Zero-copy view over an `FBListChunksResponse` control buffer. diff --git a/lib/crowdb-protocol/src/fbs/chunkdb.fbs b/lib/crowdb-protocol/src/fbs/chunkdb.fbs index 5099fb430..7bd4ae731 100644 --- a/lib/crowdb-protocol/src/fbs/chunkdb.fbs +++ b/lib/crowdb-protocol/src/fbs/chunkdb.fbs @@ -3,9 +3,7 @@ include "diskdb.fbs"; namespace crowdb.chunkdb.proto; -// chunkdb service flatbuffer control messages (R116: legacy → crowdb-rpc -// migration). Mirrors chunkdb_service.proto + chunkdb_op.proto + -// chunkdb_type.proto. +// Current ChunkDB service control messages. // // Conventions (design-crowdb-rpc.md §6, todo_fb.md "Rules to Follow"): // - Every request/response table carries `id` (request_id, matched by @@ -392,7 +390,6 @@ table FBSealChunkRequest { id:uint64; rpc_create_nano:uint64; chunk_id:crowdb.rpc.proto.FBInt128; - seal_length:uint32; seal_bytes:uint64; } @@ -443,26 +440,6 @@ table FBDeleteChunkRangeResponse { range_end:uint32; } -// ── UpdateChunkStrip ───────────────────────────────────────────── - -table FBUpdateChunkStripRequest { - id:uint64; - rpc_create_nano:uint64; - chunk_id:crowdb.rpc.proto.FBInt128; - strip_index:uint32; - strip:FBChunkStrip; -} - -table FBUpdateChunkStripResponse { - id:uint64; - rpc_create_nano:uint64; - ret_code:FBChunkdbRetCode; - error_msg:string; - range_start:uint32; - range_end:uint32; - chunk:FBChunk; -} - // ── Replacement Allocation + Fenced Range Swap ───────────────── table FBAllocateReplacementSegmentRequest { diff --git a/lib/crowdb-protocol/src/fbs/diskio.fbs b/lib/crowdb-protocol/src/fbs/diskio.fbs index defa9f3e9..d5bf1e03a 100644 --- a/lib/crowdb-protocol/src/fbs/diskio.fbs +++ b/lib/crowdb-protocol/src/fbs/diskio.fbs @@ -11,9 +11,8 @@ enum FBDiskIoRetCode : int16 { PartialWrite = 4, InvalidAlignment = 5, ConnectionError = 6, - // Retired response value kept so mixed-version peers do not reuse code 7. - StaleAllocation = 7, - OldRequest = 8, + OldRequest = 7, + FsyncFailed = 8, } // All diskio control messages carry id + rpc_create_nano as their first @@ -29,16 +28,18 @@ table FBDiskWriteRequest { zone_index:uint32; zone_offset:uint64; size:uint32; - // Retired allocation-fence field; keep its slot for wire compatibility. - allocation_ts:uint64 (deprecated); ordering_zone_offset:uint64; write_create_time_ms:uint64; + // Success requires ordered write and flush when true. + sync:bool; } table FBDiskWriteResponse { id:uint64; rpc_create_nano:uint64; ret_code:FBDiskIoRetCode; + // True only after the requested write and disk synchronization both succeed. + sync_completed:bool; } // Read request. test_pattern_offset is used by NullDisk to generate diff --git a/lib/crowdb-protocol/src/fbs/kv_consensus.fbs b/lib/crowdb-protocol/src/fbs/kv_consensus.fbs index 27e596389..1ccc3b399 100644 --- a/lib/crowdb-protocol/src/fbs/kv_consensus.fbs +++ b/lib/crowdb-protocol/src/fbs/kv_consensus.fbs @@ -49,12 +49,14 @@ table FBAcceptedValue { table FBPrepareRequest { id:uint64; rpc_create_nano:uint64; + group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; version:uint32; slot:uint64; round:uint64; leader_id:uint64; term:uint64; - group_id:uint64; membership_epoch:uint64; } @@ -82,13 +84,15 @@ table FBPromiseResponse { table FBAcceptRequest { id:uint64; rpc_create_nano:uint64; + group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; version:uint32; slot:uint64; round:uint64; leader_id:uint64; term:uint64; value:FBAcceptedValue; - group_id:uint64; membership_epoch:uint64; } @@ -115,8 +119,10 @@ table FBAcceptedResponse { table FBPreVoteRequest { id:uint64; rpc_create_nano:uint64; - version:uint32; group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; + version:uint32; term:uint64; candidate_id:uint64; accepted_log_tip_slot:uint64; @@ -140,8 +146,10 @@ table FBPreVoteResponse { table FBRequestVoteRequest { id:uint64; rpc_create_nano:uint64; - version:uint32; group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; + version:uint32; term:uint64; candidate_id:uint64; accepted_log_tip_slot:uint64; @@ -167,8 +175,10 @@ table FBRequestVoteResponse { table FBHeartbeatRequest { id:uint64; rpc_create_nano:uint64; - version:uint32; group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; + version:uint32; term:uint64; leader_id:uint64; prev_log_slot:uint64; @@ -199,8 +209,10 @@ table FBHeartbeatResponse { table FBStepDownRequest { id:uint64; rpc_create_nano:uint64; - version:uint32; group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; + version:uint32; term:uint64; target_leader_id:uint64; reason:string; @@ -223,8 +235,10 @@ table FBStepDownResponse { table FBChosenNotification { id:uint64; rpc_create_nano:uint64; - version:uint32; group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; + version:uint32; slot:uint64; term:uint64; leader_id:uint64; @@ -234,8 +248,10 @@ table FBChosenNotification { table FBBatchChosenNotification { id:uint64; rpc_create_nano:uint64; - version:uint32; group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; + version:uint32; start_slot:uint64; end_slot:uint64; term:uint64; @@ -248,8 +264,10 @@ table FBBatchChosenNotification { table FBFetchGapRequest { id:uint64; rpc_create_nano:uint64; - version:uint32; group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; + version:uint32; slot:uint64; term:uint64; leader_id:uint64; @@ -347,3 +365,12 @@ table FBSnapshotAbortResponse { boot_nonce:uint64; session_number:uint64; } + +// Common prefix used to reject consensus requests routed to another replica. +table FBConsensusTarget { + id:uint64; + rpc_create_nano:uint64; + group_id:uint64; + target_replica_id:uint64; + target_cluster_id:string; +} diff --git a/lib/crowdb-protocol/src/fbs/msg_type.fbs b/lib/crowdb-protocol/src/fbs/msg_type.fbs index 685828c6d..7fdb23937 100644 --- a/lib/crowdb-protocol/src/fbs/msg_type.fbs +++ b/lib/crowdb-protocol/src/fbs/msg_type.fbs @@ -100,7 +100,7 @@ enum FBMsgType : int16 { EWatchUnsubscribe = 1118, EWatchNotify = 1119, EWatchNotifyError = 1120, - // chunkdb service (range 3300s, see chunkdb.fbs — R116) + // ChunkDB service (range 3300s, see chunkdb.fbs). EAllocateChunkRequest = 3300, EAllocateChunkResponse = 3301, EAppendChunkRequest = 3302, @@ -113,8 +113,6 @@ enum FBMsgType : int16 { EDeleteChunkResponse = 3309, EDeleteChunkRangeRequest = 3310, EDeleteChunkRangeResponse = 3311, - EUpdateChunkStripRequest = 3312, - EUpdateChunkStripResponse = 3313, EListChunksRequest = 3314, EListChunksResponse = 3315, EAdvanceChunkWriteRequest = 3316, diff --git a/lib/crowdb-protocol/src/frame.rs b/lib/crowdb-protocol/src/frame.rs index bea3d5a04..72dd732d4 100644 --- a/lib/crowdb-protocol/src/frame.rs +++ b/lib/crowdb-protocol/src/frame.rs @@ -3,26 +3,27 @@ //! Public self-validating chunk frame encoding and location arithmetic. -use std::ops::Range; - use thiserror::Error; use crate::common::ChunkId; +mod scan; +pub use scan::{ + FramePrefixScanner, FrameScanBatch, FrameScanConfig, FrameScanError, FrameScanMode, FrameScanStop, + FrameScanSummary, ScannedFrameExtent, MAX_FRAME_SCAN_EXTENTS, MAX_FRAME_SCAN_INPUT_BYTES, +}; pub const FRAME_HEADER_PREFIX_BYTES: usize = 14; -const FRAME_HEADER_PREFIX_BYTES_U16: u16 = 14; pub const FRAME_FOOTER_BYTES: usize = 20; pub const MAX_FRAME_BYTES: usize = 64 * 1024; -pub const MAX_FRAME_PAYLOAD_BYTES: usize = MAX_FRAME_BYTES - FRAME_HEADER_PREFIX_BYTES - FRAME_FOOTER_BYTES; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] #[repr(u16)] pub enum FrameMagic { - RepoSmallV1 = 0x0101, - RepoLargeV1 = 0x0201, - StreamV1 = 0x0301, - BtreePageV1 = 0x0401, - PageIndexV1 = 0x0501, + RepoSmallV2 = 0x0102, + RepoLargeV2 = 0x0202, + StreamV2 = 0x0302, + BtreePageV2 = 0x0402, + PageIndexV2 = 0x0502, } impl TryFrom for FrameMagic { @@ -30,11 +31,11 @@ impl TryFrom for FrameMagic { fn try_from(value: u16) -> Result { match value { - 0x0101 => Ok(Self::RepoSmallV1), - 0x0201 => Ok(Self::RepoLargeV1), - 0x0301 => Ok(Self::StreamV1), - 0x0401 => Ok(Self::BtreePageV1), - 0x0501 => Ok(Self::PageIndexV1), + 0x0102 => Ok(Self::RepoSmallV2), + 0x0202 => Ok(Self::RepoLargeV2), + 0x0302 => Ok(Self::StreamV2), + 0x0402 => Ok(Self::BtreePageV2), + 0x0502 => Ok(Self::PageIndexV2), _ => Err(FrameError::UnknownMagic(value)), } } @@ -72,10 +73,15 @@ pub enum FrameError { InvalidLocationRange, #[error("locations are not physically contiguous")] NonContiguousLocation, + #[error("frame role or extension is invalid")] + InvalidRole, + #[error("append group is inconsistent or incomplete")] + InvalidGroup, } pub struct ParsedFrame<'a> { pub header: FrameHeaderPrefix, + pub role: FrameRole, pub payload: &'a [u8], pub chunk_id: ChunkId, pub physical_length: usize, @@ -84,486 +90,19 @@ pub struct ParsedFrame<'a> { /// Verified frame metadata when the physical bytes span several buffers. pub struct ParsedFrameViews { pub header: FrameHeaderPrefix, + pub role: FrameRole, pub chunk_id: ChunkId, pub physical_length: usize, } -/// Verify a frame over ordered views without assembling its payload. -/// -/// # Errors -/// Returns an error for an incomplete, malformed, or corrupt frame. -pub fn parse_frame_views( - views: &[&[u8]], - expected_chunk_id: ChunkId, -) -> Result { - let mut header_bytes = [0_u8; FRAME_HEADER_PREFIX_BYTES]; - copy_view_range(views, 0, &mut header_bytes)?; - let header = parse_header(&header_bytes)?; - let length = frame_length(header)?; - let available = views - .iter() - .fold(0_usize, |total, view| total.saturating_add(view.len())); - if available < length { - return Err(FrameError::Incomplete { - required_bytes: length, - }); - } - let footer_start = length - FRAME_FOOTER_BYTES; - let mut footer = [0_u8; FRAME_FOOTER_BYTES]; - copy_view_range(views, footer_start, &mut footer)?; - let expected_crc = u32::from_le_bytes( - footer[..4] - .try_into() - .map_err(|_| FrameError::InvalidRegionLength)?, - ); - let mut crc = 0_u32; - let mut cursor = 0_usize; - for view in views { - let included = footer_start.saturating_sub(cursor).min(view.len()); - if included > 0 { - crc = crowdb_common::ec_isal::crc32c_update(crc, &view[..included]); - } - cursor = cursor.saturating_add(view.len()); - if cursor >= footer_start { - break; - } - } - if crc != expected_crc { - return Err(FrameError::ChecksumMismatch); - } - let chunk_id = ChunkId { - high: u64::from_be_bytes( - footer[4..12] - .try_into() - .map_err(|_| FrameError::InvalidRegionLength)?, - ), - low: u64::from_be_bytes( - footer[12..20] - .try_into() - .map_err(|_| FrameError::InvalidRegionLength)?, - ), - }; - if chunk_id != expected_chunk_id { - return Err(FrameError::ChunkIdMismatch); - } - Ok(ParsedFrameViews { - header, - chunk_id, - physical_length: length, - }) -} - -fn copy_view_range(views: &[&[u8]], start: usize, target: &mut [u8]) -> Result<(), FrameError> { - let required_bytes = start.saturating_add(target.len()); - let mut cursor = 0_usize; - let mut copied = 0_usize; - for view in views { - let view_end = cursor.saturating_add(view.len()); - if view_end > start && copied < target.len() { - let from = start.saturating_sub(cursor); - let count = (view.len() - from).min(target.len() - copied); - target[copied..copied + count].copy_from_slice(&view[from..from + count]); - copied += count; - } - cursor = view_end; - if copied == target.len() { - return Ok(()); - } - } - Err(FrameError::Incomplete { required_bytes }) -} - -/// Encode a canonical v1 frame without header extensions. -/// -/// # Errors -/// -/// Returns [`FrameError::PayloadTooLarge`] when `payload` cannot fit one frame. -pub fn encode_frame( - magic: FrameMagic, - chunk_id: ChunkId, - payload: &[u8], - write_time_ms: u64, -) -> Result, FrameError> { - let mut header = [0; FRAME_HEADER_PREFIX_BYTES]; - let mut footer = [0; FRAME_FOOTER_BYTES]; - encode_frame_regions(magic, chunk_id, payload, write_time_ms, &mut header, &mut footer)?; - let length = header.len() + payload.len() + footer.len(); - let mut frame = Vec::with_capacity(length); - frame.extend_from_slice(&header); - frame.extend_from_slice(payload); - frame.extend_from_slice(&footer); - Ok(frame) -} - -/// Encode one canonical frame into caller-owned header and footer regions. -/// -/// The payload is read in place and is never copied. Native receive owners use -/// this to fill framing bytes reserved around socket-written payload. -/// -/// # Errors -/// -/// Returns an error for an oversized payload or incorrectly sized regions. -pub fn encode_frame_regions( - magic: FrameMagic, - chunk_id: ChunkId, - payload: &[u8], - write_time_ms: u64, - header_region: &mut [u8], - footer_region: &mut [u8], -) -> Result<(), FrameError> { - prepare_frame_regions(magic, payload, write_time_ms, header_region, footer_region)?; - set_frame_chunk_id(chunk_id, footer_region) -} - -/// Prepare the frame header and CRC before its destination chunk is known. -/// The CRC covers the header and payload; the chunk ID is checked separately -/// against the expected location when the frame is read. -/// -/// # Errors -/// Returns an error for an oversized payload or incorrectly sized regions. -pub fn prepare_frame_regions( - magic: FrameMagic, - payload: &[u8], - write_time_ms: u64, - header_region: &mut [u8], - footer_region: &mut [u8], -) -> Result<(), FrameError> { - if header_region.len() != FRAME_HEADER_PREFIX_BYTES || footer_region.len() != FRAME_FOOTER_BYTES { - return Err(FrameError::InvalidRegionLength); - } - let payload_size = u16::try_from(payload.len()).map_err(|_| FrameError::PayloadTooLarge)?; - let header = FrameHeaderPrefix { - magic, - payload_offset: FRAME_HEADER_PREFIX_BYTES_U16, - payload_size, - write_time_ms, - }; - frame_length(header)?; - write_header_region(header_region, header); - let checksum = crc32c_parts([header_region, payload]); - footer_region[..4].copy_from_slice(&checksum.to_le_bytes()); - Ok(()) -} - -/// Frame payload views without copying their bytes into a contiguous allocation. -/// -/// # Errors -/// Returns an error when the total payload cannot fit one canonical frame. -pub fn encode_frame_views( - magic: FrameMagic, - chunk_id: ChunkId, - payload: Vec, - write_time_ms: u64, -) -> Result, FrameError> { - let length = payload.iter().try_fold(0usize, |total, view| { - total.checked_add(view.len()).ok_or(FrameError::LengthOverflow) - })?; - let header = FrameHeaderPrefix { - magic, - payload_offset: FRAME_HEADER_PREFIX_BYTES_U16, - payload_size: u16::try_from(length).map_err(|_| FrameError::PayloadTooLarge)?, - write_time_ms, - }; - frame_length(header)?; - let mut header_region = [0; FRAME_HEADER_PREFIX_BYTES]; - let mut footer_region = [0; FRAME_FOOTER_BYTES]; - write_header_region(&mut header_region, header); - let checksum = crc32c_parts( - std::iter::once(header_region.as_slice()).chain(payload.iter().map(bytes::Bytes::as_ref)), - ); - footer_region[..4].copy_from_slice(&checksum.to_le_bytes()); - set_frame_chunk_id(chunk_id, &mut footer_region)?; - let mut views = Vec::with_capacity(payload.len() + 2); - views.push(bytes::Bytes::copy_from_slice(&header_region)); - views.extend(payload); - views.push(bytes::Bytes::copy_from_slice(&footer_region)); - Ok(views) -} - -/// Set the destination chunk after the frame header and CRC are prepared. -/// -/// # Errors -/// Returns an error for an incorrectly sized footer region. -pub fn set_frame_chunk_id(chunk_id: ChunkId, footer_region: &mut [u8]) -> Result<(), FrameError> { - if footer_region.len() != FRAME_FOOTER_BYTES { - return Err(FrameError::InvalidRegionLength); - } - footer_region[4..12].copy_from_slice(&chunk_id.high.to_be_bytes()); - footer_region[12..20].copy_from_slice(&chunk_id.low.to_be_bytes()); - Ok(()) -} - -/// Encode an object as its consecutive frame sequence. -/// -/// Every interior frame has the maximum payload and the final frame is -/// variable sized. An empty payload has no frame because it has no durable -/// object location. -/// -/// # Errors -/// -/// Returns an error if an individual frame cannot be encoded. -pub fn encode_frames( - magic: FrameMagic, - chunk_id: ChunkId, - payload: &[u8], - write_time_ms: u64, -) -> Result>, FrameError> { - payload - .chunks(MAX_FRAME_PAYLOAD_BYTES) - .map(|part| encode_frame(magic, chunk_id, part, write_time_ms)) - .collect() -} - -/// Parse and verify one complete frame. The expected chunk ID is mandatory so -/// a valid frame copied from a different chunk is rejected. -/// -/// # Errors -/// -/// Returns an error when the frame is incomplete, malformed, belongs to a -/// different chunk, or fails its CRC32C verification. -pub fn parse_frame(bytes: &[u8], expected_chunk_id: ChunkId) -> Result, FrameError> { - let header = parse_header(bytes)?; - let length = frame_length(header)?; - if bytes.len() < length { - return Err(FrameError::Incomplete { - required_bytes: length, - }); - } - let footer_start = length - FRAME_FOOTER_BYTES; - let checksum = u32::from_le_bytes(bytes[footer_start..footer_start + 4].try_into().map_err(|_| { - FrameError::Incomplete { - required_bytes: length, - } - })?); - if crc32c_parts([&bytes[..footer_start]]) != checksum { - return Err(FrameError::ChecksumMismatch); - } - let chunk_id = - ChunkId { - high: u64::from_be_bytes(bytes[footer_start + 4..footer_start + 12].try_into().map_err( - |_| FrameError::Incomplete { - required_bytes: length, - }, - )?), - low: u64::from_be_bytes(bytes[footer_start + 12..length].try_into().map_err(|_| { - FrameError::Incomplete { - required_bytes: length, - } - })?), - }; - if chunk_id != expected_chunk_id { - return Err(FrameError::ChunkIdMismatch); - } - let payload_start = usize::from(header.payload_offset); - let payload_end = payload_start + usize::from(header.payload_size); - Ok(ParsedFrame { - header, - payload: &bytes[payload_start..payload_end], - chunk_id, - physical_length: length, - }) -} - -/// -/// # Errors -/// -/// Returns an error when the prefix is incomplete, has an unknown magic, or -/// declares a payload before the fixed prefix. -pub fn parse_header(bytes: &[u8]) -> Result { - if bytes.len() < FRAME_HEADER_PREFIX_BYTES { - return Err(FrameError::Incomplete { - required_bytes: FRAME_HEADER_PREFIX_BYTES, - }); - } - let magic = FrameMagic::try_from(u16::from_le_bytes(bytes[0..2].try_into().map_err(|_| { - FrameError::Incomplete { - required_bytes: FRAME_HEADER_PREFIX_BYTES, - } - })?))?; - let payload_offset = u16::from_le_bytes(bytes[2..4].try_into().map_err(|_| FrameError::Incomplete { - required_bytes: FRAME_HEADER_PREFIX_BYTES, - })?); - if usize::from(payload_offset) < FRAME_HEADER_PREFIX_BYTES { - return Err(FrameError::InvalidPayloadOffset { - offset: payload_offset, - }); - } - Ok(FrameHeaderPrefix { - magic, - payload_offset, - payload_size: u16::from_le_bytes(bytes[4..6].try_into().map_err(|_| FrameError::Incomplete { - required_bytes: FRAME_HEADER_PREFIX_BYTES, - })?), - write_time_ms: u64::from_le_bytes(bytes[6..14].try_into().map_err(|_| FrameError::Incomplete { - required_bytes: FRAME_HEADER_PREFIX_BYTES, - })?), - }) -} - -/// -/// # Errors -/// -/// Returns an error when the encoded length overflows or exceeds 64 KiB. -pub fn frame_length(header: FrameHeaderPrefix) -> Result { - let length = usize::from(header.payload_offset) - .checked_add(usize::from(header.payload_size)) - .and_then(|value| value.checked_add(FRAME_FOOTER_BYTES)) - .ok_or(FrameError::LengthOverflow)?; - if length > MAX_FRAME_BYTES { - return Err(FrameError::FrameTooLarge); - } - Ok(length) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct ChunkLocation { - pub chunk_id: ChunkId, - pub frame_offset: u64, - pub logical_length: u64, -} - -impl ChunkLocation { - /// Return the total bytes occupied by this location's frames. - /// - /// # Errors - /// - /// Returns an error if the calculated byte count overflows. - pub fn physical_length(self) -> Result { - framed_physical_length(self.logical_length) - } +mod encoding; +mod geometry; +mod group; +mod parsing; +mod revision; - /// Return the exclusive physical end offset. - /// - /// # Errors - /// - /// Returns an error if the frame length or end offset overflows. - pub fn end_offset(self) -> Result { - self.frame_offset - .checked_add(self.physical_length()?) - .ok_or(FrameError::LengthOverflow) - } - - /// Map a logical subrange to the minimal complete-frame physical range. - /// - /// # Errors - /// - /// Returns an error for an invalid subrange or an arithmetic overflow. - pub fn physical_range_for_subrange(self, range: Range) -> Result, FrameError> { - if range.start > range.end || range.end > self.logical_length { - return Err(FrameError::InvalidLocationRange); - } - if range.is_empty() { - return Ok(self.frame_offset..self.frame_offset); - } - let first = range.start / max_payload_u64(); - let last = (range.end - 1) / max_payload_u64(); - let start = self - .frame_offset - .checked_add( - first - .checked_mul(MAX_FRAME_BYTES as u64) - .ok_or(FrameError::LengthOverflow)?, - ) - .ok_or(FrameError::LengthOverflow)?; - let end = if last + 1 == frame_count(self.logical_length) { - self.end_offset()? - } else { - self.frame_offset - .checked_add( - (last + 1) - .checked_mul(MAX_FRAME_BYTES as u64) - .ok_or(FrameError::LengthOverflow)?, - ) - .ok_or(FrameError::LengthOverflow)? - }; - Ok(start..end) - } -} - -/// Merge physically adjacent, frame-aligned locations. -/// -/// # Errors -/// -/// Returns an error when a location length or end offset overflows. -pub fn merge_adjacent_locations(locations: &[ChunkLocation]) -> Result, FrameError> { - let mut merged: Vec = Vec::with_capacity(locations.len()); - for location in locations { - if let Some(previous) = merged.last_mut() { - if previous.chunk_id == location.chunk_id - && previous.logical_length % max_payload_u64() == 0 - && previous.end_offset()? == location.frame_offset - { - previous.logical_length = previous - .logical_length - .checked_add(location.logical_length) - .ok_or(FrameError::LengthOverflow)?; - continue; - } - } - merged.push(*location); - } - Ok(merged) -} - -/// Validate that every location has a nonempty logical payload. -/// -/// # Errors -/// -/// Returns [`FrameError::NonContiguousLocation`] for an empty location. -pub fn validate_contiguous_locations(locations: &[ChunkLocation]) -> Result<(), FrameError> { - for pair in locations.windows(2) { - if pair[0].logical_length == 0 || pair[1].logical_length == 0 { - return Err(FrameError::NonContiguousLocation); - } - } - Ok(()) -} - -/// Return the physical bytes required by the consecutive canonical frames for -/// one logical payload sequence. -/// -/// # Errors -/// -/// Returns an error when the result overflows. -pub fn framed_physical_length(logical_length: u64) -> Result { - if logical_length == 0 { - return Ok(0); - } - let full = logical_length / max_payload_u64(); - let tail = logical_length % max_payload_u64(); - let full_bytes = full - .checked_mul(MAX_FRAME_BYTES as u64) - .ok_or(FrameError::LengthOverflow)?; - if tail == 0 { - Ok(full_bytes) - } else { - full_bytes - .checked_add(FRAME_HEADER_PREFIX_BYTES as u64) - .and_then(|value| value.checked_add(tail)) - .and_then(|value| value.checked_add(FRAME_FOOTER_BYTES as u64)) - .ok_or(FrameError::LengthOverflow) - } -} - -fn frame_count(logical_length: u64) -> u64 { - logical_length.div_ceil(max_payload_u64()) -} - -const fn max_payload_u64() -> u64 { - MAX_FRAME_PAYLOAD_BYTES as u64 -} - -fn write_header_region(region: &mut [u8], header: FrameHeaderPrefix) { - region[0..2].copy_from_slice(&(header.magic as u16).to_le_bytes()); - region[2..4].copy_from_slice(&header.payload_offset.to_le_bytes()); - region[4..6].copy_from_slice(&header.payload_size.to_le_bytes()); - region[6..14].copy_from_slice(&header.write_time_ms.to_le_bytes()); -} - -fn crc32c_parts<'a>(parts: impl IntoIterator) -> u32 { - let mut crc = 0_u32; - for bytes in parts { - crc = crowdb_common::ec_isal::crc32c_update(crc, bytes); - } - crc -} +pub use encoding::*; +pub use geometry::*; +pub use group::*; +pub use parsing::*; +pub use revision::*; diff --git a/lib/crowdb-protocol/src/frame/encoding.rs b/lib/crowdb-protocol/src/frame/encoding.rs new file mode 100644 index 000000000..a35cc86da --- /dev/null +++ b/lib/crowdb-protocol/src/frame/encoding.rs @@ -0,0 +1,171 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Public self-validating chunk frame encoding and location arithmetic. + +use super::{frame_length, ChunkId, FrameError, FrameHeaderPrefix, FrameMagic, FRAME_FOOTER_BYTES}; + +/// Encode one ordinary frame with the selected revision. +/// +/// # Errors +/// +/// Returns [`FrameError::PayloadTooLarge`] when `payload` cannot fit one frame. +pub fn encode_frame( + magic: FrameMagic, + chunk_id: ChunkId, + payload: &[u8], + write_time_ms: u64, +) -> Result, FrameError> { + let mut header = vec![0; super::FRAME_V2_HEADER_BYTES]; + let mut footer = [0; FRAME_FOOTER_BYTES]; + encode_frame_regions(magic, chunk_id, payload, write_time_ms, &mut header, &mut footer)?; + let length = header.len() + payload.len() + footer.len(); + let mut frame = Vec::with_capacity(length); + frame.extend_from_slice(&header); + frame.extend_from_slice(payload); + frame.extend_from_slice(&footer); + Ok(frame) +} + +/// Encode one canonical frame into caller-owned header and footer regions. +/// +/// The payload is read in place and is never copied. Native receive owners use +/// this to fill framing bytes reserved around socket-written payload. +/// +/// # Errors +/// +/// Returns an error for an oversized payload or incorrectly sized regions. +pub fn encode_frame_regions( + magic: FrameMagic, + chunk_id: ChunkId, + payload: &[u8], + write_time_ms: u64, + header_region: &mut [u8], + footer_region: &mut [u8], +) -> Result<(), FrameError> { + prepare_frame_regions(magic, payload, write_time_ms, header_region, footer_region)?; + set_frame_chunk_id(chunk_id, footer_region) +} + +/// Prepare the frame header and CRC before its destination chunk is known. +/// The CRC covers the header and payload; the chunk ID is checked separately +/// against the expected location when the frame is read. +/// +/// # Errors +/// Returns an error for an oversized payload or incorrectly sized regions. +pub fn prepare_frame_regions( + magic: FrameMagic, + payload: &[u8], + write_time_ms: u64, + header_region: &mut [u8], + footer_region: &mut [u8], +) -> Result<(), FrameError> { + if magic == FrameMagic::StreamV2 { + return Err(FrameError::InvalidRole); + } + if header_region.len() != super::FRAME_V2_HEADER_BYTES || footer_region.len() != FRAME_FOOTER_BYTES { + return Err(FrameError::InvalidRegionLength); + } + let payload_size = u16::try_from(payload.len()).map_err(|_| FrameError::PayloadTooLarge)?; + let header = FrameHeaderPrefix { + magic, + payload_offset: u16::try_from(super::FRAME_V2_HEADER_BYTES) + .map_err(|_| FrameError::LengthOverflow)?, + payload_size, + write_time_ms, + }; + frame_length(header)?; + write_header_region(header_region, header); + let checksum = crc32c_parts([header_region, payload]); + footer_region[..4].copy_from_slice(&checksum.to_le_bytes()); + Ok(()) +} + +/// Frame payload views without copying their bytes into a contiguous allocation. +/// +/// # Errors +/// Returns an error when the total payload cannot fit one canonical frame. +pub fn encode_frame_views( + magic: FrameMagic, + chunk_id: ChunkId, + payload: Vec, + write_time_ms: u64, +) -> Result, FrameError> { + if magic == FrameMagic::StreamV2 { + return Err(FrameError::InvalidRole); + } + let length = payload.iter().try_fold(0usize, |total, view| { + total.checked_add(view.len()).ok_or(FrameError::LengthOverflow) + })?; + let header = FrameHeaderPrefix { + magic, + payload_offset: u16::try_from(super::FRAME_V2_HEADER_BYTES) + .map_err(|_| FrameError::LengthOverflow)?, + payload_size: u16::try_from(length).map_err(|_| FrameError::PayloadTooLarge)?, + write_time_ms, + }; + frame_length(header)?; + let mut header_region = vec![0; super::FRAME_V2_HEADER_BYTES]; + let mut footer_region = [0; FRAME_FOOTER_BYTES]; + write_header_region(&mut header_region, header); + let checksum = crc32c_parts( + std::iter::once(header_region.as_slice()).chain(payload.iter().map(bytes::Bytes::as_ref)), + ); + footer_region[..4].copy_from_slice(&checksum.to_le_bytes()); + set_frame_chunk_id(chunk_id, &mut footer_region)?; + let mut views = Vec::with_capacity(payload.len() + 2); + views.push(bytes::Bytes::copy_from_slice(&header_region)); + views.extend(payload); + views.push(bytes::Bytes::copy_from_slice(&footer_region)); + Ok(views) +} + +/// Set the destination chunk after the frame header and CRC are prepared. +/// +/// # Errors +/// Returns an error for an incorrectly sized footer region. +pub fn set_frame_chunk_id(chunk_id: ChunkId, footer_region: &mut [u8]) -> Result<(), FrameError> { + if footer_region.len() != FRAME_FOOTER_BYTES { + return Err(FrameError::InvalidRegionLength); + } + footer_region[4..12].copy_from_slice(&chunk_id.high.to_be_bytes()); + footer_region[12..20].copy_from_slice(&chunk_id.low.to_be_bytes()); + Ok(()) +} + +/// Encode an object as its consecutive frame sequence. +/// +/// Every interior frame has the maximum payload and the final frame is +/// variable sized. An empty payload has no frame because it has no durable +/// object location. +/// +/// # Errors +/// +/// Returns an error if an individual frame cannot be encoded. +pub fn encode_frames( + magic: FrameMagic, + chunk_id: ChunkId, + payload: &[u8], + write_time_ms: u64, +) -> Result>, FrameError> { + payload + .chunks(super::MAX_FRAME_V2_PAYLOAD_BYTES) + .map(|part| encode_frame(magic, chunk_id, part, write_time_ms)) + .collect() +} + +pub(super) fn write_header_region(region: &mut [u8], header: FrameHeaderPrefix) { + region[0..2].copy_from_slice(&(header.magic as u16).to_le_bytes()); + region[2..4].copy_from_slice(&header.payload_offset.to_le_bytes()); + region[4..6].copy_from_slice(&header.payload_size.to_le_bytes()); + region[6..14].copy_from_slice(&header.write_time_ms.to_le_bytes()); + region[14] = 0; +} + +pub(super) fn crc32c_parts<'a>(parts: impl IntoIterator) -> u32 { + let mut crc = 0_u32; + for bytes in parts { + crc = crowdb_common::ec_isal::crc32c_update(crc, bytes); + } + crc +} diff --git a/lib/crowdb-protocol/src/frame/geometry.rs b/lib/crowdb-protocol/src/frame/geometry.rs new file mode 100644 index 000000000..de97ae540 --- /dev/null +++ b/lib/crowdb-protocol/src/frame/geometry.rs @@ -0,0 +1,149 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Public self-validating chunk frame encoding and location arithmetic. + +use std::ops::Range; + +use super::{ + ChunkId, FrameError, FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_BYTES, + MAX_FRAME_V2_PAYLOAD_BYTES, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct ChunkLocation { + pub chunk_id: ChunkId, + pub frame_offset: u64, + pub logical_length: u64, +} + +impl ChunkLocation { + /// Return the total bytes occupied by this location's ordinary frames. + /// + /// # Errors + /// + /// Returns an error if the calculated byte count overflows. + pub fn physical_length(self) -> Result { + framed_physical_length(self.logical_length) + } + + /// Return the exclusive physical end offset. + /// + /// # Errors + /// + /// Returns an error if the frame length or end offset overflows. + pub fn end_offset(self) -> Result { + self.frame_offset + .checked_add(self.physical_length()?) + .ok_or(FrameError::LengthOverflow) + } + + /// Map a logical subrange to the minimal complete-frame physical range. + /// + /// # Errors + /// + /// Returns an error for an invalid subrange or an arithmetic overflow. + pub fn physical_range_for_subrange(self, range: Range) -> Result, FrameError> { + if range.start > range.end || range.end > self.logical_length { + return Err(FrameError::InvalidLocationRange); + } + if range.is_empty() { + return Ok(self.frame_offset..self.frame_offset); + } + let first = range.start / max_payload_u64(); + let last = (range.end - 1) / max_payload_u64(); + let start = self + .frame_offset + .checked_add( + first + .checked_mul(MAX_FRAME_BYTES as u64) + .ok_or(FrameError::LengthOverflow)?, + ) + .ok_or(FrameError::LengthOverflow)?; + let end = if last + 1 == frame_count(self.logical_length) { + self.end_offset()? + } else { + self.frame_offset + .checked_add( + (last + 1) + .checked_mul(MAX_FRAME_BYTES as u64) + .ok_or(FrameError::LengthOverflow)?, + ) + .ok_or(FrameError::LengthOverflow)? + }; + Ok(start..end) + } +} + +/// Merge physically adjacent, frame-aligned locations. +/// +/// # Errors +/// +/// Returns an error when a location length or end offset overflows. +pub fn merge_adjacent_locations(locations: &[ChunkLocation]) -> Result, FrameError> { + let mut merged: Vec = Vec::with_capacity(locations.len()); + for location in locations { + if let Some(previous) = merged.last_mut() { + if previous.chunk_id == location.chunk_id + && previous.logical_length % max_payload_u64() == 0 + && previous.end_offset()? == location.frame_offset + { + previous.logical_length = previous + .logical_length + .checked_add(location.logical_length) + .ok_or(FrameError::LengthOverflow)?; + continue; + } + } + merged.push(*location); + } + Ok(merged) +} + +/// Validate that every location has a nonempty logical payload. +/// +/// # Errors +/// +/// Returns [`FrameError::NonContiguousLocation`] for an empty location. +pub fn validate_contiguous_locations(locations: &[ChunkLocation]) -> Result<(), FrameError> { + for pair in locations.windows(2) { + if pair[0].logical_length == 0 || pair[1].logical_length == 0 { + return Err(FrameError::NonContiguousLocation); + } + } + Ok(()) +} + +/// Return the physical bytes required by the consecutive canonical frames for +/// one logical payload sequence. +/// +/// # Errors +/// +/// Returns an error when the result overflows. +pub fn framed_physical_length(logical_length: u64) -> Result { + if logical_length == 0 { + return Ok(0); + } + let full = logical_length / max_payload_u64(); + let tail = logical_length % max_payload_u64(); + let full_bytes = full + .checked_mul(MAX_FRAME_BYTES as u64) + .ok_or(FrameError::LengthOverflow)?; + if tail == 0 { + Ok(full_bytes) + } else { + full_bytes + .checked_add(FRAME_V2_HEADER_BYTES as u64) + .and_then(|value| value.checked_add(tail)) + .and_then(|value| value.checked_add(FRAME_FOOTER_BYTES as u64)) + .ok_or(FrameError::LengthOverflow) + } +} + +fn frame_count(logical_length: u64) -> u64 { + logical_length.div_ceil(max_payload_u64()) +} + +const fn max_payload_u64() -> u64 { + MAX_FRAME_V2_PAYLOAD_BYTES as u64 +} diff --git a/lib/crowdb-protocol/src/frame/group.rs b/lib/crowdb-protocol/src/frame/group.rs new file mode 100644 index 000000000..3d6d6d190 --- /dev/null +++ b/lib/crowdb-protocol/src/frame/group.rs @@ -0,0 +1,202 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; + +use super::encoding::{crc32c_parts, write_header_region}; +use super::{ + parse_frame, set_frame_chunk_id, ChunkId, FrameError, FrameHeaderPrefix, FrameLayout, FrameMagic, + FrameRole, ParsedFrame, FRAME_FOOTER_BYTES, FRAME_GROUP_HEADER_BYTES, MAX_FRAME_GROUP_PAYLOAD_BYTES, +}; + +/// A whole append validated from a head and all its bounded followers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ParsedAppendGroup { + pub logical_length: u64, + pub physical_length: u64, + pub frame_count: u32, +} + +/// Encode a Stream append without copying payload bytes. Each group remains a +/// single validity unit even when a `DiskIO` request fragments these ordered views. +/// # Errors +/// Returns an error for empty appends, excessive frame counts, or offset overflow. +pub fn encode_append_group_views( + chunk_id: ChunkId, + head_offset: u64, + payload: &Bytes, + write_time_ms: u64, +) -> Result, FrameError> { + let logical_length = u64::try_from(payload.len()).map_err(|_| FrameError::LengthOverflow)?; + let count = payload.len().div_ceil(MAX_FRAME_GROUP_PAYLOAD_BYTES); + let frame_count = u32::try_from(count).map_err(|_| FrameError::LengthOverflow)?; + if count == 0 { + return Err(FrameError::InvalidGroup); + } + head_offset + .checked_add(FrameLayout::StreamGroupV2.physical_length(logical_length)?) + .ok_or(FrameError::LengthOverflow)?; + let mut views = Vec::with_capacity(count.checked_mul(3).ok_or(FrameError::LengthOverflow)?); + for (ordinal, start) in (0..payload.len()) + .step_by(MAX_FRAME_GROUP_PAYLOAD_BYTES) + .enumerate() + { + let end = start + .saturating_add(MAX_FRAME_GROUP_PAYLOAD_BYTES) + .min(payload.len()); + let part = payload.slice(start..end); + let mut header = [0; FRAME_GROUP_HEADER_BYTES]; + write_header_region( + &mut header, + FrameHeaderPrefix { + magic: FrameMagic::StreamV2, + payload_offset: 27, + payload_size: u16::try_from(part.len()).map_err(|_| FrameError::PayloadTooLarge)?, + write_time_ms, + }, + ); + header[14] = if ordinal == 0 { 1 } else { 2 }; + let value = if ordinal == 0 { logical_length } else { head_offset }; + let number = if ordinal == 0 { + frame_count + } else { + u32::try_from(ordinal).map_err(|_| FrameError::LengthOverflow)? + }; + header[15..23].copy_from_slice(&value.to_le_bytes()); + header[23..27].copy_from_slice(&number.to_le_bytes()); + let mut footer = [0; FRAME_FOOTER_BYTES]; + footer[..4].copy_from_slice(&crc32c_parts([header.as_slice(), part.as_ref()]).to_le_bytes()); + set_frame_chunk_id(chunk_id, &mut footer)?; + views.extend([ + Bytes::copy_from_slice(&header), + part, + Bytes::copy_from_slice(&footer), + ]); + } + Ok(views) +} + +/// Bounded state for scans that cannot retain a whole append in memory. Only +/// `finish` grants a complete append boundary; individual valid CRCs do not. +pub struct AppendGroupValidator { + head_offset: u64, + next_offset: u64, + logical_length: u64, + frame_count: u32, + observed: u32, + write_time_ms: u64, + chunk_id: ChunkId, +} + +impl AppendGroupValidator { + /// # Errors + /// Returns an error unless the first frame is a canonical Stream group head. + pub fn new(head_offset: u64, head: &ParsedFrame<'_>) -> Result { + let FrameRole::Head { + logical_length, + frame_count, + } = head.role + else { + return Err(FrameError::InvalidGroup); + }; + if head.header.magic != FrameMagic::StreamV2 + || u64::from(frame_count) != logical_length.div_ceil(MAX_FRAME_GROUP_PAYLOAD_BYTES as u64) + { + return Err(FrameError::InvalidGroup); + } + head_offset + .checked_add(FrameLayout::StreamGroupV2.physical_length(logical_length)?) + .ok_or(FrameError::LengthOverflow)?; + let mut validator = Self { + head_offset, + next_offset: head_offset, + logical_length, + frame_count, + observed: 0, + write_time_ms: head.header.write_time_ms, + chunk_id: head.chunk_id, + }; + validator.push(head_offset, head)?; + Ok(validator) + } + + /// # Errors + /// Rejects gaps, wrong group identity/order, excess frames and noncanonical lengths. + pub fn push(&mut self, offset: u64, frame: &ParsedFrame<'_>) -> Result<(), FrameError> { + let remaining = self + .logical_length + .saturating_sub(u64::from(self.observed) * MAX_FRAME_GROUP_PAYLOAD_BYTES as u64); + let expected_role = if self.observed == 0 { + FrameRole::Head { + logical_length: self.logical_length, + frame_count: self.frame_count, + } + } else { + FrameRole::Follower { + head_offset: self.head_offset, + ordinal: self.observed, + } + }; + if self.observed >= self.frame_count + || offset != self.next_offset + || frame.header.magic != FrameMagic::StreamV2 + || frame.role != expected_role + || frame.chunk_id != self.chunk_id + || frame.header.payload_offset as usize != FRAME_GROUP_HEADER_BYTES + || frame.physical_length != FRAME_GROUP_HEADER_BYTES + frame.payload.len() + FRAME_FOOTER_BYTES + || frame.header.write_time_ms != self.write_time_ms + || frame.payload.len() as u64 != remaining.min(MAX_FRAME_GROUP_PAYLOAD_BYTES as u64) + { + return Err(FrameError::InvalidGroup); + } + self.next_offset = offset + .checked_add(frame.physical_length as u64) + .ok_or(FrameError::LengthOverflow)?; + self.observed += 1; + Ok(()) + } + + /// # Errors + /// Returns an error if any declared follower is still missing. + pub fn finish(self) -> Result { + if self.observed != self.frame_count { + return Err(FrameError::InvalidGroup); + } + Ok(ParsedAppendGroup { + logical_length: self.logical_length, + physical_length: self.next_offset - self.head_offset, + frame_count: self.frame_count, + }) + } + + #[must_use] + pub fn is_complete(&self) -> bool { + self.observed == self.frame_count + } +} + +/// Validate exactly one whole append, leaving any following bytes to the caller. +/// # Errors +/// Returns an error for incomplete or inconsistent groups or frame verification failure. +pub fn parse_append_group( + bytes: &[u8], + chunk_id: ChunkId, + head_offset: u64, +) -> Result { + let head = parse_frame(bytes, chunk_id)?; + let mut cursor = head.physical_length; + let mut validator = AppendGroupValidator::new(head_offset, &head)?; + while !validator.is_complete() { + let frame = parse_frame(&bytes[cursor..], chunk_id)?; + validator.push( + head_offset + .checked_add(cursor as u64) + .ok_or(FrameError::LengthOverflow)?, + &frame, + )?; + cursor = cursor + .checked_add(frame.physical_length) + .ok_or(FrameError::LengthOverflow)?; + } + validator.finish() +} diff --git a/lib/crowdb-protocol/src/frame/parsing.rs b/lib/crowdb-protocol/src/frame/parsing.rs new file mode 100644 index 000000000..313c7c730 --- /dev/null +++ b/lib/crowdb-protocol/src/frame/parsing.rs @@ -0,0 +1,210 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Public self-validating chunk frame encoding and location arithmetic. + +use super::encoding::crc32c_parts; +use super::revision::parse_role; +use super::{ + ChunkId, FrameError, FrameHeaderPrefix, FrameMagic, ParsedFrame, ParsedFrameViews, FRAME_FOOTER_BYTES, + FRAME_GROUP_HEADER_BYTES, FRAME_HEADER_PREFIX_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_BYTES, +}; + +/// Verify a frame over ordered views without assembling its payload. +/// +/// # Errors +/// Returns an error for an incomplete, malformed, or corrupt frame. +pub fn parse_frame_views( + views: &[&[u8]], + expected_chunk_id: ChunkId, +) -> Result { + let mut header_bytes = [0_u8; FRAME_HEADER_PREFIX_BYTES]; + copy_view_range(views, 0, &mut header_bytes)?; + let header = parse_header(&header_bytes)?; + let length = frame_length(header)?; + let available = views + .iter() + .fold(0_usize, |total, view| total.saturating_add(view.len())); + if available < length { + return Err(FrameError::Incomplete { + required_bytes: length, + }); + } + let footer_start = length - FRAME_FOOTER_BYTES; + let mut footer = [0_u8; FRAME_FOOTER_BYTES]; + copy_view_range(views, footer_start, &mut footer)?; + let expected_crc = u32::from_le_bytes( + footer[..4] + .try_into() + .map_err(|_| FrameError::InvalidRegionLength)?, + ); + let mut crc = 0_u32; + let mut cursor = 0_usize; + for view in views { + let included = footer_start.saturating_sub(cursor).min(view.len()); + if included > 0 { + crc = crowdb_common::ec_isal::crc32c_update(crc, &view[..included]); + } + cursor = cursor.saturating_add(view.len()); + if cursor >= footer_start { + break; + } + } + let mut extension = [0; FRAME_GROUP_HEADER_BYTES]; + let size = usize::from(header.payload_offset); + copy_view_range(views, 0, &mut extension[..size])?; + let role = parse_role(header, &extension[..size])?; + if crc != expected_crc { + return Err(FrameError::ChecksumMismatch); + } + let chunk_id = ChunkId { + high: u64::from_be_bytes( + footer[4..12] + .try_into() + .map_err(|_| FrameError::InvalidRegionLength)?, + ), + low: u64::from_be_bytes( + footer[12..20] + .try_into() + .map_err(|_| FrameError::InvalidRegionLength)?, + ), + }; + if chunk_id != expected_chunk_id { + return Err(FrameError::ChunkIdMismatch); + } + Ok(ParsedFrameViews { + header, + role, + chunk_id, + physical_length: length, + }) +} + +fn copy_view_range(views: &[&[u8]], start: usize, target: &mut [u8]) -> Result<(), FrameError> { + let required_bytes = start.saturating_add(target.len()); + let mut cursor = 0_usize; + let mut copied = 0_usize; + for view in views { + let view_end = cursor.saturating_add(view.len()); + if view_end > start && copied < target.len() { + let from = start.saturating_sub(cursor); + let count = (view.len() - from).min(target.len() - copied); + target[copied..copied + count].copy_from_slice(&view[from..from + count]); + copied += count; + } + cursor = view_end; + if copied == target.len() { + return Ok(()); + } + } + Err(FrameError::Incomplete { required_bytes }) +} + +/// Parse and verify one complete frame. The expected chunk ID is mandatory so +/// a valid frame copied from a different chunk is rejected. +/// +/// # Errors +/// +/// Returns an error when the frame is incomplete, malformed, belongs to a +/// different chunk, or fails its CRC32C verification. +pub fn parse_frame(bytes: &[u8], expected_chunk_id: ChunkId) -> Result, FrameError> { + let header = parse_header(bytes)?; + let length = frame_length(header)?; + if bytes.len() < length { + return Err(FrameError::Incomplete { + required_bytes: length, + }); + } + let footer_start = length - FRAME_FOOTER_BYTES; + let checksum = u32::from_le_bytes(bytes[footer_start..footer_start + 4].try_into().map_err(|_| { + FrameError::Incomplete { + required_bytes: length, + } + })?); + let role = parse_role(header, bytes)?; + let crc = crc32c_parts([&bytes[..footer_start]]); + if crc != checksum { + return Err(FrameError::ChecksumMismatch); + } + let chunk_id = + ChunkId { + high: u64::from_be_bytes(bytes[footer_start + 4..footer_start + 12].try_into().map_err( + |_| FrameError::Incomplete { + required_bytes: length, + }, + )?), + low: u64::from_be_bytes(bytes[footer_start + 12..length].try_into().map_err(|_| { + FrameError::Incomplete { + required_bytes: length, + } + })?), + }; + if chunk_id != expected_chunk_id { + return Err(FrameError::ChunkIdMismatch); + } + let payload_start = usize::from(header.payload_offset); + let payload_end = payload_start + usize::from(header.payload_size); + Ok(ParsedFrame { + header, + role, + payload: &bytes[payload_start..payload_end], + chunk_id, + physical_length: length, + }) +} + +/// +/// # Errors +/// +/// Returns an error when the prefix is incomplete, has an unknown magic, or +/// declares a payload before the fixed prefix. +pub fn parse_header(bytes: &[u8]) -> Result { + if bytes.len() < FRAME_HEADER_PREFIX_BYTES { + return Err(FrameError::Incomplete { + required_bytes: FRAME_HEADER_PREFIX_BYTES, + }); + } + let magic = FrameMagic::try_from(u16::from_le_bytes(bytes[0..2].try_into().map_err(|_| { + FrameError::Incomplete { + required_bytes: FRAME_HEADER_PREFIX_BYTES, + } + })?))?; + let payload_offset = u16::from_le_bytes(bytes[2..4].try_into().map_err(|_| FrameError::Incomplete { + required_bytes: FRAME_HEADER_PREFIX_BYTES, + })?); + if usize::from(payload_offset) < FRAME_HEADER_PREFIX_BYTES { + return Err(FrameError::InvalidPayloadOffset { + offset: payload_offset, + }); + } + if ![FRAME_V2_HEADER_BYTES, FRAME_GROUP_HEADER_BYTES].contains(&usize::from(payload_offset)) { + return Err(FrameError::InvalidPayloadOffset { + offset: payload_offset, + }); + } + Ok(FrameHeaderPrefix { + magic, + payload_offset, + payload_size: u16::from_le_bytes(bytes[4..6].try_into().map_err(|_| FrameError::Incomplete { + required_bytes: FRAME_HEADER_PREFIX_BYTES, + })?), + write_time_ms: u64::from_le_bytes(bytes[6..14].try_into().map_err(|_| FrameError::Incomplete { + required_bytes: FRAME_HEADER_PREFIX_BYTES, + })?), + }) +} + +/// +/// # Errors +/// +/// Returns an error when the encoded length overflows or exceeds 64 KiB. +pub fn frame_length(header: FrameHeaderPrefix) -> Result { + let length = usize::from(header.payload_offset) + .checked_add(usize::from(header.payload_size)) + .and_then(|value| value.checked_add(FRAME_FOOTER_BYTES)) + .ok_or(FrameError::LengthOverflow)?; + if length > MAX_FRAME_BYTES { + return Err(FrameError::FrameTooLarge); + } + Ok(length) +} diff --git a/lib/crowdb-protocol/src/frame/revision.rs b/lib/crowdb-protocol/src/frame/revision.rs new file mode 100644 index 000000000..856365c0f --- /dev/null +++ b/lib/crowdb-protocol/src/frame/revision.rs @@ -0,0 +1,119 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + ChunkLocation, FrameError, FrameHeaderPrefix, FrameMagic, FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, + MAX_FRAME_BYTES, +}; + +pub const FRAME_V2_HEADER_BYTES: usize = 15; +pub const FRAME_GROUP_HEADER_BYTES: usize = 27; +pub const MAX_FRAME_V2_PAYLOAD_BYTES: usize = MAX_FRAME_BYTES - FRAME_V2_HEADER_BYTES - FRAME_FOOTER_BYTES; +pub const MAX_FRAME_GROUP_PAYLOAD_BYTES: usize = + MAX_FRAME_BYTES - FRAME_GROUP_HEADER_BYTES - FRAME_FOOTER_BYTES; + +/// Stream grouping is part of the checked header, never an ordinary object's payload. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameRole { + Ordinary, + Head { logical_length: u64, frame_count: u32 }, + Follower { head_offset: u64, ordinal: u32 }, +} + +/// Parse the bounded extension after the fixed prefix has established frame size. +pub(super) fn parse_role(header: FrameHeaderPrefix, bytes: &[u8]) -> Result { + let size = usize::from(header.payload_offset); + if bytes.len() < size { + return Err(FrameError::Incomplete { required_bytes: size }); + } + match (size, bytes[FRAME_HEADER_PREFIX_BYTES]) { + (FRAME_V2_HEADER_BYTES, 0) if header.magic != FrameMagic::StreamV2 => Ok(FrameRole::Ordinary), + (FRAME_GROUP_HEADER_BYTES, role @ (1 | 2)) if header.magic == FrameMagic::StreamV2 => { + let value = u64::from_le_bytes(bytes[15..23].try_into().map_err(|_| FrameError::InvalidRole)?); + let count = u32::from_le_bytes(bytes[23..27].try_into().map_err(|_| FrameError::InvalidRole)?); + if count == 0 || (role == 1 && value == 0) { + return Err(FrameError::InvalidGroup); + } + if role == 1 { + Ok(FrameRole::Head { + logical_length: value, + frame_count: count, + }) + } else { + Ok(FrameRole::Follower { + head_offset: value, + ordinal: count, + }) + } + } + _ => Err(FrameError::InvalidRole), + } +} + +/// Canonical framing geometry carried by the owning location or verified frame. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameLayout { + OrdinaryV2, + StreamGroupV2, +} + +impl FrameLayout { + #[must_use] + pub const fn header_bytes(self) -> usize { + match self { + Self::OrdinaryV2 => FRAME_V2_HEADER_BYTES, + Self::StreamGroupV2 => FRAME_GROUP_HEADER_BYTES, + } + } + + #[must_use] + pub const fn payload_capacity(self) -> u64 { + (MAX_FRAME_BYTES - self.header_bytes() - FRAME_FOOTER_BYTES) as u64 + } + + /// # Errors + /// Returns an error when the physical length overflows. + pub fn physical_length(self, logical_length: u64) -> Result { + let count = logical_length.div_ceil(self.payload_capacity()); + count + .checked_mul((self.header_bytes() + FRAME_FOOTER_BYTES) as u64) + .and_then(|overhead| logical_length.checked_add(overhead)) + .ok_or(FrameError::LengthOverflow) + } + + /// Map a logical subrange to its smallest complete-frame range. + /// # Errors + /// Returns an error for an invalid range or arithmetic overflow. + pub fn physical_range( + self, + location: ChunkLocation, + range: std::ops::Range, + ) -> Result, FrameError> { + if range.start > range.end || range.end > location.logical_length { + return Err(FrameError::InvalidLocationRange); + } + if range.is_empty() { + return Ok(location.frame_offset..location.frame_offset); + } + let first = range.start / self.payload_capacity(); + let last = (range.end - 1) / self.payload_capacity(); + let start = first + .checked_mul(MAX_FRAME_BYTES as u64) + .ok_or(FrameError::LengthOverflow)?; + let end = if last + 1 == location.logical_length.div_ceil(self.payload_capacity()) { + self.physical_length(location.logical_length)? + } else { + (last + 1) + .checked_mul(MAX_FRAME_BYTES as u64) + .ok_or(FrameError::LengthOverflow)? + }; + Ok(location + .frame_offset + .checked_add(start) + .ok_or(FrameError::LengthOverflow)? + ..location + .frame_offset + .checked_add(end) + .ok_or(FrameError::LengthOverflow)?) + } +} diff --git a/lib/crowdb-protocol/src/frame/scan.rs b/lib/crowdb-protocol/src/frame/scan.rs new file mode 100644 index 000000000..c715a7086 --- /dev/null +++ b/lib/crowdb-protocol/src/frame/scan.rs @@ -0,0 +1,201 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Bounded current-format prefix parsing shared by recovery and finalization. + +use sha2::{Digest, Sha256}; +use thiserror::Error; + +use super::{AppendGroupValidator, ChunkId, FrameError, MAX_FRAME_BYTES}; + +mod complete; +mod input; + +pub const MAX_FRAME_SCAN_INPUT_BYTES: usize = 1024 * 1024; +pub const MAX_FRAME_SCAN_EXTENTS: usize = 1024; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameScanMode { + Ordinary, + Grouped, +} + +#[derive(Clone, Copy, Debug)] +pub struct FrameScanConfig { + pub chunk_id: ChunkId, + pub mode: FrameScanMode, + pub physical_start: u64, + pub physical_limit: u64, + pub required_end: u64, + pub logical_start: u64, + pub max_append_bytes: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameScanStop { + Running, + Failed, + Unwritten, + IncompleteAtCapacity, + ExactCapacity, +} + +/// One complete grouped append, or one canonical ordinary frame. Empty ordinary +/// frames can have zero logical length; Stream groups are always nonempty. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ScannedFrameExtent { + pub physical_offset: u64, + pub physical_length: u64, + pub logical_offset: u64, + pub logical_length: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FrameScanSummary { + pub observed_end: u64, + pub physical_end: u64, + pub logical_end: u64, + pub complete_extents: u64, + pub frames_parsed: u64, + /// SHA-256 of the exact encoded complete prefix, excluding incomplete groups. + pub digest: [u8; 32], + pub stop: FrameScanStop, +} + +#[derive(Debug)] +pub struct FrameScanBatch { + pub consumed: usize, + pub extents: Vec, +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum FrameScanError { + #[error("frame observation has already failed")] + FailedObservation, + #[error("invalid frame scan bounds or batch budget")] + InvalidBounds, + #[error("frame scan input is not contiguous")] + OutOfOrder, + #[error("invalid frame at physical offset {offset}: {source}")] + Frame { offset: u64, source: FrameError }, + #[error("complete prefix {observed} does not cover required boundary {required}")] + RequiredPrefix { required: u64, observed: u64 }, + #[error("required checkpoint {required} is inside a grouped append ending at {observed}")] + CheckpointBoundary { required: u64, observed: u64 }, + #[error("observation stopped at {observed} before allocated limit {limit}")] + IncompleteObservation { observed: u64, limit: u64 }, +} + +struct PendingGroup { + validator: AppendGroupValidator, + physical_start: u64, + logical_length: u64, + hash: Sha256, +} + +/// Keeps only one bounded frame, constant group/hash state and scalar progress. +/// A caller must finish the observation before treating its summary as healthy. +pub struct FramePrefixScanner { + config: FrameScanConfig, + input_end: u64, + physical_end: u64, + logical_end: u64, + complete_extents: u64, + frames_parsed: u64, + buffer: Vec, + group: Option, + hash: Sha256, + stop: FrameScanStop, +} + +impl FramePrefixScanner { + #[must_use] + pub const fn stop(&self) -> FrameScanStop { + self.stop + } + + /// # Errors + /// Rejects inverted physical bounds, an out-of-range checkpoint or empty limits. + pub fn new(config: FrameScanConfig) -> Result { + if config.physical_start > config.required_end + || config.required_end > config.physical_limit + || config.max_append_bytes == 0 + { + return Err(FrameScanError::InvalidBounds); + } + Ok(Self { + input_end: config.physical_start, + physical_end: config.physical_start, + logical_end: config.logical_start, + complete_extents: 0, + frames_parsed: 0, + buffer: Vec::with_capacity(MAX_FRAME_BYTES), + group: None, + hash: Sha256::new(), + stop: FrameScanStop::Running, + config, + }) + } + + #[must_use] + pub fn summary(&self) -> FrameScanSummary { + FrameScanSummary { + observed_end: self.input_end, + physical_end: self.physical_end, + logical_end: self.logical_end, + complete_extents: self.complete_extents, + frames_parsed: self.frames_parsed, + digest: self.hash.clone().finalize().into(), + stop: self.stop, + } + } + + /// Ends an exact allocated-range observation. Short IO is an unavailable + /// observation, never a valid shorter prefix. An incomplete group is omitted. + /// # Errors + /// Rejects premature observation completion and an uncovered required prefix. + pub fn finish(&mut self) -> Result { + if self.stop == FrameScanStop::Failed { + return Err(FrameScanError::FailedObservation); + } + let result = self.finish_observation(); + if result.is_err() { + self.stop = FrameScanStop::Failed; + } + result + } + + fn finish_observation(&mut self) -> Result { + if self.stop == FrameScanStop::Running { + if self.input_end != self.config.physical_limit { + return Err(FrameScanError::IncompleteObservation { + observed: self.input_end, + limit: self.config.physical_limit, + }); + } + self.stop = if self.buffer.is_empty() && self.group.is_none() { + FrameScanStop::ExactCapacity + } else { + FrameScanStop::IncompleteAtCapacity + }; + } + self.check_required()?; + self.group = None; + self.buffer.clear(); + Ok(self.summary()) + } + + fn check_required(&self) -> Result<(), FrameScanError> { + if self.physical_end < self.config.required_end { + return Err(FrameScanError::RequiredPrefix { + required: self.config.required_end, + observed: self.physical_end, + }); + } + Ok(()) + } +} + +fn frame_error(offset: u64, source: FrameError) -> FrameScanError { + FrameScanError::Frame { offset, source } +} diff --git a/lib/crowdb-protocol/src/frame/scan/complete.rs b/lib/crowdb-protocol/src/frame/scan/complete.rs new file mode 100644 index 000000000..78ee0863d --- /dev/null +++ b/lib/crowdb-protocol/src/frame/scan/complete.rs @@ -0,0 +1,110 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use sha2::Digest; + +use super::{ + frame_error, FramePrefixScanner, FrameScanError, FrameScanMode, PendingGroup, ScannedFrameExtent, +}; +use crate::frame::{parse_frame, AppendGroupValidator, FrameError, FrameLayout, FrameMagic, FrameRole}; + +impl FramePrefixScanner { + pub(super) fn accept_scan_frame( + &mut self, + offset: u64, + bytes: &[u8], + ) -> Result, FrameScanError> { + let frame = parse_frame(bytes, self.config.chunk_id).map_err(|error| frame_error(offset, error))?; + self.frames_parsed += 1; + if self.config.mode == FrameScanMode::Ordinary { + if frame.role != FrameRole::Ordinary || frame.header.magic == FrameMagic::StreamV2 { + return Err(frame_error(offset, FrameError::InvalidRole)); + } + self.hash.update(bytes); + return self + .complete_scan_extent(offset, frame.physical_length as u64, frame.payload.len() as u64) + .map(Some); + } + if frame.header.magic != FrameMagic::StreamV2 { + return Err(frame_error(offset, FrameError::InvalidRole)); + } + if let Some(group) = &mut self.group { + group + .validator + .push(offset, &frame) + .map_err(|error| frame_error(offset, error))?; + group.hash.update(bytes); + } else { + let FrameRole::Head { logical_length, .. } = frame.role else { + return Err(frame_error(offset, FrameError::InvalidGroup)); + }; + let physical_length = FrameLayout::StreamGroupV2 + .physical_length(logical_length) + .map_err(|error| frame_error(offset, error))?; + if logical_length > self.config.max_append_bytes + || offset + .checked_add(physical_length) + .map_or(true, |end| end > self.config.physical_limit) + { + return Err(frame_error(offset, FrameError::InvalidGroup)); + } + let validator = + AppendGroupValidator::new(offset, &frame).map_err(|error| frame_error(offset, error))?; + let mut hash = self.hash.clone(); + hash.update(bytes); + self.group = Some(PendingGroup { + validator, + physical_start: offset, + logical_length, + hash, + }); + } + if !self + .group + .as_ref() + .is_some_and(|group| group.validator.is_complete()) + { + return Ok(None); + } + let group = self.group.take().expect("completed group exists"); + let parsed = group + .validator + .finish() + .map_err(|error| frame_error(offset, error))?; + self.hash = group.hash; + self.complete_scan_extent(group.physical_start, parsed.physical_length, group.logical_length) + .map(Some) + } + + fn complete_scan_extent( + &mut self, + physical_offset: u64, + physical_length: u64, + logical_length: u64, + ) -> Result { + let end = physical_offset + .checked_add(physical_length) + .ok_or_else(|| frame_error(physical_offset, FrameError::LengthOverflow))?; + if self.config.mode == FrameScanMode::Grouped + && self.physical_end < self.config.required_end + && end > self.config.required_end + { + return Err(FrameScanError::CheckpointBoundary { + required: self.config.required_end, + observed: end, + }); + } + let logical_offset = self.logical_end; + self.logical_end = logical_offset + .checked_add(logical_length) + .ok_or_else(|| frame_error(physical_offset, FrameError::LengthOverflow))?; + self.physical_end = end; + self.complete_extents += 1; + Ok(ScannedFrameExtent { + physical_offset, + physical_length, + logical_offset, + logical_length, + }) + } +} diff --git a/lib/crowdb-protocol/src/frame/scan/input.rs b/lib/crowdb-protocol/src/frame/scan/input.rs new file mode 100644 index 000000000..1070b1896 --- /dev/null +++ b/lib/crowdb-protocol/src/frame/scan/input.rs @@ -0,0 +1,104 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::{ + frame_error, FramePrefixScanner, FrameScanBatch, FrameScanError, FrameScanStop, MAX_FRAME_SCAN_EXTENTS, + MAX_FRAME_SCAN_INPUT_BYTES, +}; +use crate::frame::{frame_length, parse_header, FRAME_HEADER_PREFIX_BYTES}; + +impl FramePrefixScanner { + /// Parses at most a 1 MiB input window and emits at most 1,024 complete + /// extents. Resume unconsumed bytes at the returned contiguous input offset. + /// # Errors + /// Rejects invalid bounds/budgets, gaps, corruption, wrong roles and identities. + /// Discard the scanner after any error; a partial failed observation is invalid. + pub fn push( + &mut self, + offset: u64, + input: &[u8], + max_extents: usize, + ) -> Result { + if self.stop == FrameScanStop::Failed { + return Err(FrameScanError::FailedObservation); + } + let result = self.push_input(offset, input, max_extents); + if result.is_err() { + self.stop = FrameScanStop::Failed; + } + result + } + + fn push_input( + &mut self, + offset: u64, + input: &[u8], + max_extents: usize, + ) -> Result { + let input_len = u64::try_from(input.len()).map_err(|_| FrameScanError::InvalidBounds)?; + if max_extents == 0 + || max_extents > MAX_FRAME_SCAN_EXTENTS + || input.len() > MAX_FRAME_SCAN_INPUT_BYTES + || offset + .checked_add(input_len) + .map_or(true, |end| end > self.config.physical_limit) + { + return Err(FrameScanError::InvalidBounds); + } + if offset != self.input_end { + return Err(FrameScanError::OutOfOrder); + } + let mut result = FrameScanBatch { + consumed: 0, + extents: Vec::new(), + }; + while result.consumed < input.len() + && result.extents.len() < max_extents + && self.stop == FrameScanStop::Running + { + self.copy_scan_input(input, &mut result.consumed, FRAME_HEADER_PREFIX_BYTES); + if self.buffer.len() < FRAME_HEADER_PREFIX_BYTES { + break; + } + let frame_start = self.input_end - self.buffer.len() as u64; + if self.buffer.iter().all(|byte| *byte == 0) { + self.stop = FrameScanStop::Unwritten; + self.check_required()?; + break; + } + let header = parse_header(&self.buffer).map_err(|error| frame_error(frame_start, error))?; + let length = frame_length(header).map_err(|error| frame_error(frame_start, error))?; + if frame_start + .checked_add(length as u64) + .map_or(true, |end| end > self.config.physical_limit) + { + return Err(frame_error( + frame_start, + crate::frame::FrameError::InvalidRegionLength, + )); + } + self.copy_scan_input(input, &mut result.consumed, length); + if self.buffer.len() < length { + break; + } + let mut bytes = std::mem::take(&mut self.buffer); + let extent = self.accept_scan_frame(frame_start, &bytes)?; + bytes.clear(); + self.buffer = bytes; + if let Some(extent) = extent { + result.extents.push(extent); + } + } + Ok(result) + } + + fn copy_scan_input(&mut self, input: &[u8], consumed: &mut usize, target: usize) { + let count = target + .saturating_sub(self.buffer.len()) + .min(input.len() - *consumed); + self.buffer + .extend_from_slice(&input[*consumed..*consumed + count]); + *consumed += count; + self.input_end += count as u64; + } +} diff --git a/lib/crowdb-protocol/src/key.rs b/lib/crowdb-protocol/src/key.rs index 3e22a3911..3a730eeba 100644 --- a/lib/crowdb-protocol/src/key.rs +++ b/lib/crowdb-protocol/src/key.rs @@ -35,7 +35,10 @@ pub use chunk_kv::{ ChunkKvRangeCatalogHeadKey, ChunkKvRangeCatalogPageKey, ChunkKvSplitKey, ChunkKvTransferKey, DomainMonitorKey, ServingGrantKey, }; -pub use chunk_stream::{StreamBindingKey, StreamExtentPageKey, StreamManifestHeadKey, StreamManifestKey}; +pub use chunk_stream::{ + StreamChunkRecordKey, StreamChunkRecordOrder, StreamProcessLeaseKey, StreamRecordKey, + StreamWriterOwnerKey, +}; pub use chunk_task::{ChunkTaskKey, FinalizeChunkTaskKey, LeasedChunkTaskKey, ReadyChunkTaskKey}; pub use chunkdb::{ChunkdbRangeBindingKey, ChunkdbRangeMigrationKey}; mod chunk_slot; diff --git a/lib/crowdb-protocol/src/key/chunk_stream.rs b/lib/crowdb-protocol/src/key/chunk_stream.rs index 6d567edbe..e80b5ef7c 100644 --- a/lib/crowdb-protocol/src/key/chunk_stream.rs +++ b/lib/crowdb-protocol/src/key/chunk_stream.rs @@ -1,185 +1,33 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Group-0 binding and metadata-group keys for chunk streams. - -use std::fmt::Write; - -use crate::chunk_stream::StreamName; - +//! Stream identity, ordered chunk-record and owner keys. use super::encoding::{ - check_exact, check_path_exact, decode_header, decode_u64, encode_header, encode_path_header, encode_u64, - BinaryKey, KeyError, TextKey, + check_exact, decode_header, decode_u64, encode_header, encode_u64, BinaryKey, KeyError, }; - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct StreamBindingKey { - pub stream_name: StreamName, -} - -impl TextKey for StreamBindingKey { - const PATH_MAGIC: &'static str = "/stream"; - const PATH_TYPE: &'static str = "binding"; - - fn encode_to_path(&self, out: &mut String) { - encode_path_header(out, Self::PATH_MAGIC, Self::PATH_TYPE); - out.push('/'); - let _ = write!(out, "{:016x}{:016x}", self.stream_name.high, self.stream_name.low); - } - - fn decode_path(parts: &[&str]) -> Result { - check_path_exact(parts, 1)?; - let encoded = parts.first().ok_or(KeyError::ShortInput)?; - if encoded.len() != 32 { - return Err(KeyError::ShortInput); - } - let high = u64::from_str_radix(&encoded[..16], 16).map_err(|_| KeyError::ShortInput)?; - let low = u64::from_str_radix(&encoded[16..], 16).map_err(|_| KeyError::ShortInput)?; - Ok(Self { - stream_name: StreamName { high, low }, - }) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct StreamManifestKey { - pub stream_name: StreamName, - pub writer_epoch: u64, - pub generation: u64, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct StreamManifestHeadKey { - pub stream_name: StreamName, -} - -impl BinaryKey for StreamManifestHeadKey { - const TYPE_TAG: u16 = 0x0012; - - fn encode_to(&self, out: &mut Vec) { - encode_header(out, Self::TYPE_TAG); - encode_u64(out, self.stream_name.high); - encode_u64(out, self.stream_name.low); - } - - fn decode(buf: &[u8]) -> Result { - let fields = decode_header(buf, Self::TYPE_TAG)?; - let (high, offset) = decode_u64(fields, 0)?; - let (low, offset) = decode_u64(fields, offset)?; - check_exact(fields, offset)?; - Ok(Self { - stream_name: StreamName { high, low }, - }) - } -} - -impl BinaryKey for StreamManifestKey { - const TYPE_TAG: u16 = 0x0010; - - fn encode_to(&self, out: &mut Vec) { - encode_header(out, Self::TYPE_TAG); - encode_u64(out, self.stream_name.high); - encode_u64(out, self.stream_name.low); - encode_u64(out, self.writer_epoch); - encode_u64(out, self.generation); - } - - fn decode(buf: &[u8]) -> Result { - let fields = decode_header(buf, Self::TYPE_TAG)?; - let (high, offset) = decode_u64(fields, 0)?; - let (low, offset) = decode_u64(fields, offset)?; - let (writer_epoch, offset) = decode_u64(fields, offset)?; - let (generation, offset) = decode_u64(fields, offset)?; - check_exact(fields, offset)?; - Ok(Self { - stream_name: StreamName { high, low }, - writer_epoch, - generation, - }) - } -} - +use crate::chunk_stream::StreamName; +mod chunk_record; +mod ownership; +pub use chunk_record::{StreamChunkRecordKey, StreamChunkRecordOrder}; +pub use ownership::{StreamProcessLeaseKey, StreamWriterOwnerKey}; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct StreamExtentPageKey { +pub struct StreamRecordKey { pub stream_name: StreamName, - pub writer_epoch: u64, - pub generation: u64, - pub page_index: u64, } - -impl StreamExtentPageKey { - /// Encodes the exact key prefix shared by all extent pages for one stream. - #[must_use] - pub fn stream_prefix(stream_name: StreamName) -> Vec { - let mut out = Vec::with_capacity(19); - encode_header(&mut out, Self::TYPE_TAG); - encode_u64(&mut out, stream_name.high); - encode_u64(&mut out, stream_name.low); - out - } -} - -impl BinaryKey for StreamExtentPageKey { - const TYPE_TAG: u16 = 0x0011; - +impl BinaryKey for StreamRecordKey { + const TYPE_TAG: u16 = 0x001d; fn encode_to(&self, out: &mut Vec) { encode_header(out, Self::TYPE_TAG); encode_u64(out, self.stream_name.high); encode_u64(out, self.stream_name.low); - encode_u64(out, self.writer_epoch); - encode_u64(out, self.generation); - encode_u64(out, self.page_index); } - fn decode(buf: &[u8]) -> Result { let fields = decode_header(buf, Self::TYPE_TAG)?; let (high, offset) = decode_u64(fields, 0)?; let (low, offset) = decode_u64(fields, offset)?; - let (writer_epoch, offset) = decode_u64(fields, offset)?; - let (generation, offset) = decode_u64(fields, offset)?; - let (page_index, offset) = decode_u64(fields, offset)?; check_exact(fields, offset)?; Ok(Self { stream_name: StreamName { high, low }, - writer_epoch, - generation, - page_index, }) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn stream_keys_round_trip() { - let name = StreamName { high: 7, low: 9 }; - let binding = StreamBindingKey { stream_name: name }; - assert_eq!(StreamBindingKey::from_path(&binding.to_path()).unwrap(), binding); - - let manifest = StreamManifestKey { - stream_name: name, - writer_epoch: 11, - generation: 13, - }; - assert_eq!( - StreamManifestKey::from_bytes(&manifest.to_bytes()).unwrap(), - manifest - ); - - let head = StreamManifestHeadKey { stream_name: name }; - assert_eq!(StreamManifestHeadKey::from_bytes(&head.to_bytes()).unwrap(), head); - - let extent = StreamExtentPageKey { - stream_name: name, - writer_epoch: 11, - generation: 13, - page_index: 17, - }; - assert_eq!( - StreamExtentPageKey::from_bytes(&extent.to_bytes()).unwrap(), - extent - ); - } -} diff --git a/lib/crowdb-protocol/src/key/chunk_stream/chunk_record.rs b/lib/crowdb-protocol/src/key/chunk_stream/chunk_record.rs new file mode 100644 index 000000000..48c3bcee1 --- /dev/null +++ b/lib/crowdb-protocol/src/key/chunk_stream/chunk_record.rs @@ -0,0 +1,74 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crate::chunk_stream::StreamName; +use crate::key::encoding::{check_exact, decode_header, decode_u64, encode_header, encode_u64}; +use crate::key::{BinaryKey, KeyError}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StreamChunkRecordOrder { + Attached(u64), + Candidate(u64), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StreamChunkRecordKey { + pub stream_id: StreamName, + pub order: StreamChunkRecordOrder, +} + +impl StreamChunkRecordKey { + #[must_use] + pub fn stream_prefix(id: StreamName) -> Vec { + let mut bytes = Vec::with_capacity(19); + encode_header(&mut bytes, Self::TYPE_TAG); + encode_u64(&mut bytes, id.high); + encode_u64(&mut bytes, id.low); + bytes + } + + #[must_use] + pub fn attached_prefix(id: StreamName) -> Vec { + let mut bytes = Self::stream_prefix(id); + bytes.push(0); + bytes + } + + #[must_use] + pub fn candidate_prefix(id: StreamName) -> Vec { + let mut bytes = Self::stream_prefix(id); + bytes.push(1); + bytes + } +} + +impl BinaryKey for StreamChunkRecordKey { + const TYPE_TAG: u16 = 0x001c; + fn encode_to(&self, bytes: &mut Vec) { + bytes.extend(Self::stream_prefix(self.stream_id)); + let (kind, value) = match self.order { + StreamChunkRecordOrder::Attached(start) => (0, start), + StreamChunkRecordOrder::Candidate(sequence) => (1, sequence), + }; + bytes.push(kind); + encode_u64(bytes, value); + } + + fn decode(bytes: &[u8]) -> Result { + let fields = decode_header(bytes, Self::TYPE_TAG)?; + let (high, offset) = decode_u64(fields, 0)?; + let (low, offset) = decode_u64(fields, offset)?; + let kind = *fields.get(offset).ok_or(KeyError::ShortInput)?; + let (value, offset) = decode_u64(fields, offset + 1)?; + check_exact(fields, offset)?; + let order = match kind { + 0 => StreamChunkRecordOrder::Attached(value), + 1 => StreamChunkRecordOrder::Candidate(value), + _ => return Err(KeyError::BadTag), + }; + Ok(Self { + stream_id: StreamName { high, low }, + order, + }) + } +} diff --git a/lib/crowdb-protocol/src/key/chunk_stream/ownership.rs b/lib/crowdb-protocol/src/key/chunk_stream/ownership.rs new file mode 100644 index 000000000..cc560ce44 --- /dev/null +++ b/lib/crowdb-protocol/src/key/chunk_stream/ownership.rs @@ -0,0 +1,64 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Fixed Group-0 owner keys; process epochs remain in conditional values. + +use super::super::encoding::{check_path_exact, decode_path_u64, encode_path_header, encode_path_u64}; +use crate::{ + chunk_stream::StreamPurpose, + key::{KeyError, TextKey}, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StreamProcessLeaseKey { + pub purpose: StreamPurpose, + pub instance_id: u64, +} + +impl TextKey for StreamProcessLeaseKey { + const PATH_MAGIC: &'static str = "/stream-owner"; + const PATH_TYPE: &'static str = "process"; + + fn encode_to_path(&self, out: &mut String) { + encode_path_header(out, Self::PATH_MAGIC, Self::PATH_TYPE); + encode_path_u64(out, u64::from(self.purpose == StreamPurpose::Wal)); + encode_path_u64(out, self.instance_id); + } + + fn decode_path(parts: &[&str]) -> Result { + check_path_exact(parts, 2)?; + let purpose = match decode_path_u64(parts[0])? { + 0 => StreamPurpose::Stream, + 1 => StreamPurpose::Wal, + _ => return Err(KeyError::BadTag), + }; + Ok(Self { + purpose, + instance_id: decode_path_u64(parts[1])?, + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct StreamWriterOwnerKey { + pub stream_name: crate::chunk_stream::StreamName, +} + +impl TextKey for StreamWriterOwnerKey { + const PATH_MAGIC: &'static str = "/stream-owner"; + const PATH_TYPE: &'static str = "writer"; + fn encode_to_path(&self, out: &mut String) { + encode_path_header(out, Self::PATH_MAGIC, Self::PATH_TYPE); + encode_path_u64(out, self.stream_name.high); + encode_path_u64(out, self.stream_name.low); + } + fn decode_path(parts: &[&str]) -> Result { + check_path_exact(parts, 2)?; + Ok(Self { + stream_name: crate::chunk_stream::StreamName { + high: decode_path_u64(parts[0])?, + low: decode_path_u64(parts[1])?, + }, + }) + } +} diff --git a/lib/crowdb-protocol/src/mgmt.rs b/lib/crowdb-protocol/src/mgmt.rs index 125f59917..294df8e1d 100644 --- a/lib/crowdb-protocol/src/mgmt.rs +++ b/lib/crowdb-protocol/src/mgmt.rs @@ -18,6 +18,7 @@ use serde::{Deserialize, Serialize}; +pub mod cluster; pub mod node; /// Local connection hints for a KV process; these do not grant group membership. @@ -95,6 +96,29 @@ pub struct AddGroupRequest { /// driver with a correct quorum. `None` keeps the default (start driver). #[serde(default, skip_serializing_if = "Option::is_none")] pub start_election: Option, + /// Cluster-owned creation must match this durable operation on every retry. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deployment: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +#[serde(deny_unknown_fields)] +pub struct GroupDeploymentIdentity { + pub cluster_id: String, + pub operation_id: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +#[serde(deny_unknown_fields)] +pub struct GroupCreationStatus { + pub version: u32, + pub store_id: u64, + pub group_id: u64, + pub replica_id: u64, + pub deployment: Option, + pub retired: bool, } /// `GET /stores/{sid}/groups` item. diff --git a/lib/crowdb-protocol/src/mgmt/cluster.rs b/lib/crowdb-protocol/src/mgmt/cluster.rs new file mode 100644 index 000000000..9c9c75bd1 --- /dev/null +++ b/lib/crowdb-protocol/src/mgmt/cluster.rs @@ -0,0 +1,129 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Observational cluster readiness; these facts never authorize deployment. + +use serde::{Deserialize, Serialize}; + +/// Confirmed application identity and native management seeds for remote attach. +/// Reading this record grants no ownership of the hosting runtime resources. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClusterTargetIdentity { + pub version: u32, + pub cluster_id: String, + pub group0_management_endpoints: Vec, + pub group0_rpc_endpoints: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ClusterPhase { + Waiting, + Initializing, + Ready, + Degraded, + Unavailable, + Failed, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DependencyKind { + Identity, + Authority, + Metadata, + Storage, + Member, + Service, + Protocol, + Runtime, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DependencyState { + Ready, + Waiting, + Initializing, + Stopped, + /// A validated terminal failure, not a probe or transport timeout. + Failed, + Unknown, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReadinessReason { + pub dependency: String, + pub code: String, + /// Operator-facing explanation without credential contents. + pub detail: String, +} + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DependencyObservation { + pub id: String, + pub kind: DependencyKind, + pub required: bool, + pub state: DependencyState, + /// Current operator-facing evidence, including successful checks; no credentials. + #[serde(default)] + pub detail: String, + pub observed_at_unix_ms: u64, + pub max_age_ms: u64, + pub revision: Option, + /// Operation associated with this observation, if applicable. + pub operation_id: Option, + /// Current committed operation; a superseded observation is never current health. + pub current_operation_id: Option, + pub reason: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClusterMemberObservation { + pub discovery_id: String, + pub node_id: Option, + pub management_endpoints: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RuntimeObservation { + pub runtime: String, + pub endpoint: String, + pub source: String, + pub image_digest: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClusterObservation { + pub version: u32, + pub cluster: String, + /// Unknown until bootstrap identity is authoritatively observed. + pub cluster_id: Option, + /// A confirmed binding, not merely a UUID reserved by an in-progress bootstrap. + pub bound: bool, + pub initial_voters: u8, + pub policy: Option, + pub observed_at_unix_ms: u64, + pub members: Vec, + pub operation_id: Option, + pub work_accepted: bool, + /// Historical setup completion, distinct from the dependency facts below. + pub setup_completed: bool, + pub dependencies: Vec, + /// Remote checks may have application facts without any local runtime access. + pub runtime: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClusterStatus { + pub phase: ClusterPhase, + pub reasons: Vec, + pub observation: ClusterObservation, +} diff --git a/lib/crowdb-protocol/src/mgmt/node.rs b/lib/crowdb-protocol/src/mgmt/node.rs index 7f6ad4d2b..ea79484f9 100644 --- a/lib/crowdb-protocol/src/mgmt/node.rs +++ b/lib/crowdb-protocol/src/mgmt/node.rs @@ -5,6 +5,9 @@ use serde::{Deserialize, Serialize}; +pub mod deployment; +pub mod observation; +pub mod recovery; pub mod setup; pub mod storage; @@ -17,6 +20,8 @@ pub struct NodeAdvertisement { pub protocol_version: u32, pub monitor_endpoints: Vec, pub cluster_id: Option, + #[serde(default)] + pub formation: Option, } #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] @@ -64,12 +69,27 @@ pub struct CandidateSnapshot { #[serde(tag = "command", rename_all = "snake_case", deny_unknown_fields)] pub enum NodeControl { Identity, + DeploymentPolicy, DiskInventory, AcceptNodeSetup { request: setup::NodeSetupRequest, }, NodeSetup, NodeSetupIntent, + RuntimeObservation, + RecoveryIdentity, + PrepareBootstrap { + node_id: u64, + bootstrap: super::SystemBootstrapIdentity, + manifest: serde_json::Value, + credentials: NodeServiceCredentials, + }, + AcceptMemberTrust { + trust: serde_json::Value, + }, + AcceptConfirmedMemberTrust { + trust: serde_json::Value, + }, CheckRetainedDisk { request: storage::CheckRetainedDisk, }, diff --git a/lib/crowdb-protocol/src/mgmt/node/deployment.rs b/lib/crowdb-protocol/src/mgmt/node/deployment.rs new file mode 100644 index 000000000..c4a289dc0 --- /dev/null +++ b/lib/crowdb-protocol/src/mgmt/node/deployment.rs @@ -0,0 +1,249 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Accepted startup inputs, distinct from discovered or confirmed membership. + +use super::setup::{NodeSetupChoices, SetupService, SetupServiceKind, SetupStorage}; +use serde::{Deserialize, Serialize}; + +mod public_access; +pub use public_access::PublicAccessUris; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum DeploymentMode { + Local, + Production, +} + +/// Untrusted discovery hints. Admission still authenticates the actual peer. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FormationPolicy { + pub version: u32, + pub mode: DeploymentMode, + pub cluster: String, + pub initial_voters: u8, +} + +impl FormationPolicy { + /// # Errors + /// Rejects unsupported versions, malformed scopes and voter counts. + pub fn validate(&self) -> Result<(), &'static str> { + if self.version != 1 || !matches!(self.initial_voters, 1 | 3) || !valid_cluster(&self.cluster) { + return Err("invalid node formation policy"); + } + Ok(()) + } +} + +/// Sealed before admission; omitted restart inputs retain these values. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NodeDeploymentPolicy { + pub version: u32, + pub mode: DeploymentMode, + pub cluster: String, + pub initial_voters: u8, + pub setup: NodeSetupChoices, + pub public_access: PublicAccessUris, +} + +/// Explicit inputs from a configuration file or command/environment overrides. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NodeDeploymentOverrides { + pub version: Option, + pub mode: Option, + pub cluster: Option, + pub initial_voters: Option, + pub storage: Option, + pub services: Option>, + #[serde(default)] + pub public_access: PublicAccessUris, +} + +impl Default for NodeDeploymentPolicy { + fn default() -> Self { + let mut setup = NodeSetupChoices::default(); + if let SetupStorage::Automatic { sparse, .. } = &mut setup.storage { + *sparse = true; + } + for service in &mut setup.services { + let listeners = &mut service.listeners; + match service.kind { + SetupServiceKind::PaxosKv => { + listeners.http_port = Some(10000); + listeners.rpc_port = Some(10100); + } + SetupServiceKind::Diskdb => listeners.rpc_port = Some(7100), + SetupServiceKind::Diskio => listeners.rpc_port = Some(7200), + SetupServiceKind::Chunkdb => { + listeners.http_port = Some(7300); + listeners.rpc_port = Some(7301); + } + SetupServiceKind::ChunkKv => { + listeners.http_port = Some(7400); + listeners.rpc_port = Some(7401); + } + SetupServiceKind::AccessServer => { + listeners.s3_port = Some(9091); + listeners.http_port = Some(9092); + listeners.dataset_port = Some(9093); + listeners.health_port = Some(9094); + } + } + } + Self { + version: 1, + mode: DeploymentMode::Local, + cluster: "default".into(), + initial_voters: 1, + setup, + public_access: PublicAccessUris { + s3: Some("http://127.0.0.1:9091".into()), + iceberg: Some("http://127.0.0.1:9092".into()), + dataset: Some("http://127.0.0.1:9093".into()), + }, + } + } +} + +impl NodeDeploymentPolicy { + #[must_use] + pub fn formation(&self) -> FormationPolicy { + FormationPolicy { + version: self.version, + mode: self.mode, + cluster: self.cluster.clone(), + initial_voters: self.initial_voters, + } + } + + /// Check the entire accepted input before performing node effects. + /// + /// # Errors + /// Rejects unknown versions, invalid scopes/counts and unsafe production files. + pub fn validate(&self) -> Result<(), &'static str> { + if self.version != 1 { + return Err("unsupported node deployment policy version"); + } + if !matches!(self.initial_voters, 1 | 3) { + return Err("initial voters must be one or three"); + } + if !valid_cluster(&self.cluster) { + return Err("cluster scope must contain 1 to 48 name characters"); + } + if self.mode == DeploymentMode::Production + && matches!(self.setup.storage, SetupStorage::Automatic { sparse: true, .. }) + { + return Err("production automatic storage cannot use sparse local-test files"); + } + for service in self.setup.services.iter().filter(|service| service.enabled) { + let ports = &service.listeners; + if service.kind == SetupServiceKind::PaxosKv + && (ports.http_port != Some(10000) || ports.rpc_port != Some(10100)) + { + return Err("bootstrap KV requires management port 10000 and RPC port 10100"); + } + if [ + ports.http_port, + ports.rpc_port, + ports.s3_port, + ports.dataset_port, + ports.health_port, + ] + .into_iter() + .flatten() + .any(|port| matches!(port, 2222 | 9090 | 9095 | 10101)) + { + return Err("service listener conflicts with node management or bootstrap transport"); + } + let complete = match service.kind { + SetupServiceKind::PaxosKv | SetupServiceKind::Chunkdb | SetupServiceKind::ChunkKv => { + ports.http_port.is_some() && ports.rpc_port.is_some() + } + SetupServiceKind::Diskdb | SetupServiceKind::Diskio => ports.rpc_port.is_some(), + SetupServiceKind::AccessServer => { + ports.s3_port.is_some() + && ports.http_port.is_some() + && ports.dataset_port.is_some() + && ports.health_port.is_some() + } + }; + if !complete { + return Err("accepted service listeners must be explicit"); + } + } + self.public_access.validate()?; + self.setup.validate() + } +} + +fn valid_cluster(cluster: &str) -> bool { + !cluster.is_empty() + && cluster.len() <= 48 + && cluster.as_bytes()[0].is_ascii_alphanumeric() + && cluster + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_.-".contains(&byte)) +} + +impl NodeDeploymentOverrides { + /// Resolve explicit overrides over file inputs, retaining omitted accepted values. + /// + /// # Errors + /// Rejects changed accepted inputs and invalid fresh or persisted policies. + pub fn resolve( + &self, + file: &Self, + accepted: Option<&NodeDeploymentPolicy>, + ) -> Result { + if [self.version, file.version] + .into_iter() + .flatten() + .any(|version| version != 1) + { + return Err("unsupported node deployment policy version"); + } + if let Some(accepted) = accepted { + accepted.validate()?; + } + let mut policy = accepted.cloned().unwrap_or_default(); + if accepted.is_none() && self.mode.or(file.mode) == Some(DeploymentMode::Production) { + policy.mode = DeploymentMode::Production; + policy.public_access = PublicAccessUris::default(); + policy.setup.storage = NodeSetupChoices::default().storage; + } + for inputs in [file, self] { + inputs.apply(&mut policy); + } + policy.validate()?; + if accepted.is_some_and(|accepted| *accepted != policy) { + return Err("deployment inputs differ from accepted node policy"); + } + Ok(policy) + } + + fn apply(&self, policy: &mut NodeDeploymentPolicy) { + self.public_access.apply(&mut policy.public_access); + if let Some(version) = self.version { + policy.version = version; + } + if let Some(mode) = self.mode { + policy.mode = mode; + } + if let Some(cluster) = &self.cluster { + policy.cluster.clone_from(cluster); + } + if let Some(voters) = self.initial_voters { + policy.initial_voters = voters; + } + if let Some(storage) = &self.storage { + policy.setup.storage.clone_from(storage); + } + if let Some(services) = &self.services { + policy.setup.services.clone_from(services); + } + } +} diff --git a/lib/crowdb-protocol/src/mgmt/node/deployment/public_access.rs b/lib/crowdb-protocol/src/mgmt/node/deployment/public_access.rs new file mode 100644 index 000000000..cc353e1ea --- /dev/null +++ b/lib/crowdb-protocol/src/mgmt/node/deployment/public_access.rs @@ -0,0 +1,65 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Explicit client origins, independent of internal service listeners. + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PublicAccessUris { + pub s3: Option, + pub iceberg: Option, + pub dataset: Option, +} + +impl PublicAccessUris { + /// # Errors + /// Require HTTP origins without credentials, paths, queries or fragments. + pub fn validate(&self) -> Result<(), &'static str> { + for value in [&self.s3, &self.iceberg, &self.dataset].into_iter().flatten() { + let uri = url::Url::parse(value).map_err(|_| "invalid public access URI")?; + if value.len() > 2048 + || !(value.starts_with("http://") || value.starts_with("https://")) + || value + .bytes() + .any(|byte| byte.is_ascii_whitespace() || byte == b'\\') + || !matches!(uri.scheme(), "http" | "https") + || uri.host_str().is_none() + || !uri.username().is_empty() + || uri.password().is_some() + || uri.path() != "/" + || uri.query().is_some() + || uri.fragment().is_some() + { + return Err("public access URI must be an HTTP origin without credentials"); + } + } + Ok(()) + } + + pub(super) fn apply(&self, target: &mut Self) { + for (value, destination) in [ + (&self.s3, &mut target.s3), + (&self.iceberg, &mut target.iceberg), + (&self.dataset, &mut target.dataset), + ] { + if value.is_some() { + destination.clone_from(value); + } + } + } + + /// Include only explicit choices; the service builder supplies internal defaults. + #[must_use] + pub fn environment(&self) -> std::collections::BTreeMap { + [ + ("CROWDB_S3_PUBLIC_URI", &self.s3), + ("CROWDB_ICEBERG_PUBLIC_URI", &self.iceberg), + ("CROWDB_DATASET_PUBLIC_URI", &self.dataset), + ] + .into_iter() + .filter_map(|(key, value)| value.as_ref().map(|value| (key.into(), value.clone()))) + .collect() + } +} diff --git a/lib/crowdb-protocol/src/mgmt/node/observation.rs b/lib/crowdb-protocol/src/mgmt/node/observation.rs new file mode 100644 index 000000000..3220e18b8 --- /dev/null +++ b/lib/crowdb-protocol/src/mgmt/node/observation.rs @@ -0,0 +1,37 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Current local execution evidence, separate from accepted setup progress. + +use super::{deployment::NodeDeploymentPolicy, setup::NodeSetupStatus, NodeBinding}; +use crate::mgmt::cluster::DependencyState; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NodeRuntimeObservation { + pub version: u32, + pub discovery_id: String, + pub policy: NodeDeploymentPolicy, + pub binding: Option, + pub setup: Option, + pub observed_at_unix_ms: u64, + pub services: Vec, + pub protocols: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NodeServiceObservation { + pub service_id: String, + pub kind: String, + /// The executed operation; aggregate readers compare current committed intent. + pub operation_id: Option, + pub state: DependencyState, + /// Current execution/health evidence without credentials or raw configuration. + #[serde(default)] + pub detail: String, + pub observed_at_unix_ms: u64, + pub max_age_ms: u64, + pub reason_code: Option, +} diff --git a/lib/crowdb-protocol/src/mgmt/node/recovery.rs b/lib/crowdb-protocol/src/mgmt/node/recovery.rs new file mode 100644 index 000000000..3a12e62af --- /dev/null +++ b/lib/crowdb-protocol/src/mgmt/node/recovery.rs @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +//! Authenticated member endpoints; discovery advertisements carry no authority. + +use super::NodeBinding; +use crate::mgmt::SystemBootstrapIdentity; +use serde::{Deserialize, Serialize}; +use std::net::IpAddr; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MemberEndpoint { + pub discovery_id: String, + pub node_id: u64, + pub boot_id: String, + pub addresses: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MemberRecoveryIdentity { + pub version: u32, + pub binding: NodeBinding, + pub endpoint: MemberEndpoint, +} + +/// Local monitor-owned snapshot, scoped to one live observer incarnation. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct MemberEndpoints { + pub version: u32, + pub bootstrap: SystemBootstrapIdentity, + pub observer_boot_id: String, + pub observed_at_unix_ms: u64, + pub members: Vec, + pub conflicts: Vec, +} + +/// Physical owners captured from accepted bootstrap or committed topology. +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecoveryGroup { + pub store_id: u64, + pub group_id: u64, + pub deployment: Option, + pub members: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecoveryReplica { + pub replica_id: u64, + pub node_id: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RecoveryGroups { + pub bootstrap: SystemBootstrapIdentity, + pub groups: Vec, +} diff --git a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs index 73755a722..c312f2e7a 100644 --- a/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs +++ b/lib/crowdb-protocol/src/mgmt/node/setup/validation.rs @@ -1,7 +1,9 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use super::{NodeSetupRequest, NodeSetupStatus, SetupService, SetupServiceKind, SetupStorage}; +use super::{ + NodeSetupChoices, NodeSetupRequest, NodeSetupStatus, SetupService, SetupServiceKind, SetupStorage, +}; use std::collections::BTreeSet; impl NodeSetupStatus { @@ -49,17 +51,27 @@ impl NodeSetupRequest { { return Err("node setup exceeds its control message bound"); } + self.choices.validate() + } +} + +impl NodeSetupChoices { + /// Validate storage and service inputs before accepting a deployment policy. + /// + /// # Errors + /// Rejects invalid geometry, duplicated media and ambiguous listeners. + pub fn validate(&self) -> Result<(), &'static str> { let mut kinds = BTreeSet::new(); let mut ports = BTreeSet::new(); - if self.choices.services.len() != 6 { + if self.services.len() != 6 { return Err("node setup requires exactly six service choices"); } - for service in &self.choices.services { + for service in &self.services { if !kinds.insert(service.kind) || !valid_listeners(service, &mut ports) { return Err("duplicate service choice or invalid listener configuration"); } } - match &self.choices.storage { + match &self.storage { SetupStorage::Diskless => {} SetupStorage::Automatic { capacity_bytes, diff --git a/lib/crowdb-protocol/src/timing.rs b/lib/crowdb-protocol/src/timing.rs index 8537c64f2..70e5f8502 100644 --- a/lib/crowdb-protocol/src/timing.rs +++ b/lib/crowdb-protocol/src/timing.rs @@ -1,12 +1,12 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -//! Shared stale-write and owner self-fence timing defaults. +//! Shared write-admission ceilings and owner self-fence timing defaults. -pub const DEFAULT_MAX_WRITE_REQUEST_AGE_MS: u64 = 30_000; -pub const DEFAULT_MAX_CLOCK_SKEW_MS: u64 = 1_000; +pub const MAX_WRITE_REQUEST_AGE_MS: u64 = 30_000; +pub const MAX_WRITE_CLOCK_SKEW_MS: u64 = 1_000; pub const DEFAULT_SELF_FENCE_MARGIN_MS: u64 = 1_000; /// Extra delay before an expired liveness task reads an abandoned tail. -/// Together with request age and peer skew this prevents a pre-expiry write -/// from racing the final frame scan. +/// Request age bounds late admission. Ordered drains must separately wait +/// for already submitted backend IO before a final frame scan or reuse. pub const DEFAULT_FINALIZER_SCANNER_MARGIN_MS: u64 = 1_000; diff --git a/lib/crowdb-protocol/src/types/chunkdb.rs b/lib/crowdb-protocol/src/types/chunkdb.rs index 288e81b87..e94ee9a0f 100644 --- a/lib/crowdb-protocol/src/types/chunkdb.rs +++ b/lib/crowdb-protocol/src/types/chunkdb.rs @@ -10,8 +10,7 @@ clippy::too_many_lines )] -//! Hand-written Rust types replacing the prost-generated `crow.chunkdb.rpc` -//! types. API-compatible with the former proto-generated structs. +//! Typed records for the current ChunkDB service and metadata contract. use serde::{Deserialize, Serialize}; @@ -187,14 +186,11 @@ pub struct ChunkStrip { pub usage_bitmap: Vec, /// Replica identities known unavailable until background recovery. pub unavailable_segments: Vec, - /// Policy selected when the strip was created. Legacy strips default to rack-first. - #[serde(default)] + /// Policy selected when the strip was created. pub placement_priority: i32, /// Creation-time physical failure-domain assessment. - #[serde(default)] pub placement_assessment: Option, /// A durable placement task must improve this temporary EC layout. - #[serde(default)] pub placement_repair_required: bool, } @@ -226,7 +222,7 @@ pub struct Chunk { pub cleanup_intents: Vec, /// Most recently committed fenced replacement operation. pub last_strip_replacement: Option, - /// Stable logical owner identity. Empty only for legacy unattributed chunks. + /// Stream/Wal owner identity; other chunk types leave this empty. pub owner_key: Vec, } @@ -317,9 +313,7 @@ pub struct StripReservationGroup { pub preferred_survivors: Vec, pub data_num: u32, pub code_num: u32, - #[serde(default)] pub planned_cursors: Vec, - #[serde(default)] pub planned_closed_sequences: Vec, } @@ -337,7 +331,6 @@ pub struct ReserveStripGroupRequest { pub conversion_data_num: u32, pub conversion_code_num: u32, /// Append position supplied by the exclusive writer, including hidden strips. - #[serde(default)] pub reservation_offset_kb: Option, } @@ -453,9 +446,7 @@ pub struct RelocateSegmentHandoffResponse { #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] pub struct SealChunkRequest { pub chunk_id: Option, - pub seal_length: u32, - /// Exact written bytes. Zero selects the legacy KiB-only request. - #[serde(default)] + /// Exact written bytes, including zero for an empty chunk. pub seal_bytes: u64, } @@ -486,18 +477,6 @@ pub struct DeleteChunkRangeRequest { #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] pub struct DeleteChunkRangeResponse {} -#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] -pub struct UpdateChunkStripRequest { - pub chunk_id: Option, - pub strip_index: u32, - pub strip: Option, -} - -#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] -pub struct UpdateChunkStripResponse { - pub chunk: Option, -} - #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] pub struct AllocateReplacementSegmentRequest { pub chunk_id: Option, diff --git a/lib/crowdb-protocol/src/types/common.rs b/lib/crowdb-protocol/src/types/common.rs index d8ad94743..10eb68a5b 100644 --- a/lib/crowdb-protocol/src/types/common.rs +++ b/lib/crowdb-protocol/src/types/common.rs @@ -10,15 +10,12 @@ clippy::too_many_lines )] -//! Hand-written Rust types replacing the prost-generated `crow.common` -//! types. API-compatible with the former proto-generated structs — same -//! field names, types, and derives. No `prost::Message` trait (removed -//! with the protobuf cutover). +//! Shared service-registry, hardware and RPC value types. use serde::{Deserialize, Serialize}; /// Implement `From for i32` and `TryFrom for Enum` for a -/// `#[repr(i32)]` enum, matching prost's generated conversions. +/// `#[repr(i32)]` enum. macro_rules! impl_enum_conversions { ($enum:ident, $($variant:ident = $value:expr),+ $(,)?) => { impl From<$enum> for i32 { @@ -265,6 +262,7 @@ pub struct ServiceExtra { #[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] pub struct ChunkKvExtra { + pub process_epoch: u64, #[serde(default, skip_serializing_if = "Option::is_none")] pub node_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -273,7 +271,6 @@ pub struct ChunkKvExtra { pub durable_bytes: u64, pub request_rate: u64, pub hosted: Vec, - #[serde(default)] pub partition_loads: Vec, } @@ -281,12 +278,9 @@ pub struct ChunkKvExtra { pub struct ChunkKvPartitionLoad { pub partition_id: crate::chunk_kv::Id128, pub durable_bytes: u64, - #[serde(default)] pub logical_bytes: u64, - #[serde(default)] pub logical_metrics_exact: bool, pub live_byte_samples: Vec<(Vec, u64)>, - #[serde(default)] pub independently_recoverable: bool, } diff --git a/lib/crowdb-protocol/tests/chunk_kv_catalog_test.rs b/lib/crowdb-protocol/tests/chunk_kv_catalog_test.rs index a0884e560..ec3fd5b05 100644 --- a/lib/crowdb-protocol/tests/chunk_kv_catalog_test.rs +++ b/lib/crowdb-protocol/tests/chunk_kv_catalog_test.rs @@ -92,6 +92,7 @@ fn catalog_rejects_holes_overlaps_and_checksum_changes() { fn serving_grant_digest_binds_sorted_partition_epochs() { let mut grant = ServingGrant { instance_id: 9, + process_epoch: 1, lease_sequence: 2, catalog_generation: 5, issued_at_ms: 100, @@ -219,7 +220,7 @@ fn split_transition_requires_retained_parent_and_exact_child_cutover() { parent_owner: owner.clone(), parent_epoch: 4, parent_artifact: artifact(5, 5), - retained_parent_artifact: artifact(6, 6), + retained_parent_artifact: artifact(5, 5), parent_next_epoch: 5, split_key: b"m".to_vec(), child: child(3, b"m", Some(b"z"), 7), @@ -234,7 +235,7 @@ fn split_transition_requires_retained_parent_and_exact_child_cutover() { source_partition_id: transition.parent_id, source_epoch: transition.parent_epoch, source_stream_name: transition.parent_artifact.stream_name, - source_stream_manifest_generation: 1, + source_stream_generation: 1, replay_offset: 0, cutover_offset: 11, base_root_manifest_generation: 1, @@ -243,7 +244,6 @@ fn split_transition_requires_retained_parent_and_exact_child_cutover() { cutover_seq: 11, target_stream_start_seq: 12, }; - transition.retained_parent_artifact.tail_overlay = Some(overlay.clone()); transition.child.artifact.tail_overlay = Some(overlay.clone()); transition.phase = SplitPhase::ChildPrepared; transition.readiness_proof = Some(SplitReadinessProof { @@ -256,12 +256,11 @@ fn split_transition_requires_retained_parent_and_exact_child_cutover() { child_applied_seq: 11, child_tree_manifest: 1, child_root_manifest_generation: 1, - retained_parent_tail_overlay: Some(overlay.clone()), child_tail_overlay: overlay, }); transition.validate().unwrap(); - verify_split_parent_recovery_formats(&mut transition); + verify_current_split_recovery_record(&transition); transition.child.range.start = b"n".to_vec(); assert_eq!( @@ -280,39 +279,25 @@ fn split_transition_requires_retained_parent_and_exact_child_cutover() { ); } -fn verify_split_parent_recovery_formats(transition: &mut SplitTransition) { - let legacy_json = serde_json::to_value(&*transition).unwrap(); - assert!(legacy_json["readiness_proof"]["retained_parent_tail_overlay"].is_object()); +fn verify_current_split_recovery_record(transition: &SplitTransition) { + let current = serde_json::to_value(transition).unwrap(); assert_eq!( - serde_json::from_value::(legacy_json).unwrap(), + serde_json::from_value::(current.clone()).unwrap(), *transition ); - transition.retained_parent_artifact = transition.parent_artifact.clone(); - let proof = transition.readiness_proof.as_mut().unwrap(); - proof.retained_parent_artifact = transition.retained_parent_artifact.clone(); - proof.retained_parent_tail_overlay = None; - transition.validate().unwrap(); - let mut stable_json = serde_json::to_value(&*transition).unwrap(); - stable_json["readiness_proof"] - .as_object_mut() - .unwrap() - .remove("retained_parent_tail_overlay"); - assert_eq!( - serde_json::from_value::(stable_json).unwrap(), - *transition - ); - let mut invalid = transition.clone(); - invalid - .readiness_proof - .as_mut() - .unwrap() - .retained_parent_tail_overlay = Some( - invalid - .readiness_proof - .as_ref() - .unwrap() - .child_tail_overlay - .clone(), - ); - assert!(invalid.validate().is_err()); + let mut retired = current.clone(); + retired["readiness_proof"]["retained_parent_tail_overlay"] = serde_json::Value::Null; + assert!(serde_json::from_value::(retired).is_err()); + let mut partial = current; + partial.as_object_mut().unwrap().remove("planned_at_ms"); + assert!(serde_json::from_value::(partial).is_err()); + for phase in [SplitPhase::Planned, SplitPhase::ChildPrepared] { + let mut invalid = transition.clone(); + invalid.phase = phase; + if phase == SplitPhase::Planned { + invalid.readiness_proof = None; + } + invalid.retained_parent_artifact.tree_id += 1; + assert!(invalid.validate().is_err()); + } } diff --git a/lib/crowdb-protocol/tests/chunk_kv_service_identity_test.rs b/lib/crowdb-protocol/tests/chunk_kv_service_identity_test.rs index 6409a7da9..ef5cc6e0f 100644 --- a/lib/crowdb-protocol/tests/chunk_kv_service_identity_test.rs +++ b/lib/crowdb-protocol/tests/chunk_kv_service_identity_test.rs @@ -4,16 +4,21 @@ use crowdb_protocol::common::ChunkKvExtra; #[test] -fn old_owner_registrations_do_not_invent_management_identity() { - let old = r#"{"capacity_bytes":1,"durable_bytes":0,"request_rate":0,"hosted":[]}"#; - let decoded: ChunkKvExtra = serde_json::from_str(old).unwrap(); - assert!(decoded.node_id.is_none()); - assert!(decoded.http_endpoint.is_none()); +fn current_owner_registration_requires_its_process_and_load_fields() { let current = ChunkKvExtra { + process_epoch: 1, node_id: Some(7), http_endpoint: Some("http://127.0.0.1:15101".into()), - ..decoded + ..ChunkKvExtra::default() }; let encoded = serde_json::to_vec(¤t).unwrap(); assert_eq!(serde_json::from_slice::(&encoded).unwrap(), current); + for field in ["process_epoch", "partition_loads"] { + let mut partial = serde_json::to_value(¤t).unwrap(); + partial.as_object_mut().unwrap().remove(field); + assert!( + serde_json::from_value::(partial).is_err(), + "{field}" + ); + } } diff --git a/lib/crowdb-protocol/tests/chunk_purpose_test.rs b/lib/crowdb-protocol/tests/chunk_purpose_test.rs index cad5bcbb0..68d382946 100644 --- a/lib/crowdb-protocol/tests/chunk_purpose_test.rs +++ b/lib/crowdb-protocol/tests/chunk_purpose_test.rs @@ -1,7 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. -use crowdb_protocol::chunk_stream::{StreamBinding, StreamName, StreamPurpose}; +use crowdb_protocol::chunk_stream::{StreamName, StreamPurpose, StreamRecord}; use crowdb_protocol::chunkdb::rpc::ChunkType; #[test] @@ -15,13 +15,13 @@ fn retired_and_unknown_chunk_types_are_not_supported() { } #[test] -fn stream_binding_requires_a_durable_purpose() { - let binding = StreamBinding::creating(StreamName::generate(), None, StreamPurpose::Wal); +fn stream_record_requires_a_durable_purpose() { + let binding = StreamRecord::creating(StreamName::generate(), None, StreamPurpose::Wal); let mut json = serde_json::to_value(&binding).unwrap(); assert_eq!( - serde_json::from_value::(json.clone()).unwrap(), + serde_json::from_value::(json.clone()).unwrap(), binding ); json.as_object_mut().unwrap().remove("purpose"); - assert!(serde_json::from_value::(json).is_err()); + assert!(serde_json::from_value::(json).is_err()); } diff --git a/lib/crowdb-protocol/tests/chunkdb_metadata_shape_test.rs b/lib/crowdb-protocol/tests/chunkdb_metadata_shape_test.rs new file mode 100644 index 000000000..2ab12b4f7 --- /dev/null +++ b/lib/crowdb-protocol/tests/chunkdb_metadata_shape_test.rs @@ -0,0 +1,43 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::chunkdb::rpc::{ChunkStrip, StripReservationGroup, StripReservationState}; + +#[test] +fn placement_metadata_requires_explicit_policy_and_repair_state() { + let strip = ChunkStrip::default(); + let value = serde_json::to_value(&strip).unwrap(); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + strip + ); + for field in ["placement_priority", "placement_repair_required"] { + let mut missing = value.clone(); + missing.as_object_mut().unwrap().remove(field); + assert!(serde_json::from_value::(missing).is_err(), "{field}"); + } +} + +#[test] +fn consumed_reservation_metadata_requires_explicit_planned_progress() { + let group = StripReservationGroup { + strips: vec![ChunkStrip::default()], + states: vec![StripReservationState::Consumed as i32], + planned_cursors: vec![1024], + planned_closed_sequences: vec![u32::MAX], + ..Default::default() + }; + let value = serde_json::to_value(&group).unwrap(); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + group + ); + for field in ["planned_cursors", "planned_closed_sequences"] { + let mut missing = value.clone(); + missing.as_object_mut().unwrap().remove(field); + assert!( + serde_json::from_value::(missing).is_err(), + "{field}" + ); + } +} diff --git a/lib/crowdb-protocol/tests/chunkdb_seal_wire_test.rs b/lib/crowdb-protocol/tests/chunkdb_seal_wire_test.rs new file mode 100644 index 000000000..a9da2a113 --- /dev/null +++ b/lib/crowdb-protocol/tests/chunkdb_seal_wire_test.rs @@ -0,0 +1,39 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::chunkdb::rpc::SealChunkRequest; +use crowdb_protocol::chunkdb_fb::{FBInt128, FBSealChunkRequest, FBSealChunkRequestArgs}; +use flatbuffers::FlatBufferBuilder; + +#[test] +fn seal_requests_keep_exact_bytes_including_empty_and_partial_kib_boundaries() { + assert_eq!(FBSealChunkRequest::VT_SEAL_BYTES, 10); + let id = FBInt128::new(2 << 56, 7); + for bytes in [0, 4, 1025, u64::MAX] { + let mut builder = FlatBufferBuilder::new(); + let request = FBSealChunkRequest::create( + &mut builder, + &FBSealChunkRequestArgs { + id: 41, + rpc_create_nano: 0, + chunk_id: Some(&id), + seal_bytes: bytes, + }, + ); + builder.finish(request, None); + let request = flatbuffers::root::(builder.finished_data()).unwrap(); + assert_eq!(request.id(), 41); + assert_eq!(request.chunk_id(), Some(&id)); + assert_eq!(request.seal_bytes(), bytes); + } +} + +#[test] +fn typed_seal_requires_an_explicit_byte_count_without_a_missing_field_default() { + assert!(serde_json::from_value::(serde_json::json!({"chunk_id": null})).is_err()); + let empty = serde_json::from_value::(serde_json::json!({ + "chunk_id": null, "seal_bytes": 0, + })) + .unwrap(); + assert_eq!(empty.seal_bytes, 0); +} diff --git a/lib/crowdb-protocol/tests/common/frame_prefix.rs b/lib/crowdb-protocol/tests/common/frame_prefix.rs new file mode 100644 index 000000000..225ed62ab --- /dev/null +++ b/lib/crowdb-protocol/tests/common/frame_prefix.rs @@ -0,0 +1,52 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::frame::{ + encode_append_group_views, FramePrefixScanner, FrameScanConfig, FrameScanError, FrameScanMode, + FrameScanStop, ScannedFrameExtent, +}; + +pub struct TestFramePrefix; + +impl TestFramePrefix { + pub const CHUNK: ChunkId = ChunkId { + high: 4 << 56, + low: 17, + }; + pub fn config(limit: usize) -> FrameScanConfig { + FrameScanConfig { + chunk_id: Self::CHUNK, + mode: FrameScanMode::Grouped, + physical_start: 0, + physical_limit: limit as u64, + required_end: 0, + logical_start: 100, + max_append_bytes: 64 * 1024 * 1024, + } + } + pub fn group(offset: usize, length: usize) -> Vec { + encode_append_group_views(Self::CHUNK, offset as u64, &Bytes::from(vec![0x5a; length]), 1) + .unwrap() + .concat() + } + pub fn feed( + scanner: &mut FramePrefixScanner, + bytes: &[u8], + fragment: usize, + max_extents: usize, + ) -> Result, FrameScanError> { + let mut offset = 0; + let mut result = Vec::new(); + while offset < bytes.len() && scanner.stop() == FrameScanStop::Running { + let end = (offset + fragment).min(bytes.len()); + let batch = scanner.push(offset as u64, &bytes[offset..end], max_extents)?; + assert!(batch.extents.len() <= max_extents); + assert!(batch.consumed > 0 || scanner.stop() != FrameScanStop::Running); + offset += batch.consumed; + result.extend(batch.extents); + } + Ok(result) + } +} diff --git a/lib/crowdb-protocol/tests/data/frame-vectors.txt b/lib/crowdb-protocol/tests/data/frame-vectors.txt new file mode 100644 index 000000000..fc0d78e7f --- /dev/null +++ b/lib/crowdb-protocol/tests/data/frame-vectors.txt @@ -0,0 +1,9 @@ +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +# name hex; payload=010203, chunk=(7,11), time=42 +ordinary-1 02010f0003002a0000000000000000010203af5aa0ae0000000000000007000000000000000b +ordinary-2 02020f0003002a00000000000000000102034e3e8d4e0000000000000007000000000000000b +ordinary-4 02040f0003002a00000000000000000102037d813b8b0000000000000007000000000000000b +ordinary-5 02050f0003002a0000000000000000010203225ddfd40000000000000007000000000000000b +head 02031b0003002a00000000000000010300000000000000010000000102037190e7b80000000000000007000000000000000b +follower 02031b0003002a0000000000000002001000000000000001000000010203d40ea4510000000000000007000000000000000b diff --git a/lib/crowdb-protocol/tests/diskio_wire_test.rs b/lib/crowdb-protocol/tests/diskio_wire_test.rs new file mode 100644 index 000000000..a6962c222 --- /dev/null +++ b/lib/crowdb-protocol/tests/diskio_wire_test.rs @@ -0,0 +1,95 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::diskio_fb::{ + FBDiskIoRetCode, FBDiskWriteRequest, FBDiskWriteRequestArgs, FBDiskWriteResponse, + FBDiskWriteResponseArgs, FBInt128, +}; +use flatbuffers::FlatBufferBuilder; + +#[test] +fn synchronized_write_uses_the_current_compact_layout() { + assert_eq!(FBDiskWriteRequest::VT_ORDERING_ZONE_OFFSET, 16); + assert_eq!(FBDiskWriteRequest::VT_WRITE_CREATE_TIME_MS, 18); + assert_eq!(FBDiskWriteRequest::VT_SYNC, 20); + assert_eq!(FBDiskWriteResponse::VT_SYNC_COMPLETED, 10); + let mut builder = FlatBufferBuilder::new(); + let disk = FBInt128([7; 16]); + let offset = FBDiskWriteRequest::create( + &mut builder, + &FBDiskWriteRequestArgs { + id: 41, + disk_id: Some(&disk), + zone_index: 3, + zone_offset: 8192, + ordering_zone_offset: 4096, + write_create_time_ms: 12345, + size: 4096, + sync: true, + ..Default::default() + }, + ); + builder.finish(offset, None); + let request = flatbuffers::root::(builder.finished_data()).unwrap(); + assert!(request.sync()); + assert_eq!(request.id(), 41); + assert_eq!(request.disk_id().unwrap().0, disk.0); + assert_eq!(request.zone_index(), 3); + assert_eq!(request.zone_offset(), 8192); + assert_eq!(request.ordering_zone_offset(), 4096); + assert_eq!(request.write_create_time_ms(), 12345); + assert_eq!(request.size(), 4096); +} + +#[test] +fn synchronization_requires_an_explicit_request_and_completion() { + let mut builder = FlatBufferBuilder::new(); + let offset = FBDiskWriteRequest::create( + &mut builder, + &FBDiskWriteRequestArgs { + id: 42, + write_create_time_ms: 12345, + ..Default::default() + }, + ); + builder.finish(offset, None); + let request = flatbuffers::root::(builder.finished_data()).unwrap(); + assert!(!request.sync()); + builder.reset(); + let offset = FBDiskWriteResponse::create( + &mut builder, + &FBDiskWriteResponseArgs { + id: 42, + ret_code: FBDiskIoRetCode::Success, + ..Default::default() + }, + ); + builder.finish(offset, None); + let response = flatbuffers::root::(builder.finished_data()).unwrap(); + assert_eq!(response.ret_code(), FBDiskIoRetCode::Success); + assert!(!response.sync_completed()); +} + +#[test] +fn synchronized_response_and_flush_failure_have_explicit_wire_values() { + assert_eq!(FBDiskIoRetCode::FsyncFailed.0, 8); + for (code, completed) in [ + (FBDiskIoRetCode::Success, true), + (FBDiskIoRetCode::FsyncFailed, false), + ] { + let mut builder = FlatBufferBuilder::new(); + let offset = FBDiskWriteResponse::create( + &mut builder, + &FBDiskWriteResponseArgs { + id: 43, + ret_code: code, + sync_completed: completed, + ..Default::default() + }, + ); + builder.finish(offset, None); + let response = flatbuffers::root::(builder.finished_data()).unwrap(); + assert_eq!(response.ret_code(), code); + assert_eq!(response.sync_completed(), completed); + } +} diff --git a/lib/crowdb-protocol/tests/fb_schema_test.rs b/lib/crowdb-protocol/tests/fb_schema_test.rs index 897f5d66b..a4c09ce7e 100644 --- a/lib/crowdb-protocol/tests/fb_schema_test.rs +++ b/lib/crowdb-protocol/tests/fb_schema_test.rs @@ -142,8 +142,8 @@ fn disk_io_ret_code_values() { assert_eq!(FBDiskIoRetCode::PartialWrite.0, 4); assert_eq!(FBDiskIoRetCode::InvalidAlignment.0, 5); assert_eq!(FBDiskIoRetCode::ConnectionError.0, 6); - assert_eq!(FBDiskIoRetCode::StaleAllocation.0, 7); - assert_eq!(FBDiskIoRetCode::OldRequest.0, 8); + assert_eq!(FBDiskIoRetCode::OldRequest.0, 7); + assert_eq!(FBDiskIoRetCode::FsyncFailed.0, 8); } #[test] @@ -164,6 +164,7 @@ fn disk_write_request_round_trip() { size: 4096, ordering_zone_offset: 2048, write_create_time_ms: 1234, + sync: false, }, ); fbb.finish(req, None); diff --git a/lib/crowdb-protocol/tests/frame_prefix_scan_test.rs b/lib/crowdb-protocol/tests/frame_prefix_scan_test.rs new file mode 100644 index 000000000..1e85e4bc2 --- /dev/null +++ b/lib/crowdb-protocol/tests/frame_prefix_scan_test.rs @@ -0,0 +1,133 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#[path = "common/frame_prefix.rs"] +mod frame_prefix; + +use crowdb_protocol::frame::{ + FramePrefixScanner, FrameScanError, FrameScanMode, FrameScanStop, MAX_FRAME_GROUP_PAYLOAD_BYTES, +}; +use frame_prefix::TestFramePrefix as Test; +use sha2::{Digest, Sha256}; + +#[path = "frame_prefix_scan_test/faults.rs"] +mod faults; + +#[test] +fn fragmented_input_preserves_complete_groups_offsets_and_exact_prefix_digest() { + let first = Test::group(0, 1); + let large_len = 2 * MAX_FRAME_GROUP_PAYLOAD_BYTES + 17; + let large = Test::group(first.len(), large_len); + let last = Test::group(first.len() + large.len(), 1); + let prefix = [first.as_slice(), large.as_slice(), last.as_slice()].concat(); + let mut bytes = prefix.clone(); + bytes.resize(bytes.len() + 512, 0); + for fragment in [1, 13, 14, 27, 4095, 65535, 1024 * 1024] { + let mut scanner = FramePrefixScanner::new(Test::config(bytes.len())).unwrap(); + let extents = Test::feed(&mut scanner, &bytes, fragment, 1).unwrap(); + let summary = scanner.finish().unwrap(); + assert_eq!(summary.stop, FrameScanStop::Unwritten); + assert_eq!(summary.physical_end, prefix.len() as u64); + assert_eq!(summary.logical_end, 102 + large_len as u64); + assert_eq!(summary.complete_extents, 3); + assert_eq!(summary.frames_parsed, 5); + assert_eq!(summary.digest, <[u8; 32]>::from(Sha256::digest(&prefix))); + assert_eq!(extents.len(), 3); + assert_eq!(extents[0].physical_offset, 0); + assert_eq!(extents[1].physical_offset, first.len() as u64); + assert_eq!(extents[1].physical_length, large.len() as u64); + assert_eq!(extents[1].logical_offset, 101); + assert_eq!(extents[1].logical_length, large_len as u64); + assert_eq!(extents[2].logical_offset, 101 + large_len as u64); + } +} + +#[test] +fn a_missing_follower_stops_at_the_previous_append_and_cannot_skip_to_later_groups() { + let first = Test::group(0, 1); + let group = Test::group(first.len(), 2 * MAX_FRAME_GROUP_PAYLOAD_BYTES + 1); + let later = Test::group(first.len() + group.len(), 1); + let mut bytes = [first.as_slice(), group.as_slice(), later.as_slice()].concat(); + bytes[first.len() + 65536..first.len() + group.len()].fill(0); + let mut config = Test::config(bytes.len()); + config.required_end = first.len() as u64; + let mut scanner = FramePrefixScanner::new(config).unwrap(); + let extents = Test::feed(&mut scanner, &bytes, 1024 * 1024, 1024).unwrap(); + let summary = scanner.finish().unwrap(); + assert_eq!(extents.len(), 1); + assert_eq!(summary.physical_end, first.len() as u64); + assert_eq!(summary.logical_end, 101); + assert_eq!(summary.frames_parsed, 2); + assert_eq!(summary.digest, <[u8; 32]>::from(Sha256::digest(&first))); +} + +#[test] +fn exact_capacity_and_small_extent_batches_need_no_extra_header_input() { + let mut bytes = Vec::new(); + for _ in 0..3000 { + bytes.extend(Test::group(bytes.len(), 1)); + } + let mut scanner = FramePrefixScanner::new(Test::config(bytes.len())).unwrap(); + let extents = Test::feed(&mut scanner, &bytes, 1024 * 1024, 16).unwrap(); + let summary = scanner.finish().unwrap(); + assert_eq!(summary.stop, FrameScanStop::ExactCapacity); + assert_eq!(summary.physical_end, bytes.len() as u64); + assert_eq!(summary.observed_end, summary.physical_end); + assert_eq!(summary.complete_extents, 3000); + assert_eq!(extents.len(), 3000); + assert!(scanner.push(summary.physical_end, &[0], 1).is_err()); +} + +#[test] +fn an_incomplete_capacity_suffix_is_excluded_but_a_short_observation_is_an_error() { + let first = Test::group(0, 1); + let mut bytes = first.clone(); + bytes.extend(&Test::group(first.len(), 1)[..7]); + let mut config = Test::config(bytes.len()); + config.required_end = first.len() as u64; + let mut scanner = FramePrefixScanner::new(config).unwrap(); + Test::feed(&mut scanner, &bytes, 13, 16).unwrap(); + let summary = scanner.finish().unwrap(); + assert_eq!(summary.stop, FrameScanStop::IncompleteAtCapacity); + assert_eq!(summary.physical_end, first.len() as u64); + let mut scanner = FramePrefixScanner::new(Test::config(bytes.len() + 1)).unwrap(); + Test::feed(&mut scanner, &bytes, 13, 16).unwrap(); + assert!(matches!( + scanner.finish(), + Err(FrameScanError::IncompleteObservation { .. }) + )); +} + +#[test] +fn canonical_ordinary_frames_keep_their_independent_completion_boundary() { + use crowdb_protocol::frame::{encode_frame, FrameMagic}; + let bytes = encode_frame(FrameMagic::RepoLargeV2, Test::CHUNK, b"data", 1).unwrap(); + let mut config = Test::config(bytes.len()); + config.mode = FrameScanMode::Ordinary; + config.required_end = 3; + let mut scanner = FramePrefixScanner::new(config).unwrap(); + let extents = Test::feed(&mut scanner, &bytes, 1, 1).unwrap(); + let summary = scanner.finish().unwrap(); + assert_eq!(summary.stop, FrameScanStop::ExactCapacity); + assert_eq!(summary.physical_end, bytes.len() as u64); + assert_eq!(extents[0].logical_length, 4); +} + +#[test] +fn a_validated_checkpoint_starts_at_its_original_physical_and_logical_offsets() { + let prefix = Test::group(0, 1); + let bytes = Test::group(prefix.len(), 100_000); + let mut config = Test::config(prefix.len() + bytes.len()); + config.physical_start = prefix.len() as u64; + config.required_end = config.physical_start; + config.logical_start = 700; + let mut scanner = FramePrefixScanner::new(config).unwrap(); + let batch = scanner.push(config.physical_start, &bytes, 1).unwrap(); + assert_eq!(batch.consumed, bytes.len()); + assert_eq!(batch.extents.len(), 1); + assert_eq!(batch.extents[0].physical_offset, config.physical_start); + assert_eq!(batch.extents[0].logical_offset, 700); + let summary = scanner.finish().unwrap(); + assert_eq!(summary.logical_end, 100_700); + assert_eq!(summary.digest, <[u8; 32]>::from(Sha256::digest(&bytes))); +} diff --git a/lib/crowdb-protocol/tests/frame_prefix_scan_test/faults.rs b/lib/crowdb-protocol/tests/frame_prefix_scan_test/faults.rs new file mode 100644 index 000000000..3c12393e2 --- /dev/null +++ b/lib/crowdb-protocol/tests/frame_prefix_scan_test/faults.rs @@ -0,0 +1,123 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use super::*; +use crowdb_protocol::frame::{encode_frame, FrameError, FrameMagic}; + +#[test] +fn corrupt_or_wrong_identity_frames_are_errors_instead_of_shorter_candidates() { + let first = Test::group(0, 1); + let second = Test::group(first.len(), 1); + for offset in [27, second.len() - 1] { + let mut bad = second.clone(); + bad[offset] ^= 1; + let bytes = [first.as_slice(), bad.as_slice()].concat(); + let mut scanner = FramePrefixScanner::new(Test::config(bytes.len())).unwrap(); + assert!(matches!( + Test::feed(&mut scanner, &bytes, 1024, 16), + Err(FrameScanError::Frame { offset: 48, .. }) + )); + assert_eq!(scanner.stop(), FrameScanStop::Failed); + assert!(matches!(scanner.finish(), Err(FrameScanError::FailedObservation))); + } +} + +#[test] +fn wrong_roles_and_unsupported_revisions_are_rejected() { + let mut unsupported = Test::group(0, 1); + unsupported[..2].copy_from_slice(&0x0301_u16.to_le_bytes()); + let ordinary = encode_frame(FrameMagic::RepoSmallV2, Test::CHUNK, b"data", 1).unwrap(); + for bytes in [unsupported, ordinary] { + let mut scanner = FramePrefixScanner::new(Test::config(bytes.len())).unwrap(); + assert!(Test::feed(&mut scanner, &bytes, 1024, 16).is_err()); + } + let bytes = Test::group(0, 1); + let mut config = Test::config(bytes.len()); + config.mode = FrameScanMode::Ordinary; + let mut scanner = FramePrefixScanner::new(config).unwrap(); + assert!(Test::feed(&mut scanner, &bytes, 1024, 16).is_err()); +} + +#[test] +fn a_full_header_exceeding_allocated_capacity_is_not_a_healthy_short_candidate() { + let mut bytes = Test::group(0, 1); + bytes[4..6].copy_from_slice(&100_u16.to_le_bytes()); + let mut scanner = FramePrefixScanner::new(Test::config(bytes.len())).unwrap(); + assert!(matches!( + Test::feed(&mut scanner, &bytes, 1024, 16), + Err(FrameScanError::Frame { + source: FrameError::InvalidRegionLength, + .. + }) + )); +} + +#[test] +fn grouped_checkpoints_must_be_covered_exact_append_boundaries() { + let bytes = [Test::group(0, 1), Test::group(48, 1)].concat(); + let mut config = Test::config(bytes.len()); + config.required_end = 49; + let mut scanner = FramePrefixScanner::new(config).unwrap(); + assert!(matches!( + Test::feed(&mut scanner, &bytes, 1024, 16), + Err(FrameScanError::CheckpointBoundary { .. }) + )); + let mut bytes = Test::group(0, 1); + bytes.resize(100, 0); + config.physical_limit = bytes.len() as u64; + let mut scanner = FramePrefixScanner::new(config).unwrap(); + assert!(matches!( + Test::feed(&mut scanner, &bytes, 1024, 16), + Err(FrameScanError::RequiredPrefix { .. }) + )); +} + +#[test] +fn declared_group_limits_and_arithmetic_overflow_fail_without_allocating_group_payloads() { + let bytes = Test::group(0, MAX_FRAME_GROUP_PAYLOAD_BYTES + 1); + let mut config = Test::config(bytes.len()); + config.max_append_bytes = 1; + let mut scanner = FramePrefixScanner::new(config).unwrap(); + assert!(matches!( + Test::feed(&mut scanner, &bytes, 65536, 1), + Err(FrameScanError::Frame { + source: FrameError::InvalidGroup, + .. + }) + )); + let mut config = Test::config(bytes.len()); + config.logical_start = u64::MAX; + let mut scanner = FramePrefixScanner::new(config).unwrap(); + assert!(matches!( + Test::feed(&mut scanner, &bytes, 65536, 1), + Err(FrameScanError::Frame { + source: FrameError::LengthOverflow, + .. + }) + )); +} + +#[test] +fn input_and_output_budgets_reject_gaps_and_unbounded_batches() { + let bytes = Test::group(0, 1); + let mut scanner = FramePrefixScanner::new(Test::config(bytes.len())).unwrap(); + assert!(matches!( + scanner.push(1, &bytes[..1], 1), + Err(FrameScanError::OutOfOrder) + )); + for budget in [0, 1025] { + let mut scanner = FramePrefixScanner::new(Test::config(bytes.len())).unwrap(); + assert!(matches!( + scanner.push(0, &bytes, budget), + Err(FrameScanError::InvalidBounds) + )); + } + let mut config = Test::config(1024 * 1024 + 1); + let mut scanner = FramePrefixScanner::new(config).unwrap(); + assert!(matches!( + scanner.push(0, &vec![0; 1024 * 1024 + 1], 1), + Err(FrameScanError::InvalidBounds) + )); + config.required_end = config.physical_limit + 1; + assert!(FramePrefixScanner::new(config).is_err()); +} diff --git a/lib/crowdb-protocol/tests/frame_revision_test.rs b/lib/crowdb-protocol/tests/frame_revision_test.rs new file mode 100644 index 000000000..1665de93f --- /dev/null +++ b/lib/crowdb-protocol/tests/frame_revision_test.rs @@ -0,0 +1,252 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use bytes::Bytes; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::frame::{ + encode_append_group_views, encode_frame, encode_frame_regions, encode_frame_views, parse_append_group, + parse_frame, parse_frame_views, AppendGroupValidator, ChunkLocation, FrameError, FrameLayout, FrameMagic, + FrameRole, FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, MAX_FRAME_BYTES, MAX_FRAME_GROUP_PAYLOAD_BYTES, + MAX_FRAME_V2_PAYLOAD_BYTES, +}; + +const CHUNK: ChunkId = ChunkId { high: 7, low: 11 }; + +fn vectors() -> impl Iterator)> { + include_str!("data/frame-vectors.txt") + .lines() + .filter(|line| !line.starts_with('#')) + .map(|line| { + let (name, hex) = line.split_once(' ').unwrap(); + let bytes = hex + .as_bytes() + .chunks_exact(2) + .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) + .collect(); + (name, bytes) + }) +} + +#[test] +fn both_languages_read_all_shared_vectors_and_emit_canonical_v2() { + let vectors: Vec<_> = vectors().collect(); + assert_eq!(vectors.len(), 6); + for (name, bytes) in vectors { + let parsed = parse_frame(&bytes, CHUNK).unwrap(); + assert_eq!(parsed.payload, &[1, 2, 3], "{name}"); + assert_eq!(parsed.header.write_time_ms, 42); + if name.starts_with("ordinary") { + assert_eq!( + encode_frame(parsed.header.magic, CHUNK, parsed.payload, 42).unwrap(), + bytes + ); + } else if name == "head" { + assert_eq!( + encode_append_group_views(CHUNK, 4096, &Bytes::from_static(&[1, 2, 3]), 42) + .unwrap() + .concat(), + bytes + ); + assert_eq!(parse_append_group(&bytes, CHUNK, 4096).unwrap().frame_count, 1); + } + for split in 0..bytes.len() { + let views = [&bytes[..split], &[], &bytes[split..]]; + let scattered = parse_frame_views(&views, CHUNK).unwrap(); + assert_eq!(scattered.role, parsed.role, "{name} at {split}"); + assert_eq!(scattered.physical_length, bytes.len()); + } + } +} + +#[test] +fn ordinary_v2_regions_and_views_preserve_payload_ownership() { + for magic in [ + FrameMagic::RepoSmallV2, + FrameMagic::RepoLargeV2, + FrameMagic::BtreePageV2, + FrameMagic::PageIndexV2, + ] { + let payload = Bytes::from(vec![0x91; MAX_FRAME_V2_PAYLOAD_BYTES]); + let mut header = [0; FRAME_V2_HEADER_BYTES]; + let mut footer = [0; FRAME_FOOTER_BYTES]; + encode_frame_regions(magic, CHUNK, &payload, 42, &mut header, &mut footer).unwrap(); + let views = encode_frame_views(magic, CHUNK, vec![payload.clone()], 42).unwrap(); + assert_eq!(views[1].as_ptr(), payload.as_ptr()); + let bytes = views.concat(); + assert_eq!(bytes.len(), MAX_FRAME_BYTES); + assert_eq!(&bytes[..header.len()], header); + assert_eq!(&bytes[bytes.len() - footer.len()..], footer); + assert_eq!(parse_frame(&bytes, CHUNK).unwrap().role, FrameRole::Ordinary); + assert!(encode_frame(magic, CHUNK, &vec![0; MAX_FRAME_V2_PAYLOAD_BYTES + 1], 42).is_err()); + } + assert!(encode_frame(FrameMagic::StreamV2, CHUNK, &[1], 42).is_err()); +} + +#[test] +fn grouped_append_is_zero_copy_and_requires_every_follower() { + for length in [ + 1, + MAX_FRAME_GROUP_PAYLOAD_BYTES, + MAX_FRAME_GROUP_PAYLOAD_BYTES + 1, + 3 * MAX_FRAME_GROUP_PAYLOAD_BYTES + 17, + ] { + let payload = Bytes::from(vec![0x51; length]); + let views = encode_append_group_views(CHUNK, 4096, &payload, 42).unwrap(); + for (index, parts) in views.chunks_exact(3).enumerate() { + assert_eq!( + parts[1].as_ptr(), + payload.slice(index * MAX_FRAME_GROUP_PAYLOAD_BYTES..).as_ptr() + ); + } + let bytes = views.concat(); + let group = parse_append_group(&bytes, CHUNK, 4096).unwrap(); + assert_eq!(group.logical_length, length as u64); + assert_eq!( + group.physical_length, + FrameLayout::StreamGroupV2.physical_length(length as u64).unwrap() + ); + assert_eq!( + group.frame_count as usize, + length.div_ceil(MAX_FRAME_GROUP_PAYLOAD_BYTES) + ); + for cursor in (0..bytes.len()).step_by(MAX_FRAME_BYTES) { + for end in [ + cursor.saturating_sub(1), + cursor, + (cursor + 1).min(bytes.len() - 1), + ] { + assert!( + parse_append_group(&bytes[..end], CHUNK, 4096).is_err(), + "length={length}, end={end}" + ); + } + } + assert!(parse_append_group(&bytes[..bytes.len() - 1], CHUNK, 4096).is_err()); + if group.frame_count > 1 { + assert!(parse_append_group(&bytes, CHUNK, 4097).is_err()); + } + } +} + +#[test] +fn valid_crc_does_not_make_reordered_or_cross_group_followers_valid() { + let payload = Bytes::from(vec![0x51; MAX_FRAME_GROUP_PAYLOAD_BYTES * 3]); + let views = encode_append_group_views(CHUNK, 4096, &payload, 42).unwrap(); + let frames: Vec<_> = views.chunks_exact(3).map(<[Bytes]>::concat).collect(); + let reordered = [frames[0].clone(), frames[2].clone(), frames[1].clone()].concat(); + assert_eq!( + parse_append_group(&reordered, CHUNK, 4096).err(), + Some(FrameError::InvalidGroup) + ); + let other = encode_append_group_views(CHUNK, 8192, &payload, 42).unwrap(); + let mixed = [frames[0].clone(), other[3..6].concat(), frames[2].clone()].concat(); + assert_eq!( + parse_append_group(&mixed, CHUNK, 4096).err(), + Some(FrameError::InvalidGroup) + ); + let mut damaged = views.concat(); + damaged[MAX_FRAME_BYTES + 15] ^= 1; + assert!(parse_append_group(&damaged, CHUNK, 4096).is_err()); +} + +#[test] +fn group_validator_keeps_previous_boundary_until_finish() { + let bytes = encode_append_group_views( + CHUNK, + 4096, + &Bytes::from(vec![7; MAX_FRAME_GROUP_PAYLOAD_BYTES + 1]), + 42, + ) + .unwrap() + .concat(); + let head = parse_frame(&bytes, CHUNK).unwrap(); + let validator = AppendGroupValidator::new(4096, &head).unwrap(); + assert!(!validator.is_complete()); + assert_eq!(validator.finish().err(), Some(FrameError::InvalidGroup)); + let mut validator = AppendGroupValidator::new(4096, &head).unwrap(); + let tail = parse_frame(&bytes[head.physical_length..], CHUNK).unwrap(); + assert!(validator + .push(4096 + head.physical_length as u64 + 1, &tail) + .is_err()); + validator.push(4096 + head.physical_length as u64, &tail).unwrap(); + assert!(validator.is_complete()); + assert!(validator.push(4096 + bytes.len() as u64, &tail).is_err()); + assert_eq!(validator.finish().unwrap().physical_length, bytes.len() as u64); +} + +#[test] +fn geometry_maps_boundaries_and_rejects_overflow_for_each_revision() { + for layout in [FrameLayout::OrdinaryV2, FrameLayout::StreamGroupV2] { + let capacity = layout.payload_capacity(); + let location = ChunkLocation { + chunk_id: CHUNK, + frame_offset: 4096, + logical_length: capacity * 2 + 7, + }; + let end = 4096 + layout.physical_length(location.logical_length).unwrap(); + assert_eq!( + layout + .physical_range(location, capacity - 1..capacity + 1) + .unwrap(), + 4096..4096 + 2 * MAX_FRAME_BYTES as u64 + ); + assert_eq!( + layout + .physical_range(location, capacity * 2..capacity * 2 + 7) + .unwrap(), + 4096 + 2 * MAX_FRAME_BYTES as u64..end + ); + assert!(layout.physical_length(u64::MAX).is_err()); + assert!(layout + .physical_range( + ChunkLocation { + frame_offset: u64::MAX, + ..location + }, + 0..1 + ) + .is_err()); + assert!(layout + .physical_range(location, 0..location.logical_length + 1) + .is_err()); + assert_eq!(layout.physical_length(0).unwrap(), 0); + } +} + +#[test] +fn malformed_extensions_fail_without_panics_and_v2_crc_is_unambiguous() { + for (_, mut bytes) in vectors().filter(|(name, _)| name.starts_with("ordinary")) { + bytes[14] = 1; + assert!(parse_frame(&bytes, CHUNK).is_err()); + bytes[2..4].copy_from_slice(&14_u16.to_le_bytes()); + assert!(parse_frame_views(&[&bytes], CHUNK).is_err()); + } + let mut bytes = encode_frame(FrameMagic::RepoSmallV2, CHUNK, &[1, 2, 3], 42).unwrap(); + let footer = bytes.len() - FRAME_FOOTER_BYTES; + let prefix_crc = crowdb_common::ec_isal::crc32c_update(0, &bytes[..footer]); + let footer_inclusive_crc = crowdb_common::ec_isal::crc32c_update(prefix_crc, &bytes[footer + 4..]); + bytes[footer..footer + 4].copy_from_slice(&footer_inclusive_crc.to_le_bytes()); + assert_eq!( + parse_frame(&bytes, CHUNK).err(), + Some(FrameError::ChecksumMismatch) + ); + assert_eq!( + parse_frame_views(&[&bytes], CHUNK).err(), + Some(FrameError::ChecksumMismatch) + ); + assert!(encode_append_group_views(CHUNK, 0, &Bytes::new(), 42).is_err()); + assert!(encode_append_group_views(CHUNK, u64::MAX, &Bytes::from_static(b"x"), 42).is_err()); +} + +#[test] +fn removed_magics_are_unsupported() { + let canonical = encode_frame(FrameMagic::RepoSmallV2, CHUNK, b"data", 42).unwrap(); + for magic in [0x0101_u16, 0x0201, 0x0301, 0x0401, 0x0501, 0x0103] { + let mut bytes = canonical.clone(); + bytes[..2].copy_from_slice(&magic.to_le_bytes()); + assert!(matches!(parse_frame(&bytes, CHUNK), Err(FrameError::UnknownMagic(value)) if value == magic)); + assert!( + matches!(parse_frame_views(&[&bytes], CHUNK), Err(FrameError::UnknownMagic(value)) if value == magic) + ); + } +} diff --git a/lib/crowdb-protocol/tests/frame_test.rs b/lib/crowdb-protocol/tests/frame_test.rs index 05bcecd55..7d2ca43d9 100644 --- a/lib/crowdb-protocol/tests/frame_test.rs +++ b/lib/crowdb-protocol/tests/frame_test.rs @@ -4,20 +4,20 @@ use crowdb_protocol::common::ChunkId; use crowdb_protocol::frame::{ encode_frame, encode_frame_regions, encode_frames, merge_adjacent_locations, parse_frame, - parse_frame_views, ChunkLocation, FrameError, FrameMagic, FRAME_FOOTER_BYTES, FRAME_HEADER_PREFIX_BYTES, - MAX_FRAME_BYTES, MAX_FRAME_PAYLOAD_BYTES, + parse_frame_views, ChunkLocation, FrameError, FrameMagic, FRAME_FOOTER_BYTES, FRAME_V2_HEADER_BYTES, + MAX_FRAME_BYTES, MAX_FRAME_V2_PAYLOAD_BYTES, }; const CHUNK: ChunkId = ChunkId { high: 7, low: 11 }; -const REPO_SMALL_VECTOR: [u8; 37] = [ - 0x01, 0x01, 0x0E, 0x00, 0x03, 0x00, 0x2A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, - 0xEB, 0x45, 0x53, 0x37, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x0B, +const REPO_SMALL_VECTOR: [u8; 38] = [ + 0x02, 0x01, 0x0f, 0x00, 0x03, 0x00, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, + 0x03, 0xaf, 0x5a, 0xa0, 0xae, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0b, ]; #[test] -fn frame_matches_cross_language_vector() { - let encoded = encode_frame(FrameMagic::RepoSmallV1, CHUNK, &[1, 2, 3], 42).unwrap(); +fn frame_matches_canonical_cross_language_vector() { + let encoded = encode_frame(FrameMagic::RepoSmallV2, CHUNK, &[1, 2, 3], 42).unwrap(); assert_eq!(encoded, REPO_SMALL_VECTOR); assert_eq!( parse_frame(&REPO_SMALL_VECTOR, CHUNK).unwrap().payload, @@ -28,7 +28,7 @@ fn frame_matches_cross_language_vector() { #[test] fn split_frame_views_verify_without_assembling_payload() { let payload = vec![0x6b; 1024]; - let frame = encode_frame(FrameMagic::RepoSmallV1, CHUNK, &payload, 42).unwrap(); + let frame = encode_frame(FrameMagic::RepoSmallV2, CHUNK, &payload, 42).unwrap(); let boundaries = [3, 11, 127, frame.len() - 9, frame.len() - 2]; let mut views = Vec::new(); let mut start = 0; @@ -37,7 +37,7 @@ fn split_frame_views_verify_without_assembling_payload() { start = end; } let parsed = parse_frame_views(&views, CHUNK).unwrap(); - assert_eq!(parsed.header.magic, FrameMagic::RepoSmallV1); + assert_eq!(parsed.header.magic, FrameMagic::RepoSmallV2); assert_eq!(parsed.physical_length, frame.len()); let mut corrupted = frame; @@ -56,11 +56,11 @@ fn split_frame_views_verify_without_assembling_payload() { #[test] fn frame_crc_matches_bitwise_reference_across_header_and_payload() { - for length in [0, 1, 17, 1024, MAX_FRAME_PAYLOAD_BYTES] { + for length in [0, 1, 17, 1024, MAX_FRAME_V2_PAYLOAD_BYTES] { let payload: Vec = (0..length) .map(|index| u8::try_from(index % 251).unwrap()) .collect(); - let frame = encode_frame(FrameMagic::RepoLargeV1, CHUNK, &payload, 42).unwrap(); + let frame = encode_frame(FrameMagic::RepoLargeV2, CHUNK, &payload, 42).unwrap(); let footer = frame.len() - FRAME_FOOTER_BYTES; let mut crc = 0_u32; for byte in &frame[..footer] { @@ -76,24 +76,24 @@ fn frame_crc_matches_bitwise_reference_across_header_and_payload() { #[test] fn frame_round_trips_and_has_canonical_maximum_size() { - let payload = vec![0xA5; MAX_FRAME_PAYLOAD_BYTES]; - let frame = encode_frame(FrameMagic::RepoLargeV1, CHUNK, &payload, 42).unwrap(); + let payload = vec![0xA5; MAX_FRAME_V2_PAYLOAD_BYTES]; + let frame = encode_frame(FrameMagic::RepoLargeV2, CHUNK, &payload, 42).unwrap(); assert_eq!(frame.len(), MAX_FRAME_BYTES); let decoded = parse_frame(&frame, CHUNK).unwrap(); - assert_eq!(decoded.header.magic, FrameMagic::RepoLargeV1); - assert_eq!(decoded.header.payload_offset as usize, FRAME_HEADER_PREFIX_BYTES); + assert_eq!(decoded.header.magic, FrameMagic::RepoLargeV2); + assert_eq!(decoded.header.payload_offset as usize, FRAME_V2_HEADER_BYTES); assert_eq!(decoded.payload, payload); assert_eq!(decoded.physical_length, MAX_FRAME_BYTES); } #[test] fn separated_frame_regions_match_contiguous_encoding() { - let payload = vec![0x5a; MAX_FRAME_PAYLOAD_BYTES]; - let expected = encode_frame(FrameMagic::RepoLargeV1, CHUNK, &payload, 42).unwrap(); - let mut header = [0; FRAME_HEADER_PREFIX_BYTES]; + let payload = vec![0x5a; MAX_FRAME_V2_PAYLOAD_BYTES]; + let expected = encode_frame(FrameMagic::RepoLargeV2, CHUNK, &payload, 42).unwrap(); + let mut header = [0; FRAME_V2_HEADER_BYTES]; let mut footer = [0; FRAME_FOOTER_BYTES]; encode_frame_regions( - FrameMagic::RepoLargeV1, + FrameMagic::RepoLargeV2, CHUNK, &payload, 42, @@ -105,10 +105,10 @@ fn separated_frame_regions_match_contiguous_encoding() { assert_eq!(&expected[..header.len()], &header); assert_eq!(&expected[header.len()..header.len() + payload.len()], &payload); assert_eq!(&expected[header.len() + payload.len()..], &footer); - let mut short_header = [0; FRAME_HEADER_PREFIX_BYTES - 1]; + let mut short_header = [0; FRAME_V2_HEADER_BYTES - 1]; assert_eq!( encode_frame_regions( - FrameMagic::RepoLargeV1, + FrameMagic::RepoLargeV2, CHUNK, &payload, 42, @@ -121,26 +121,26 @@ fn separated_frame_regions_match_contiguous_encoding() { #[test] fn large_payload_has_full_interior_and_variable_tail_frames() { - let payload = vec![0xA5; MAX_FRAME_PAYLOAD_BYTES + 7]; - let frames = encode_frames(FrameMagic::RepoLargeV1, CHUNK, &payload, 42).unwrap(); + let payload = vec![0xA5; MAX_FRAME_V2_PAYLOAD_BYTES + 7]; + let frames = encode_frames(FrameMagic::RepoLargeV2, CHUNK, &payload, 42).unwrap(); assert_eq!(frames.len(), 2); assert_eq!(frames[0].len(), MAX_FRAME_BYTES); assert_eq!(parse_frame(&frames[1], CHUNK).unwrap().payload, &[0xA5; 7]); - assert!(encode_frames(FrameMagic::RepoLargeV1, CHUNK, &[], 42) + assert!(encode_frames(FrameMagic::RepoLargeV2, CHUNK, &[], 42) .unwrap() .is_empty()); } #[test] fn frame_rejects_corruption_and_wrong_chunk() { - let mut frame = encode_frame(FrameMagic::RepoSmallV1, CHUNK, b"payload", 42).unwrap(); - frame[FRAME_HEADER_PREFIX_BYTES] ^= 1; + let mut frame = encode_frame(FrameMagic::RepoSmallV2, CHUNK, b"payload", 42).unwrap(); + frame[FRAME_V2_HEADER_BYTES] ^= 1; assert!(matches!( parse_frame(&frame, CHUNK), Err(FrameError::ChecksumMismatch) )); - let frame = encode_frame(FrameMagic::RepoSmallV1, CHUNK, b"payload", 42).unwrap(); + let frame = encode_frame(FrameMagic::RepoSmallV2, CHUNK, b"payload", 42).unwrap(); assert!(matches!( parse_frame(&frame, ChunkId { high: 7, low: 12 }), Err(FrameError::ChunkIdMismatch) @@ -149,7 +149,7 @@ fn frame_rejects_corruption_and_wrong_chunk() { #[test] fn location_maps_large_object_subrange_to_containing_frames() { - let logical = (MAX_FRAME_PAYLOAD_BYTES as u64) + 10; + let logical = (MAX_FRAME_V2_PAYLOAD_BYTES as u64) + 10; let location = ChunkLocation { chunk_id: CHUNK, frame_offset: 4096, @@ -157,11 +157,11 @@ fn location_maps_large_object_subrange_to_containing_frames() { }; assert_eq!( location.physical_length().unwrap(), - (MAX_FRAME_BYTES + FRAME_HEADER_PREFIX_BYTES + 10 + FRAME_FOOTER_BYTES) as u64 + (MAX_FRAME_BYTES + FRAME_V2_HEADER_BYTES + 10 + FRAME_FOOTER_BYTES) as u64 ); assert_eq!( location - .physical_range_for_subrange((MAX_FRAME_PAYLOAD_BYTES as u64 - 1)..logical) + .physical_range_for_subrange((MAX_FRAME_V2_PAYLOAD_BYTES as u64 - 1)..logical) .unwrap(), 4096..(4096 + location.physical_length().unwrap()) ); @@ -172,7 +172,7 @@ fn only_frame_aligned_locations_merge() { let aligned = ChunkLocation { chunk_id: CHUNK, frame_offset: 0, - logical_length: MAX_FRAME_PAYLOAD_BYTES as u64, + logical_length: MAX_FRAME_V2_PAYLOAD_BYTES as u64, }; let adjacent = ChunkLocation { chunk_id: CHUNK, @@ -201,18 +201,18 @@ fn encoding_payload_views_preserves_pointers_and_canonical_bytes() { let pieces = vec![payload.slice(..7), payload.slice(7..13), payload.slice(13..)]; let pointers: Vec<_> = pieces.iter().map(|view| view.as_ptr()).collect(); let encoded = - crowdb_protocol::frame::encode_frame_views(FrameMagic::RepoSmallV1, CHUNK, pieces, 42).unwrap(); + crowdb_protocol::frame::encode_frame_views(FrameMagic::RepoSmallV2, CHUNK, pieces, 42).unwrap(); assert_eq!( encoded[1..4].iter().map(|view| view.as_ptr()).collect::>(), pointers ); assert_eq!( encoded.concat(), - encode_frame(FrameMagic::RepoSmallV1, CHUNK, &payload, 42).unwrap() + encode_frame(FrameMagic::RepoSmallV2, CHUNK, &payload, 42).unwrap() ); let refs: Vec<_> = encoded.iter().map(bytes::Bytes::as_ref).collect(); assert_eq!( parse_frame_views(&refs, CHUNK).unwrap().physical_length, - payload.len() + 34 + payload.len() + 35 ); } diff --git a/lib/crowdb-protocol/tests/node_deployment_test.rs b/lib/crowdb-protocol/tests/node_deployment_test.rs new file mode 100644 index 000000000..86194b55f --- /dev/null +++ b/lib/crowdb-protocol/tests/node_deployment_test.rs @@ -0,0 +1,222 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::mgmt::node::{ + deployment::{DeploymentMode, NodeDeploymentOverrides, NodeDeploymentPolicy, PublicAccessUris}, + setup::{SetupServiceKind, SetupStorage}, +}; + +#[test] +fn fresh_local_policy_has_one_voter_and_all_public_access_listeners() { + let policy = NodeDeploymentOverrides::default() + .resolve(&NodeDeploymentOverrides::default(), None) + .unwrap(); + policy.validate().unwrap(); + assert_eq!(policy.initial_voters, 1); + assert_eq!(policy.mode, DeploymentMode::Local); + assert!(matches!( + policy.setup.storage, + SetupStorage::Automatic { sparse: true, .. } + )); + assert!(policy.setup.services.iter().all(|service| service.enabled)); + let access = policy + .setup + .services + .iter() + .find(|service| service.kind == SetupServiceKind::AccessServer) + .unwrap(); + assert_eq!(access.listeners.s3_port, Some(9091)); + assert_eq!(access.listeners.http_port, Some(9092)); + assert_eq!(access.listeners.dataset_port, Some(9093)); +} + +#[test] +fn public_origins_resolve_per_field_and_survive_omitted_restart_inputs() { + let file = NodeDeploymentOverrides { + public_access: PublicAccessUris { + s3: Some("http://127.0.0.1:19091".into()), + iceberg: Some("https://catalog.example.test".into()), + dataset: None, + }, + ..Default::default() + }; + let explicit = NodeDeploymentOverrides { + public_access: PublicAccessUris { + s3: Some("http://[::1]:29091".into()), + dataset: Some("http://127.0.0.1:19093".into()), + iceberg: None, + }, + ..Default::default() + }; + let accepted = explicit.resolve(&file, None).unwrap(); + assert_eq!(accepted.public_access.s3, explicit.public_access.s3); + assert_eq!(accepted.public_access.iceberg, file.public_access.iceberg); + let omitted = NodeDeploymentOverrides::default(); + assert_eq!(omitted.resolve(&omitted, Some(&accepted)).unwrap(), accepted); + assert!(file.resolve(&omitted, Some(&accepted)).is_err()); + let environment = accepted.public_access.environment(); + assert_eq!(environment["CROWDB_DATASET_PUBLIC_URI"], "http://127.0.0.1:19093"); + assert_eq!(environment.len(), 3); + assert_eq!(accepted.setup, NodeDeploymentPolicy::default().setup); +} + +#[test] +fn malformed_or_secret_bearing_public_origins_never_resolve() { + for uri in [ + "", + "http:example.test", + "http://", + "ftp://example.test", + "http://user:password@example.test", + "http://example.test/path", + "http://example.test?token=secret", + "http://example.test#fragment", + "http://example.test\n", + "http://example.test:99999", + "http://example.test\\path", + ] { + let inputs = NodeDeploymentOverrides { + public_access: PublicAccessUris { + s3: Some(uri.into()), + ..Default::default() + }, + ..Default::default() + }; + assert!( + inputs.resolve(&NodeDeploymentOverrides::default(), None).is_err(), + "{uri:?}" + ); + } +} + +#[test] +fn explicit_inputs_override_file_fields_without_erasing_other_file_choices() { + let file = NodeDeploymentOverrides { + cluster: Some("file-scope".into()), + initial_voters: Some(3), + ..Default::default() + }; + let explicit = NodeDeploymentOverrides { + cluster: Some("explicit-scope".into()), + ..Default::default() + }; + let policy = explicit.resolve(&file, None).unwrap(); + assert_eq!(policy.cluster, "explicit-scope"); + assert_eq!(policy.initial_voters, 3); +} + +#[test] +fn omitted_restart_inputs_preserve_accepted_values_instead_of_new_defaults() { + let mut accepted = NodeDeploymentPolicy { + cluster: "retained-cluster".into(), + initial_voters: 3, + ..Default::default() + }; + accepted.setup.services[5].listeners.dataset_port = Some(19093); + let omitted = NodeDeploymentOverrides::default(); + assert_eq!(omitted.resolve(&omitted, Some(&accepted)).unwrap(), accepted); + let changed = NodeDeploymentOverrides { + initial_voters: Some(1), + ..Default::default() + }; + assert!(changed.resolve(&omitted, Some(&accepted)).is_err()); + assert!(omitted.resolve(&changed, Some(&accepted)).is_err()); + let unchanged = NodeDeploymentOverrides { + initial_voters: Some(3), + ..Default::default() + }; + assert_eq!(unchanged.resolve(&changed, Some(&accepted)).unwrap(), accepted); +} + +#[test] +fn production_uses_preallocated_storage_and_never_inherits_local_sparse_files() { + let production = NodeDeploymentOverrides { + mode: Some(DeploymentMode::Production), + ..Default::default() + }; + let policy = production + .resolve(&NodeDeploymentOverrides::default(), None) + .unwrap(); + assert!(matches!( + policy.setup.storage, + SetupStorage::Automatic { sparse: false, .. } + )); + let sparse = NodeDeploymentOverrides { + storage: Some(NodeDeploymentPolicy::default().setup.storage), + ..production + }; + assert!(sparse.resolve(&NodeDeploymentOverrides::default(), None).is_err()); +} + +#[test] +fn invalid_policies_are_rejected_before_any_resolution_can_replace_them() { + let valid = NodeDeploymentPolicy::default(); + for invalid in [ + NodeDeploymentPolicy { + version: 2, + ..valid.clone() + }, + NodeDeploymentPolicy { + initial_voters: 2, + ..valid.clone() + }, + NodeDeploymentPolicy { + cluster: "../foreign".into(), + ..valid.clone() + }, + NodeDeploymentPolicy { + cluster: String::new(), + ..valid.clone() + }, + ] { + assert!(invalid.validate().is_err()); + assert!(NodeDeploymentOverrides::default() + .resolve(&NodeDeploymentOverrides::default(), Some(&invalid)) + .is_err()); + } + let mut duplicate_port = valid; + duplicate_port.setup.services[5].listeners.dataset_port = Some(9092); + assert!(duplicate_port.validate().is_err()); + duplicate_port.setup.services[5].listeners.dataset_port = None; + assert!(duplicate_port.validate().is_err()); + for reserved in [2222, 9090, 9095, 10101] { + let mut policy = NodeDeploymentPolicy::default(); + policy.setup.services[5].listeners.dataset_port = Some(reserved); + assert!(policy.validate().is_err()); + } + for (http, rpc) in [(10001, 10100), (10000, 10102)] { + let mut policy = NodeDeploymentPolicy::default(); + let kv = policy + .setup + .services + .iter_mut() + .find(|service| service.kind == SetupServiceKind::PaxosKv) + .unwrap(); + kv.listeners.http_port = Some(http); + kv.listeners.rpc_port = Some(rpc); + assert!(policy.validate().unwrap_err().contains("bootstrap KV")); + } +} + +#[test] +fn accepted_serialization_requires_explicit_version_and_complete_policy() { + let policy = NodeDeploymentPolicy::default(); + let bytes = serde_json::to_vec(&policy).unwrap(); + assert_eq!( + serde_json::from_slice::(&bytes).unwrap(), + policy + ); + assert!(serde_json::from_str::(r#"{"cluster":"default"}"#).is_err()); + let mut value = serde_json::to_value(policy).unwrap(); + value["password"] = serde_json::json!("unexpected"); + assert!(serde_json::from_value::(value).is_err()); + let current = NodeDeploymentOverrides { + version: Some(1), + ..Default::default() + }; + let future = NodeDeploymentOverrides { + version: Some(99), + ..Default::default() + }; + assert!(current.resolve(&future, None).is_err()); +} diff --git a/lib/crowdb-protocol/tests/stream_chunk_record_test.rs b/lib/crowdb-protocol/tests/stream_chunk_record_test.rs new file mode 100644 index 000000000..983455f76 --- /dev/null +++ b/lib/crowdb-protocol/tests/stream_chunk_record_test.rs @@ -0,0 +1,217 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::chunk_stream::{ + decode_stream_chunk_record, encode_stream_chunk_record, StreamChunkRecord, StreamChunkState, + StreamMappings, StreamName, StreamPurpose, +}; +use crowdb_protocol::chunkdb::rpc::ChunkType; +use crowdb_protocol::common::ChunkId; +use crowdb_protocol::frame::FrameLayout; +use crowdb_protocol::key::{BinaryKey, StreamChunkRecordKey, StreamChunkRecordOrder}; + +fn record(lengths: &[u32]) -> StreamChunkRecord { + StreamChunkRecord { + stream_id: StreamName { high: 7, low: 9 }, + purpose: StreamPurpose::Stream, + chunk_id: ChunkId { + high: (ChunkType::Stream as u64) << 56, + low: 1, + }, + state: StreamChunkState::Sealed, + logical_start: Some(123), + logical_end: Some(123 + lengths.iter().map(|length| u64::from(*length)).sum::()), + sequence: 1, + writer_epoch: 2, + generation: 3, + owner_session: [4; 16], + trim_offset: 0, + closed: false, + mappings: StreamMappings::new(lengths).unwrap(), + } +} + +#[test] +fn keys_sort_by_stream_then_logical_start_with_candidate_reservations_at_tail() { + let id = StreamName { high: 7, low: 9 }; + let orders = [ + StreamChunkRecordOrder::Attached(0), + StreamChunkRecordOrder::Attached(255), + StreamChunkRecordOrder::Attached(256), + StreamChunkRecordOrder::Attached(u64::MAX), + StreamChunkRecordOrder::Candidate(1), + StreamChunkRecordOrder::Candidate(2), + ]; + let mut previous = Vec::new(); + for order in orders { + let key = StreamChunkRecordKey { stream_id: id, order }; + let bytes = key.to_bytes(); + assert_eq!(bytes.len(), 28); + assert!(bytes.starts_with(&StreamChunkRecordKey::stream_prefix(id))); + assert!(bytes > previous); + assert_eq!(StreamChunkRecordKey::from_bytes(&bytes).unwrap(), key); + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(StreamChunkRecordKey::from_bytes(&trailing).is_err()); + previous = bytes; + } +} + +#[test] +fn mapping_codec_preserves_group_boundaries_and_bounded_random_seek() { + for lengths in [ + vec![4096; 4096], + vec![4 * 1024 * 1024; 20], + (0..3073) + .map(|index| 1 + u32::try_from((index * 731) % 16384).unwrap()) + .collect(), + ] { + let record = record(&lengths); + let bytes = encode_stream_chunk_record(&record).unwrap(); + let decoded = decode_stream_chunk_record(&bytes).unwrap(); + assert_eq!(decoded, record); + let index = decoded.validate().unwrap(); + let mut logical = 123; + let mut physical = 0; + for length in lengths { + for offset in [logical, logical + u64::from(length) - 1] { + let mapping = index.locate(&decoded.mappings, offset).unwrap(); + assert_eq!( + (mapping.logical_start, mapping.logical_end, mapping.physical_start), + (logical, logical + u64::from(length), physical) + ); + } + logical += u64::from(length); + physical += FrameLayout::StreamGroupV2 + .physical_length(u64::from(length)) + .unwrap(); + } + assert_eq!(index.physical_end(), physical); + assert!(index.locate(&decoded.mappings, logical).is_err()); + assert!(index.locate(&decoded.mappings, 122).is_err()); + println!( + "groups={} encoded_record_bytes={} seek_samples_bytes={}", + decoded.mappings.count(), + bytes.len(), + index.resident_bytes() + ); + } +} + +#[test] +fn bounds_checksum_and_current_schema_detect_corruption() { + let record = record(&[1, 4096, 65536, 4 * 1024 * 1024]); + let bytes = encode_stream_chunk_record(&record).unwrap(); + for offset in 0..bytes.len() { + let mut damaged = bytes.clone(); + damaged[offset] ^= 1; + assert!(decode_stream_chunk_record(&damaged).is_err(), "offset={offset}"); + } + let mut invalid = record.clone(); + invalid.logical_end = Some(record.logical_end.unwrap() + 1); + assert!(encode_stream_chunk_record(&invalid).is_err()); + invalid = record.clone(); + invalid.logical_start = None; + assert!(encode_stream_chunk_record(&invalid).is_err()); + invalid = record; + invalid.purpose = StreamPurpose::Wal; + assert!(encode_stream_chunk_record(&invalid).is_err()); + assert!(StreamMappings::new(&[0]).is_err()); + assert!(StreamMappings::new(&[u32::MAX]).is_err()); +} + +#[test] +fn candidate_has_no_assigned_logical_interval() { + let mut candidate = record(&[]); + candidate.state = StreamChunkState::Candidate; + candidate.logical_start = None; + candidate.logical_end = None; + candidate.generation = 0; + let bytes = encode_stream_chunk_record(&candidate).unwrap(); + assert_eq!(decode_stream_chunk_record(&bytes).unwrap(), candidate); + candidate.logical_start = Some(0); + assert!(encode_stream_chunk_record(&candidate).is_err()); +} + +fn rewrite_checksum(bytes: &mut [u8]) { + let end = bytes.len() - 4; + let length = u32::try_from(bytes.len() - 16).unwrap(); + bytes[8..12].copy_from_slice(&length.to_le_bytes()); + let checksum = crowdb_common::ec_isal::crc32c_update(0, &bytes[..end]); + bytes[end..].copy_from_slice(&checksum.to_le_bytes()); +} + +#[test] +fn valid_checksum_does_not_authorize_excessive_counts_or_noncanonical_blocks() { + let record = record(&[1, 2]); + let bytes = encode_stream_chunk_record(&record).unwrap(); + let length_offset = bytes.len() - 4 - record.mappings.encoded_len() - 8; + let count_offset = length_offset - 4; + let mut excessive = bytes.clone(); + excessive[count_offset..count_offset + 4].copy_from_slice(&u32::MAX.to_le_bytes()); + rewrite_checksum(&mut excessive); + assert!(decode_stream_chunk_record(&excessive).is_err()); + excessive = bytes.clone(); + excessive[length_offset..length_offset + 8].copy_from_slice(&u64::MAX.to_le_bytes()); + rewrite_checksum(&mut excessive); + assert!(decode_stream_chunk_record(&excessive).is_err()); + let mut split = bytes[..length_offset].to_vec(); + // Two singleton blocks would inflate seek samples instead of using one block. + let blocks = [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0]; + split.extend_from_slice(&u64::try_from(blocks.len()).unwrap().to_le_bytes()); + split.extend_from_slice(&blocks); + split.extend_from_slice(&[0; 4]); + rewrite_checksum(&mut split); + assert!(decode_stream_chunk_record(&split).is_err()); +} + +#[test] +fn stream_identity_record_has_an_id_key_and_preserves_consumer_creation_metadata() { + use crowdb_protocol::{chunk_stream::StreamRecord, key::StreamRecordKey}; + let id = StreamName { + high: u64::MAX, + low: 17, + }; + let mut record = StreamRecord::creating(id, Some("chunk-kv-partition".into()), StreamPurpose::Wal); + record.consumer_id = Some("partition-17".into()); + let key = StreamRecordKey { stream_name: id }; + assert_eq!(key.to_bytes().len(), 19); + assert_eq!(StreamRecordKey::from_bytes(&key.to_bytes()).unwrap(), key); + assert!(record.created_at_ms > 0); + let bytes = serde_json::to_vec(&record).unwrap(); + assert_eq!(serde_json::from_slice::(&bytes).unwrap(), record); + let mut later = record.clone(); + later.created_at_ms += 1; + assert!(record.same_identity(&later)); + later.consumer_id = Some("other-partition".into()); + assert!(!record.same_identity(&later)); + let mut value = serde_json::to_value(&record).unwrap(); + value["old_tail_pointer"] = serde_json::json!(1); + assert!(serde_json::from_value::(value).is_err()); +} + +#[test] +fn full_chunk_mapping_sizes_and_seek_samples_stay_bounded() { + use crowdb_protocol::chunk_stream::MAX_STREAM_CHUNK_BYTES; + for length in [1_u32, 4096] { + let stride = FrameLayout::StreamGroupV2 + .physical_length(u64::from(length)) + .unwrap(); + let count = usize::try_from(MAX_STREAM_CHUNK_BYTES / stride).unwrap(); + let record = record(&vec![length; count]); + let encoded = encode_stream_chunk_record(&record).unwrap(); + let index = record.validate().unwrap(); + assert!(encoded.len() < 200_000); + assert!(index.resident_bytes() < 1024 * 1024); + for group in [0, count / 2, count - 1] { + let offset = 123 + u64::try_from(group).unwrap() * u64::from(length); + let mapping = index.locate(&record.mappings, offset).unwrap(); + assert_eq!(mapping.physical_start, u64::try_from(group).unwrap() * stride); + } + println!( + "append_bytes={length} groups={count} full_chunk_record_bytes={} sample_bytes={}", + encoded.len(), + index.resident_bytes() + ); + } +} diff --git a/lib/crowdb-protocol/tests/stream_process_lease_test.rs b/lib/crowdb-protocol/tests/stream_process_lease_test.rs new file mode 100644 index 000000000..c50a6115a --- /dev/null +++ b/lib/crowdb-protocol/tests/stream_process_lease_test.rs @@ -0,0 +1,104 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +use crowdb_protocol::chunk_stream::{ + StreamName, StreamProcessIdentity, StreamProcessLease, StreamPurpose, StreamWriterOwner, +}; +use crowdb_protocol::key::{StreamProcessLeaseKey, StreamWriterOwnerKey, TextKey}; + +#[test] +fn owner_keys_and_records_keep_purpose_instance_and_process_identity() { + for purpose in [StreamPurpose::Wal, StreamPurpose::Stream] { + let key = StreamProcessLeaseKey { + purpose, + instance_id: 7, + }; + assert_eq!(StreamProcessLeaseKey::from_path(&key.to_path()).unwrap(), key); + let record = StreamProcessLease { + identity: StreamProcessIdentity { + purpose, + instance_id: 7, + process_epoch: 1, + boot_nonce: [1; 16], + }, + expires_at_ms: 2_000, + prior_exclusion_deadline_ms: 0, + grant_sequence: 0, + grant_expires_at_ms: 0, + }; + record.validate().unwrap(); + let value = serde_json::to_value(&record).unwrap(); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + record + ); + let mut successor = record.clone(); + successor.identity.process_epoch = 2; + assert!(successor.validate().is_err()); + successor.prior_exclusion_deadline_ms = record.expires_at_ms; + successor.validate().unwrap(); + let mut unknown = value; + unknown + .as_object_mut() + .unwrap() + .insert("unknown".into(), 1.into()); + assert!(serde_json::from_value::(unknown).is_err()); + } +} + +#[test] +fn business_stream_processes_cannot_claim_wal_grant_fencing_fields() { + let mut record = StreamProcessLease { + identity: StreamProcessIdentity { + purpose: StreamPurpose::Stream, + instance_id: 7, + process_epoch: 1, + boot_nonce: [1; 16], + }, + expires_at_ms: 2_000, + prior_exclusion_deadline_ms: 0, + grant_sequence: 1, + grant_expires_at_ms: 3_000, + }; + assert!(record.validate().is_err()); + record.identity.purpose = StreamPurpose::Wal; + record.validate().unwrap(); + record.grant_expires_at_ms = 0; + assert!(record.validate().is_err()); +} + +#[test] +fn writer_assignment_requires_a_complete_unique_identity() { + let record = StreamWriterOwner { + stream_name: StreamName { high: 12, low: 34 }, + writer_epoch: 3, + process: StreamProcessIdentity { + purpose: StreamPurpose::Stream, + instance_id: 7, + process_epoch: 1, + boot_nonce: [1; 16], + }, + assignment_nonce: [2; 16], + released: false, + }; + record.validate().unwrap(); + let key = StreamWriterOwnerKey { + stream_name: record.stream_name, + }; + assert_eq!(StreamWriterOwnerKey::from_path(&key.to_path()).unwrap(), key); + assert_eq!( + serde_json::from_slice::(&serde_json::to_vec(&record).unwrap()).unwrap(), + record + ); + for field in 0..5 { + let mut invalid = record.clone(); + match field { + 0 => invalid.writer_epoch = 0, + 1 => invalid.process.instance_id = 0, + 2 => invalid.process.process_epoch = 0, + 3 => invalid.process.boot_nonce = [0; 16], + _ => invalid.assignment_nonce = [0; 16], + } + assert!(invalid.validate().is_err()); + } +} diff --git a/lib/crowdb-rpc/ffi/src/client.rs b/lib/crowdb-rpc/ffi/src/client.rs index 4d70fecd3..ca464b570 100644 --- a/lib/crowdb-rpc/ffi/src/client.rs +++ b/lib/crowdb-rpc/ffi/src/client.rs @@ -29,9 +29,6 @@ pub struct Response { /// a future that resolves when the response arrives (or on error). pub struct RpcClient { handle: sys::crowdb_rpc_client_t, - // Raw pointers from Box::into_raw in register_handler. Freed in - // Drop to avoid leaking the Rust handler closures. - handler_ptrs: std::sync::Mutex>, } impl std::fmt::Debug for RpcClient { @@ -46,10 +43,7 @@ impl RpcClient { /// Create a new RpcClient. pub fn new() -> Self { let handle = unsafe { sys::crowdb_rpc_client_create() }; - RpcClient { - handle, - handler_ptrs: std::sync::Mutex::new(Vec::new()), - } + RpcClient { handle } } /// Attach the client to a connection so responses are routed to the @@ -94,23 +88,11 @@ impl RpcClient { unsafe { sys::crowdb_rpc_client_stop_reaper(self.handle) }; } - /// Clear all registered handlers and free the Rust closures. + /// Remove registered handlers; active invocations retain their Rust closures. /// Called automatically by `stop_reaper`. Safe to call multiple times. pub fn clear_handlers(&self) { - // Clear the C++ handler registry first so no dispatch can - // access the closures after we free them. + // Native dispatch retains each closure until its final invocation returns. unsafe { sys::crowdb_rpc_client_clear_handlers(self.handle) }; - // Free the Rust handler closures. - let ptrs = std::mem::take(&mut *self.handler_ptrs.lock().unwrap()); - for user_data in ptrs { - if !user_data.is_null() { - unsafe { - drop(Box::from_raw( - user_data.cast::>(), - )); - } - } - } } /// Dump all pending request IDs + deadlines to the C++ log. For @@ -417,9 +399,9 @@ impl RpcClient { msg_type, Some(rust_client_handler_trampoline), user_data, + Some(release_handler), ); } - self.handler_ptrs.lock().unwrap().push(user_data); } } @@ -432,18 +414,6 @@ impl Default for RpcClient { impl Drop for RpcClient { fn drop(&mut self) { if !self.handle.is_null() { - // Free any handler closures not already freed by - // clear_handlers/stop_reaper (e.g. if stop was never called). - let ptrs = std::mem::take(&mut *self.handler_ptrs.lock().unwrap()); - for user_data in ptrs { - if !user_data.is_null() { - unsafe { - drop(Box::from_raw( - user_data.cast::>(), - )); - } - } - } unsafe { sys::crowdb_rpc_client_destroy(self.handle) }; self.handle = ptr::null_mut(); } @@ -476,7 +446,10 @@ impl std::future::Future for CallFuture { ) -> std::task::Poll { std::pin::Pin::new(&mut self.rx) .poll(cx) - .map_ok(|(result, _io_thread_ts)| result) + .map_ok(|(result, io_thread_ts)| { + response_schedule_histogram().observe(io_thread_ts.elapsed().as_nanos() as u64); + result + }) .map(|r| r.unwrap_or(Err(RpcError::ConnectionClosed))) } } @@ -485,12 +458,14 @@ impl std::future::Future for CallFuture { // requires a non-null `on_complete` even when no reply is expected. unsafe extern "C" fn noop_on_complete( _request_id: u64, - _control: sys::crowdb_rpc_buffer_t, - _data: sys::crowdb_rpc_buffer_t, + control: sys::crowdb_rpc_buffer_t, + data: sys::crowdb_rpc_buffer_t, _status: i32, _user_data: *mut std::ffi::c_void, ) { - // Discard — fire-and-forget. + // Completion owns both handles even when the application discards its reply. + drop(Buffer::from_raw(control)); + drop(Buffer::from_raw(data)); } /// Get the no-op completion callback for fire-and-forget `send()` calls. @@ -563,6 +538,12 @@ fn e2e_histogram() -> std::sync::Arc { .clone() } +fn response_schedule_histogram() -> &'static metrics::LatencyHistogram { + use std::sync::{Arc, OnceLock}; + static HISTOGRAM: OnceLock> = OnceLock::new(); + HISTOGRAM.get_or_init(|| metrics::global_histogram("rpc.response.schedule.lh")) +} + // ── Client-side request handler dispatch (R114) ──────────────────── /// An incoming client-side request (server→client direction) that owns @@ -656,3 +637,11 @@ unsafe extern "C" fn rust_client_handler_trampoline( }; closure(req); } + +unsafe extern "C" fn release_handler(user_data: *mut std::ffi::c_void) { + if !user_data.is_null() { + drop(Box::from_raw( + user_data.cast::>(), + )); + } +} diff --git a/lib/crowdb-rpc/ffi/src/server.rs b/lib/crowdb-rpc/ffi/src/server.rs index 32d014773..fdfe10972 100644 --- a/lib/crowdb-rpc/ffi/src/server.rs +++ b/lib/crowdb-rpc/ffi/src/server.rs @@ -11,9 +11,6 @@ use std::sync::Arc; /// An RPC server. Accepts connections and dispatches to handlers. pub struct RpcServer { handle: sys::crowdb_rpc_server_t, - // Raw pointers from Box::into_raw in register_handler. Freed in - // Drop to avoid leaking the Rust handler closures. - handler_ptrs: std::sync::Mutex>, } impl std::fmt::Debug for RpcServer { @@ -76,10 +73,7 @@ impl RpcServer { let pool_handle = pool.map(|p| p.handle()).unwrap_or(ptr::null_mut()); let handle = unsafe { sys::crowdb_rpc_server_create_with_engines(pool_handle, io_engines, effective_workers) }; - RpcServer { - handle, - handler_ptrs: std::sync::Mutex::new(Vec::new()), - } + RpcServer { handle } } /// Listen on the given address and port. If port is 0, the OS assigns @@ -109,23 +103,11 @@ impl RpcServer { unsafe { sys::crowdb_rpc_server_stop(self.handle) }; } - /// Clear all registered handlers and free the Rust closures. + /// Remove registered handlers; active invocations retain their Rust closures. /// Called automatically by `stop`. Safe to call multiple times. pub fn clear_handlers(&self) { - // Clear the C++ handler registry first so no dispatch can - // access the closures after we free them. + // Native dispatch retains each closure until its final invocation returns. unsafe { sys::crowdb_rpc_server_clear_handlers(self.handle) }; - // Free the Rust handler closures (drops captured Arcs). - let ptrs = std::mem::take(&mut *self.handler_ptrs.lock().unwrap()); - for user_data in ptrs { - if !user_data.is_null() { - unsafe { - drop(Box::from_raw( - user_data.cast::>(), - )); - } - } - } } /// The port the server is listening on (0 if not listening). @@ -208,10 +190,8 @@ impl RpcServer { /// for zero-copy flatbuffer parsing — the Frame is released when the /// `ServerRequest` is dropped. This mirrors the C++ async-handler /// pattern (return nullptr, submit later). Re-registering the same - /// msg_type replaces the prior handler (the old closure is leaked — - /// there is no unregister API; handlers live for the server's - /// lifetime, which for a server registered once at startup is the - /// process lifetime). + /// msg_type replaces the prior handler. The previous closure remains alive + /// until its last active invocation returns. pub fn register_handler(&self, msg_type: u16, handler: F) where F: Fn(ServerRequest) + Send + 'static, @@ -225,9 +205,9 @@ impl RpcServer { msg_type, Some(rust_handler_trampoline), user_data, + Some(release_handler), ); } - self.handler_ptrs.lock().unwrap().push(user_data); } /// Submit a response on a server-side connection. Thread-safe — may @@ -320,18 +300,6 @@ impl RpcServer { impl Drop for RpcServer { fn drop(&mut self) { if !self.handle.is_null() { - // Free any handler closures not already freed by - // clear_handlers/stop (e.g. if stop was never called). - let ptrs = std::mem::take(&mut *self.handler_ptrs.lock().unwrap()); - for user_data in ptrs { - if !user_data.is_null() { - unsafe { - drop(Box::from_raw( - user_data.cast::>(), - )); - } - } - } unsafe { sys::crowdb_rpc_server_destroy(self.handle) }; self.handle = ptr::null_mut(); } @@ -608,3 +576,11 @@ unsafe extern "C" fn rust_handler_trampoline( closure(req); // req is dropped here if the closure didn't move it — frame released. } + +unsafe extern "C" fn release_handler(user_data: *mut std::ffi::c_void) { + if !user_data.is_null() { + drop(Box::from_raw( + user_data.cast::>(), + )); + } +} diff --git a/lib/crowdb-rpc/ffi/src/sys.rs b/lib/crowdb-rpc/ffi/src/sys.rs index f5e1deeca..1f5d6e8dc 100644 --- a/lib/crowdb-rpc/ffi/src/sys.rs +++ b/lib/crowdb-rpc/ffi/src/sys.rs @@ -96,6 +96,8 @@ pub type crowdb_rpc_handler_fn = Option< ), >; +pub type crowdb_rpc_handler_release_fn = Option; + // ── Coroutine client (Option 3: C++ coroutine + Rust FFI) ──────── pub type crowdb_rpc_co_build_request = Option< unsafe extern "C" fn( @@ -259,6 +261,7 @@ extern "C" { msg_type: u16, callback: crowdb_rpc_handler_fn, user_data: *mut c_void, + release: crowdb_rpc_handler_release_fn, ); pub fn crowdb_rpc_server_submit_response( @@ -287,6 +290,7 @@ extern "C" { msg_type: u16, callback: crowdb_rpc_handler_fn, user_data: *mut c_void, + release: crowdb_rpc_handler_release_fn, ); pub fn crowdb_rpc_client_clear_handlers(client: crowdb_rpc_client_t); diff --git a/lib/crowdb-rpc/ffi/tests/buffer_test.rs b/lib/crowdb-rpc/ffi/tests/buffer_test.rs index 859b43869..2f6cb8b2f 100644 --- a/lib/crowdb-rpc/ffi/tests/buffer_test.rs +++ b/lib/crowdb-rpc/ffi/tests/buffer_test.rs @@ -72,3 +72,35 @@ fn buffer_chain_retains_views_and_enforces_native_limit() { } )); } + +#[test] +fn discarded_completion_releases_both_response_owners() { + use crowdb_rpc_ffi::{noop_completion, sys}; + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }; + + struct TestOwner(Arc); + impl AsRef<[u8]> for TestOwner { + fn as_ref(&self) -> &[u8] { + b"response" + } + } + impl Drop for TestOwner { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::Release); + } + } + for status in [sys::CROWDB_RPC_OK, sys::CROWDB_RPC_ERR_CONN_CLOSED] { + let released = Arc::new(AtomicUsize::new(0)); + let make_buffer = || Buffer::from_owned_bytes(Bytes::from_owner(TestOwner(Arc::clone(&released)))); + let control = make_buffer().into_raw(); + let data = make_buffer().into_raw(); + // Both live handles transfer to the completion once; it does not use context. + unsafe { + noop_completion().unwrap()(1, control, data, status, std::ptr::null_mut()); + } + assert_eq!(released.load(Ordering::Acquire), 2); + } +} diff --git a/lib/crowdb-rpc/ffi/tests/ffi_handler_test.rs b/lib/crowdb-rpc/ffi/tests/ffi_handler_test.rs index 357f98395..0c509de1f 100644 --- a/lib/crowdb-rpc/ffi/tests/ffi_handler_test.rs +++ b/lib/crowdb-rpc/ffi/tests/ffi_handler_test.rs @@ -175,3 +175,99 @@ async fn custom_rust_handler_with_data() { server.stop(); } + +struct TestHandlerOwner(std::sync::Arc); + +impl Drop for TestHandlerOwner { + fn drop(&mut self) { + self.0.fetch_add(1, std::sync::atomic::Ordering::Release); + } +} + +fn check_active_handler_lifetime(client_side: bool) { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, Barrier, + }; + use std::time::Duration; + + let server = Arc::new(RpcServer::new(None)); + server.listen("127.0.0.1", 0).unwrap(); + let client = RpcClient::new(); + let released = Arc::new(AtomicUsize::new(0)); + let owner = TestHandlerOwner(Arc::clone(&released)); + let gate = Arc::new(Barrier::new(2)); + let (entered_tx, entered_rx) = mpsc::channel(); + let invoke = { + let gate = Arc::clone(&gate); + move || { + std::hint::black_box(&owner); + entered_tx.send(()).unwrap(); + gate.wait(); + } + }; + if client_side { + client.register_handler(31002, move |_request| invoke()); + server.register_handler(31001, { + let server = Arc::clone(&server); + let push = RpcClient::new(); + move |request| { + push.send_one_way_to_handle( + &server, + request.conn_handle, + 2, + handler_test_control(2), + None, + 31002, + ) + .unwrap(); + } + }); + } else { + server.register_handler(31001, move |_request| invoke()); + } + server.start(); + let connection = server.connect("127.0.0.1", server.port()).unwrap(); + client.attach(&connection); + let call = client + .call(&server, &connection, 1, handler_test_control(1), None, 31001) + .unwrap(); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + if client_side { + client.clear_handlers(); + } else { + server.clear_handlers(); + } + let released_while_running = released.load(Ordering::Acquire); + gate.wait(); + server.stop(); + drop(call); + assert_eq!( + released_while_running, 0, + "active dispatch must retain its Rust closure" + ); + assert_eq!(released.load(Ordering::Acquire), 1); +} + +#[test] +fn clearing_server_handlers_retains_an_active_callback() { + check_active_handler_lifetime(false); +} + +#[test] +fn clearing_client_handlers_retains_an_active_callback() { + check_active_handler_lifetime(true); +} + +fn handler_test_control(id: u64) -> crowdb_rpc_ffi::Buffer { + let mut builder = FlatBufferBuilder::new(); + let value = ConnectionPingRequest::create( + &mut builder, + &ConnectionPingRequestArgs { + id, + rpc_create_nano: 0, + }, + ); + builder.finish(value, None); + crowdb_rpc_ffi::Buffer::from_bytes(builder.finished_data()) +} diff --git a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h index 6ee44d741..6dc5269b3 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/c_api.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/c_api.h @@ -264,6 +264,10 @@ typedef void (*crowdb_rpc_handler_fn)(uint64_t request_id, uint64_t rpc_create_n const uint8_t *control, uint32_t control_len, const uint8_t *data, uint32_t data_len, void *conn_handle, void *frame_handle, void *user_data); +// Optional owner release callback. Registration takes ownership immediately; +// replacement, clear and destruction release it after the last active dispatch. +typedef void (*crowdb_rpc_handler_release_fn)(void *user_data); + // Release a frame_handle previously passed to a crowdb_rpc_handler_fn. // Must be called exactly once per frame_handle. Passing nullptr is a no-op. void crowdb_rpc_frame_release(void *frame_handle); @@ -274,7 +278,7 @@ void crowdb_rpc_frame_release(void *frame_handle); // a C++ HandlerFn. Re-registering the same msg_type replaces the prior // handler. void crowdb_rpc_server_register_handler(crowdb_rpc_server_t server, uint16_t msg_type, crowdb_rpc_handler_fn callback, - void *user_data); + void *user_data, crowdb_rpc_handler_release_fn release); // Submit a response on a server-side connection. Allocates buffers from // the pool, builds an OutFrame, and calls transport->submit (enqueue + @@ -303,7 +307,7 @@ crowdb_rpc_status crowdb_rpc_server_submit_response_buffer(crowdb_rpc_server_t s // response via crowdb_rpc_server_submit_response (the Rust closure // captures the server handle). The callback must be non-blocking. void crowdb_rpc_client_register_handler(crowdb_rpc_client_t client, uint16_t msg_type, crowdb_rpc_handler_fn callback, - void *user_data); + void *user_data, crowdb_rpc_handler_release_fn release); // Clear all registered client-side handlers. Called during shutdown // to break reference cycles (Rust handler closures capture Arcs). void crowdb_rpc_client_clear_handlers(crowdb_rpc_client_t client); diff --git a/lib/crowdb-rpc/include/crowdb-rpc/client/client.h b/lib/crowdb-rpc/include/crowdb-rpc/client/client.h index e4888a5cb..6fbff0c41 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/client/client.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/client/client.h @@ -146,15 +146,11 @@ class RpcClient // is a C callback (same type as server-side handlers) that receives // the request fields + conn_handle and submits the response later // via crowdb_rpc_server_submit_response. - void register_handler(uint16_t msg_type, crowdb_rpc_handler_fn callback, void *user_data); + void register_handler(uint16_t msg_type, crowdb_rpc_handler_fn callback, std::shared_ptr context); // Clear all registered request handlers. Called during shutdown // to break reference cycles (Rust handler closures capture Arcs). - void clear_handlers() - { - std::scoped_lock lock(handler_mu_); - request_handlers_.clear(); - } + void clear_handlers(); // Set the transport for submitting UnknownMessage responses when // no handler matches an incoming request msg_type. If not set, @@ -202,8 +198,8 @@ class RpcClient // Handler registry for incoming requests (server→client direction). // Maps msg_type → (C callback, user_data). Same trampoline pattern // as the server-side handler dispatch. - std::mutex handler_mu_; - std::unordered_map> request_handlers_; + std::mutex handler_mu_; + std::unordered_map>> request_handlers_; // Transport for submitting UnknownMessage responses when no handler // matches an incoming request msg_type. Set via set_transport. diff --git a/lib/crowdb-rpc/include/crowdb-rpc/connection.h b/lib/crowdb-rpc/include/crowdb-rpc/connection.h index 9d7e37daa..443f32bfa 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/connection.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/connection.h @@ -63,10 +63,38 @@ class Connection : public std::enable_shared_from_this return open_.load(std::memory_order_relaxed); } + class SocketUse + { + public: + explicit SocketUse(Connection &connection); + ~SocketUse(); + SocketUse(const SocketUse &) = delete; + SocketUse &operator=(const SocketUse &) = delete; + + explicit operator bool() const + { + return connection_ != nullptr; + } + + private: + Connection *connection_ = nullptr; + }; + + // Closed descriptors cannot be reused until active syscalls and re-arms retire. + [[nodiscard]] SocketUse use_socket() + { + return SocketUse{*this}; + } + + [[nodiscard]] bool socket_cleanup_ready() const + { + return socket_users_.load(std::memory_order_acquire) == SOCKET_CLOSED; + } + // Exactly one worker owns descriptor teardown, including duplicate events. bool claim_socket_cleanup() { - return !socket_cleanup_claimed_.exchange(true, std::memory_order_acq_rel); + return socket_cleanup_ready() && !socket_cleanup_claimed_.exchange(true, std::memory_order_acq_rel); } // TCP ECONNREFUSED proves the peer never accepted this connection. @@ -198,9 +226,11 @@ class Connection : public std::enable_shared_from_this BufferPool *pool_; FrameParser parser_; - std::atomic open_{true}; - std::atomic connect_refused_{false}; - std::atomic socket_cleanup_claimed_{false}; + static constexpr uint32_t SOCKET_CLOSED = uint32_t{1} << 31; + std::atomic socket_users_{0}; + std::atomic open_{true}; + std::atomic connect_refused_{false}; + std::atomic socket_cleanup_claimed_{false}; // Lock-free bounded MPSC ring buffer (crowdb::common::MpscQueue). Multiple // producer threads push via enqueue_send (Transport::submit); the I/O diff --git a/lib/crowdb-rpc/include/crowdb-rpc/server/handler.h b/lib/crowdb-rpc/include/crowdb-rpc/server/handler.h index 4678b0d6f..32b16332c 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/server/handler.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/server/handler.h @@ -42,6 +42,32 @@ class HandlerRegistry public: using HandlerTable = std::unordered_map; + class HandlerLease + { + public: + explicit operator bool() const + { + return table_ && handler_ != nullptr; + } + + OutFrame *operator()(Frame *request, Connection *connection) const + { + return (*handler_)(request, connection); + } + + private: + friend class HandlerRegistry; + + HandlerLease(std::shared_ptr table, const HandlerFn *handler) + : table_(std::move(table)), + handler_(handler) + { + } + + std::shared_ptr table_; + const HandlerFn *handler_; + }; + HandlerRegistry() : handlers_(std::make_shared()) { } @@ -60,14 +86,12 @@ class HandlerRegistry } } - HandlerFn get_handler(uint16_t msg_type) const + HandlerLease get_handler(uint16_t msg_type) const { - auto table = handlers_.load(std::memory_order_acquire); - auto it = table->find(msg_type); - if (it != table->end()) { - return it->second; - } - return nullptr; + auto table = handlers_.load(std::memory_order_acquire); + auto it = table->find(msg_type); + const HandlerFn *handler = it != table->end() ? &it->second : nullptr; + return {std::move(table), handler}; } void clear() diff --git a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h index 1d2cd117d..d27463f16 100644 --- a/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h +++ b/lib/crowdb-rpc/include/crowdb-rpc/transport/socket_transport.h @@ -221,6 +221,9 @@ class Worker // read_fd and write_fd are dup'd fds for independent epoll arming. void add_connection(int read_fd, int write_fd, std::shared_ptr conn); + // Schedule descriptor cleanup on the owning worker, including idle sockets. + void request_close(std::shared_ptr conn); + // Check if the current thread is this worker's thread. Used by // submit() to detect inline calls (coroutine resumed on I/O worker // thread) and use submit_inline instead of notify. @@ -263,11 +266,23 @@ class Worker // writev all pending responses in one syscall. std::vector pending_write_conns_; + struct CloseRequest + { + std::shared_ptr connection; + CloseRequest *next; + }; + + // Producers retain identity until the owner retires its map entry after I/O. + std::atomic pending_closes_{nullptr}; + friend class SocketTransport; using ClosedConnections = std::vector>; void run_loop(); void drain_notified(); + void drain_closes(ClosedConnections &closed_fds); + void discard_closes(); + void enqueue_close(CloseRequest *request); void process_readable(const EngineEvent &ev, ClosedConnections &closed_fds); void process_writable(const EngineEvent &ev, ClosedConnections &closed_fds); void process_error(const EngineEvent &ev, ClosedConnections &closed_fds); diff --git a/lib/crowdb-rpc/src/c_api.cpp b/lib/crowdb-rpc/src/c_api.cpp index a797c720f..50909b1d3 100644 --- a/lib/crowdb-rpc/src/c_api.cpp +++ b/lib/crowdb-rpc/src/c_api.cpp @@ -873,222 +873,6 @@ void crowdb_rpc_server_register_echo_handler(crowdb_rpc_server_t server, uint16_ } } -// ── Custom handler dispatch (R115: Rust server handlers) ────────── - -// Shared handler trampoline: extracts request fields from the frame, -// invokes the C dispatch callback, and transfers frame ownership to the -// callback. Used by both server-side (c_handler_trampoline) and client-side -// (RpcClient::dispatch_request) handler dispatch. The callback submits the -// response later via crowdb_rpc_server_submit_response (async pattern). -// -// The callback OWNS the frame via frame_handle — it must call -// crowdb_rpc_frame_release(frame_handle) when done. The dispatch layer does -// NOT delete the frame. -void crowdb::rpc::invoke_c_handler(crowdb_rpc_handler_fn callback, void *user_data, Frame *request, Connection *conn) -{ - if (callback == nullptr || request == nullptr) { - delete request; - return; - } - uint64_t req_id = request->request_id; - uint64_t create_nano = request->rpc_create_nano; - uint16_t msg_type = request->header.msg_type; - const uint8_t *ctrl_ptr = request->control.empty() ? nullptr : request->control.data(); - auto ctrl_len = static_cast(request->control.size()); - const uint8_t *data_ptr = nullptr; - uint32_t data_len = 0; - if (request->data_buf != nullptr && request->data_buf->len > 0) { - data_ptr = request->data_buf->data; - data_len = request->data_buf->len; - } - void *conn_handle = static_cast(conn); - void *frame_handle = static_cast(request); - - // Transfer frame ownership to the callback. The callback must call - // crowdb_rpc_frame_release(frame_handle) when done. - callback(req_id, create_nano, msg_type, ctrl_ptr, ctrl_len, data_ptr, data_len, conn_handle, frame_handle, - user_data); -} - -// Release a frame_handle passed to a crowdb_rpc_handler_fn callback. -void crowdb_rpc_frame_release(void *frame_handle) -{ - if (frame_handle != nullptr) { - delete static_cast(frame_handle); - } -} - -void crowdb_rpc_server_register_handler(crowdb_rpc_server_t server, uint16_t msg_type, crowdb_rpc_handler_fn callback, - void *user_data) -{ - try { - if (server == nullptr || callback == nullptr) { - return; - } - server->server->register_handler(msg_type, - [callback, user_data](crowdb::rpc::Frame *req, crowdb::rpc::Connection *conn) { - crowdb::rpc::invoke_c_handler(callback, user_data, req, conn); - return nullptr; // async — callback submits response later. - }); - } - catch (...) { - } -} - -crowdb_rpc_status crowdb_rpc_server_submit_response(crowdb_rpc_server_t server, void *conn_handle, - const uint8_t *control, uint32_t control_len, const uint8_t *data, - uint32_t data_len, uint16_t msg_type, uint64_t request_id) -{ - try { - if (server == nullptr || conn_handle == nullptr) { - return CROWDB_RPC_ERR_INVALID_ARG; - } - - auto *conn = static_cast(conn_handle); - auto *pool = server->server->pool(); - - // Allocate response buffers from the pool and copy data. - crowdb::rpc::Buffer *resp_ctrl = nullptr; - if (control != nullptr && control_len > 0) { - resp_ctrl = pool->alloc(control_len); - if (resp_ctrl == nullptr) { - return CROWDB_RPC_ERR_SEND_QUEUE; - } - resp_ctrl->write(control, control_len); - } - - crowdb::rpc::Buffer *resp_data = nullptr; - if (data != nullptr && data_len > 0) { - resp_data = pool->alloc(data_len); - if (resp_data == nullptr) { - if (resp_ctrl != nullptr) { - resp_ctrl->release(); - } - return CROWDB_RPC_ERR_SEND_QUEUE; - } - resp_data->write(data, data_len); - } - - auto *frame = crowdb::rpc::build_out_frame(request_id, msg_type, resp_ctrl, resp_data); - if (!server->server->transport()->submit(conn, frame)) { - if (frame->control != nullptr) { - frame->control->release(); - } - frame->release_data(); - delete frame; - return CROWDB_RPC_ERR_SEND_QUEUE; - } - return CROWDB_RPC_OK; - } - catch (...) { - return CROWDB_RPC_ERR_CONN_ERROR; - } -} - -// Submit a response using pre-filled buffer handles (zero-copy). The -// server takes ownership of the buffers. -crowdb_rpc_status crowdb_rpc_server_submit_response_buffer(crowdb_rpc_server_t server, void *conn_handle, - crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, - uint16_t msg_type, uint64_t request_id) -{ - try { - if (server == nullptr || conn_handle == nullptr) { - // Release any provided buffers to avoid leaks. - if (control != nullptr) { - crowdb_rpc_buffer_release(control); - } - if (data != nullptr) { - crowdb_rpc_buffer_release(data); - } - return CROWDB_RPC_ERR_INVALID_ARG; - } - - auto *conn = static_cast(conn_handle); - - // Extract Buffer* from handles (steal ownership — the handle's - // release is a no-op after this since we set buf = nullptr). - crowdb::rpc::Buffer *resp_ctrl = nullptr; - if (control != nullptr && control->buf != nullptr) { - resp_ctrl = control->buf; - control->buf = nullptr; - delete control; - } - crowdb::rpc::Buffer *resp_data = nullptr; - if (data != nullptr && data->buf != nullptr) { - resp_data = data->buf; - data->buf = nullptr; - delete data; - } - - auto *frame = crowdb::rpc::build_out_frame(request_id, msg_type, resp_ctrl, resp_data); - if (!server->server->transport()->submit(conn, frame)) { - if (frame->control != nullptr) { - frame->control->release(); - } - frame->release_data(); - delete frame; - return CROWDB_RPC_ERR_SEND_QUEUE; - } - return CROWDB_RPC_OK; - } - catch (...) { - return CROWDB_RPC_ERR_CONN_ERROR; - } -} - -// ── Client-side request handler dispatch (R114) ────────────────── - -void crowdb_rpc_client_register_handler(crowdb_rpc_client_t client, uint16_t msg_type, crowdb_rpc_handler_fn callback, - void *user_data) -{ - try { - if (client == nullptr || callback == nullptr) { - return; - } - client->client->register_handler(msg_type, callback, user_data); - } - catch (...) { - } -} - -void crowdb_rpc_client_clear_handlers(crowdb_rpc_client_t client) -{ - try { - if (client == nullptr) { - return; - } - client->client->clear_handlers(); - } - catch (...) { - } -} - -void crowdb_rpc_client_set_transport(crowdb_rpc_client_t client, crowdb_rpc_server_t server) -{ - try { - if (client == nullptr || server == nullptr) { - return; - } - client->client->set_transport(server->server->transport()); - } - catch (...) { - } -} - -// ── Server-side request-response correlation (R114) ────────────── - -void crowdb_rpc_server_set_request_client(crowdb_rpc_server_t server, crowdb_rpc_client_t client) -{ - try { - if (server == nullptr || client == nullptr) { - return; - } - server->server->set_request_client(client->client.get()); - } - catch (...) { - } -} - // ── Logging ─────────────────────────────────────────────────────── void crowdb_rpc_init_logging(const char *log_dir, const char *level, size_t max_file_mb, size_t max_files, diff --git a/lib/crowdb-rpc/src/c_api/handlers.cpp b/lib/crowdb-rpc/src/c_api/handlers.cpp new file mode 100644 index 000000000..b2ad34e79 --- /dev/null +++ b/lib/crowdb-rpc/src/c_api/handlers.cpp @@ -0,0 +1,234 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-rpc/c_api_internal.h" +#include "crowdb-rpc/server/message.h" + +#include + +static std::shared_ptr own_handler_context(void *user_data, crowdb_rpc_handler_release_fn release) +{ + return {user_data, [release](void *context) { + if (release != nullptr) { + release(context); + } + }}; +} + +// ── Custom handler dispatch ────────── + +// Shared handler trampoline: extracts request fields from the frame, +// invokes the C dispatch callback, and transfers frame ownership to the +// callback. Used by both server-side (c_handler_trampoline) and client-side +// (RpcClient::dispatch_request) handler dispatch. The callback submits the +// response later via crowdb_rpc_server_submit_response (async pattern). +// +// The callback OWNS the frame via frame_handle — it must call +// crowdb_rpc_frame_release(frame_handle) when done. The dispatch layer does +// NOT delete the frame. +void crowdb::rpc::invoke_c_handler(crowdb_rpc_handler_fn callback, void *user_data, Frame *request, Connection *conn) +{ + if (callback == nullptr || request == nullptr) { + delete request; + return; + } + uint64_t req_id = request->request_id; + uint64_t create_nano = request->rpc_create_nano; + uint16_t msg_type = request->header.msg_type; + const uint8_t *ctrl_ptr = request->control.empty() ? nullptr : request->control.data(); + auto ctrl_len = static_cast(request->control.size()); + const uint8_t *data_ptr = nullptr; + uint32_t data_len = 0; + if (request->data_buf != nullptr && request->data_buf->len > 0) { + data_ptr = request->data_buf->data; + data_len = request->data_buf->len; + } + void *conn_handle = static_cast(conn); + void *frame_handle = static_cast(request); + + // Transfer frame ownership to the callback. The callback must call + // crowdb_rpc_frame_release(frame_handle) when done. + callback(req_id, create_nano, msg_type, ctrl_ptr, ctrl_len, data_ptr, data_len, conn_handle, frame_handle, + user_data); +} + +// Release a frame_handle passed to a crowdb_rpc_handler_fn callback. +void crowdb_rpc_frame_release(void *frame_handle) +{ + if (frame_handle != nullptr) { + delete static_cast(frame_handle); + } +} + +void crowdb_rpc_server_register_handler(crowdb_rpc_server_t server, uint16_t msg_type, crowdb_rpc_handler_fn callback, + void *user_data, crowdb_rpc_handler_release_fn release) +{ + try { + auto owner = own_handler_context(user_data, release); + if (server == nullptr || callback == nullptr) { + return; + } + server->server->register_handler( + msg_type, [callback, owner = std::move(owner)](crowdb::rpc::Frame *req, crowdb::rpc::Connection *conn) { + crowdb::rpc::invoke_c_handler(callback, owner.get(), req, conn); + return nullptr; // async — callback submits response later. + }); + } + catch (...) { + } +} + +crowdb_rpc_status crowdb_rpc_server_submit_response(crowdb_rpc_server_t server, void *conn_handle, + const uint8_t *control, uint32_t control_len, const uint8_t *data, + uint32_t data_len, uint16_t msg_type, uint64_t request_id) +{ + try { + if (server == nullptr || conn_handle == nullptr) { + return CROWDB_RPC_ERR_INVALID_ARG; + } + + auto *conn = static_cast(conn_handle); + auto *pool = server->server->pool(); + + // Allocate response buffers from the pool and copy data. + crowdb::rpc::Buffer *resp_ctrl = nullptr; + if (control != nullptr && control_len > 0) { + resp_ctrl = pool->alloc(control_len); + if (resp_ctrl == nullptr) { + return CROWDB_RPC_ERR_SEND_QUEUE; + } + resp_ctrl->write(control, control_len); + } + + crowdb::rpc::Buffer *resp_data = nullptr; + if (data != nullptr && data_len > 0) { + resp_data = pool->alloc(data_len); + if (resp_data == nullptr) { + if (resp_ctrl != nullptr) { + resp_ctrl->release(); + } + return CROWDB_RPC_ERR_SEND_QUEUE; + } + resp_data->write(data, data_len); + } + + auto *frame = crowdb::rpc::build_out_frame(request_id, msg_type, resp_ctrl, resp_data); + if (!server->server->transport()->submit(conn, frame)) { + if (frame->control != nullptr) { + frame->control->release(); + } + frame->release_data(); + delete frame; + return CROWDB_RPC_ERR_SEND_QUEUE; + } + return CROWDB_RPC_OK; + } + catch (...) { + return CROWDB_RPC_ERR_CONN_ERROR; + } +} + +// Submit a response using pre-filled buffer handles (zero-copy). The +// server takes ownership of the buffers. +crowdb_rpc_status crowdb_rpc_server_submit_response_buffer(crowdb_rpc_server_t server, void *conn_handle, + crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, + uint16_t msg_type, uint64_t request_id) +{ + try { + if (server == nullptr || conn_handle == nullptr) { + // Release any provided buffers to avoid leaks. + if (control != nullptr) { + crowdb_rpc_buffer_release(control); + } + if (data != nullptr) { + crowdb_rpc_buffer_release(data); + } + return CROWDB_RPC_ERR_INVALID_ARG; + } + + auto *conn = static_cast(conn_handle); + + // Extract Buffer* from handles (steal ownership — the handle's + // release is a no-op after this since we set buf = nullptr). + crowdb::rpc::Buffer *resp_ctrl = nullptr; + if (control != nullptr && control->buf != nullptr) { + resp_ctrl = control->buf; + control->buf = nullptr; + delete control; + } + crowdb::rpc::Buffer *resp_data = nullptr; + if (data != nullptr && data->buf != nullptr) { + resp_data = data->buf; + data->buf = nullptr; + delete data; + } + + auto *frame = crowdb::rpc::build_out_frame(request_id, msg_type, resp_ctrl, resp_data); + if (!server->server->transport()->submit(conn, frame)) { + if (frame->control != nullptr) { + frame->control->release(); + } + frame->release_data(); + delete frame; + return CROWDB_RPC_ERR_SEND_QUEUE; + } + return CROWDB_RPC_OK; + } + catch (...) { + return CROWDB_RPC_ERR_CONN_ERROR; + } +} + +// ── Client-side request handler dispatch ────────────────── + +void crowdb_rpc_client_register_handler(crowdb_rpc_client_t client, uint16_t msg_type, crowdb_rpc_handler_fn callback, + void *user_data, crowdb_rpc_handler_release_fn release) +{ + try { + auto owner = own_handler_context(user_data, release); + if (client == nullptr || callback == nullptr) { + return; + } + client->client->register_handler(msg_type, callback, std::move(owner)); + } + catch (...) { + } +} + +void crowdb_rpc_client_clear_handlers(crowdb_rpc_client_t client) +{ + try { + if (client == nullptr) { + return; + } + client->client->clear_handlers(); + } + catch (...) { + } +} + +void crowdb_rpc_client_set_transport(crowdb_rpc_client_t client, crowdb_rpc_server_t server) +{ + try { + if (client == nullptr || server == nullptr) { + return; + } + client->client->set_transport(server->server->transport()); + } + catch (...) { + } +} + +// ── Server-side request-response correlation ────────────── + +void crowdb_rpc_server_set_request_client(crowdb_rpc_server_t server, crowdb_rpc_client_t client) +{ + try { + if (server == nullptr || client == nullptr) { + return; + } + server->server->set_request_client(client->client.get()); + } + catch (...) { + } +} diff --git a/lib/crowdb-rpc/src/client/client.cpp b/lib/crowdb-rpc/src/client/client.cpp index d40969d47..1a6ff5b50 100644 --- a/lib/crowdb-rpc/src/client/client.cpp +++ b/lib/crowdb-rpc/src/client/client.cpp @@ -251,10 +251,22 @@ bool RpcClient::send_one_way(Transport *transport, Connection *conn, uint64_t re return false; } -void RpcClient::register_handler(uint16_t msg_type, crowdb_rpc_handler_fn callback, void *user_data) +void RpcClient::register_handler(uint16_t msg_type, crowdb_rpc_handler_fn callback, std::shared_ptr context) { - std::scoped_lock lock(handler_mu_); - request_handlers_[msg_type] = {callback, user_data}; + std::pair> retired{callback, std::move(context)}; + { + std::scoped_lock lock(handler_mu_); + std::swap(request_handlers_[msg_type], retired); + } +} + +void RpcClient::clear_handlers() +{ + decltype(request_handlers_) retired; + { + std::scoped_lock lock(handler_mu_); + retired.swap(request_handlers_); + } } void RpcClient::dispatch_request(Frame *frame, Connection *conn) @@ -262,14 +274,14 @@ void RpcClient::dispatch_request(Frame *frame, Connection *conn) uint16_t msg_type = frame->header.msg_type; bool is_one_way = (frame->header.flags & FLAG_ONE_WAY) != 0; - crowdb_rpc_handler_fn cb = nullptr; - void *user_data = nullptr; + crowdb_rpc_handler_fn cb = nullptr; + std::shared_ptr context; { std::scoped_lock lock(handler_mu_); auto it = request_handlers_.find(msg_type); if (it != request_handlers_.end()) { - cb = it->second.first; - user_data = it->second.second; + cb = it->second.first; + context = it->second.second; } } @@ -277,7 +289,7 @@ void RpcClient::dispatch_request(Frame *frame, Connection *conn) // Same trampoline as server-side dispatch: extract fields, // invoke C callback, delete frame. The callback submits the // response later via crowdb_rpc_server_submit_response. - invoke_c_handler(cb, user_data, frame, conn); + invoke_c_handler(cb, context.get(), frame, conn); return; } diff --git a/lib/crowdb-rpc/src/connection.cpp b/lib/crowdb-rpc/src/connection.cpp index 689f098df..a98c2e1d6 100644 --- a/lib/crowdb-rpc/src/connection.cpp +++ b/lib/crowdb-rpc/src/connection.cpp @@ -173,8 +173,31 @@ int Connection::drain_send_queue(OutFrame **out, int max) // re-enqueued to the MPSC queue, which would break order with concurrent // enqueues). The next try_send sends partials first, then drains the queue. // The caller arms EPOLLOUT for retry. +Connection::SocketUse::SocketUse(Connection &connection) +{ + auto state = connection.socket_users_.load(std::memory_order_acquire); + while ((state & SOCKET_CLOSED) == 0) { + if (connection.socket_users_.compare_exchange_weak(state, state + 1, std::memory_order_acq_rel, + std::memory_order_acquire)) { + connection_ = &connection; + return; + } + } +} + +Connection::SocketUse::~SocketUse() +{ + if (connection_ != nullptr) { + connection_->socket_users_.fetch_sub(1, std::memory_order_release); + } +} + bool Connection::try_send(int fd, TransportStats *stats) { + const auto socket_use = use_socket(); + if (!socket_use) { + return false; + } bool expected = false; if (!in_send_.compare_exchange_strong(expected, true)) { return true; // another thread is sending; our frame is queued @@ -364,6 +387,7 @@ void Connection::close() if (!open_.exchange(false, std::memory_order_acq_rel)) { return; // already closed } + socket_users_.fetch_or(SOCKET_CLOSED, std::memory_order_acq_rel); CRB_LOG_INFO("close: conn_id={} name={}", static_cast(id_), name_); bool expected = false; if (in_send_.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { diff --git a/lib/crowdb-rpc/src/server/server.cpp b/lib/crowdb-rpc/src/server/server.cpp index c5ffd50ce..a81259fba 100644 --- a/lib/crowdb-rpc/src/server/server.cpp +++ b/lib/crowdb-rpc/src/server/server.cpp @@ -281,7 +281,7 @@ void RpcServer::dispatch(Frame *frame, Connection *conn) // this msg_type, dispatch as a request. This ensures request // frames are not intercepted by on_response (which matches by // request_id and can't distinguish a request from its ack). - HandlerFn handler = handlers_.get_handler(msg_type); + auto handler = handlers_.get_handler(msg_type); if (handler) { if (msg_type == static_cast(proto::FBMsgType_EConnectionPingRequest)) { cnt_response_ping().inc(); @@ -309,8 +309,7 @@ void RpcServer::dispatch(Frame *frame, Connection *conn) // Unknown msg_type and not an ack — send UnknownMessage (if not // one-way) or drop. if (!is_one_way) { - handler = handle_unknown; - OutFrame *response = handler(frame, conn); + OutFrame *response = handle_unknown(frame, conn); if (response != nullptr) { transport_->submit_inline(conn, response); } diff --git a/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp b/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp index 0a1229652..69d70172f 100644 --- a/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp +++ b/lib/crowdb-rpc/src/transport/epoll/epoll_engine.cpp @@ -152,12 +152,22 @@ void EpollEngine::remove_connection(int read_fd, int write_fd) void EpollEngine::arm_read(int read_fd, Connection *conn) { + const auto socket_use = conn->use_socket(); + if (!socket_use) { + return; + } + // MOD read fd with EPOLLIN only — does not touch the write fd. mod_fd(read_fd, EPOLLIN, conn); } void EpollEngine::arm_write(int write_fd, Connection *conn) { + const auto socket_use = conn->use_socket(); + if (!socket_use) { + return; + } + // MOD write fd with EPOLLOUT only — does not re-arm read. // Uses mod_fd_write to preserve the token's write bit // so wait() can distinguish write events from read events. @@ -166,6 +176,11 @@ void EpollEngine::arm_write(int write_fd, Connection *conn) void EpollEngine::disarm_write(int write_fd, Connection *conn) { + const auto socket_use = conn->use_socket(); + if (!socket_use) { + return; + } + // MOD write fd with 0 events — disarms write without touching read. mod_fd_write(write_fd, 0, conn); } diff --git a/lib/crowdb-rpc/src/transport/kqueue/kqueue_engine.cpp b/lib/crowdb-rpc/src/transport/kqueue/kqueue_engine.cpp index 0f0ecd09c..303837527 100644 --- a/lib/crowdb-rpc/src/transport/kqueue/kqueue_engine.cpp +++ b/lib/crowdb-rpc/src/transport/kqueue/kqueue_engine.cpp @@ -87,6 +87,11 @@ void KqueueEngine::remove_connection(int read_fd, int write_fd) void KqueueEngine::arm_read(int read_fd, Connection *conn) { + const auto socket_use = conn->use_socket(); + if (!socket_use) { + return; + } + int flags = EV_ADD | (oneshot_ ? EV_ONESHOT : 0); struct kevent change; EV_SET(&change, read_fd, EVFILT_READ, flags, 0, 0, reinterpret_cast(static_cast(conn->id()))); @@ -95,6 +100,11 @@ void KqueueEngine::arm_read(int read_fd, Connection *conn) void KqueueEngine::arm_write(int write_fd, Connection *conn) { + const auto socket_use = conn->use_socket(); + if (!socket_use) { + return; + } + // kqueue uses EVFILT_WRITE on the same fd — independent from EVFILT_READ. int flags = EV_ADD | (oneshot_ ? EV_ONESHOT : 0); struct kevent change; @@ -102,8 +112,13 @@ void KqueueEngine::arm_write(int write_fd, Connection *conn) ::kevent(kq_, &change, 1, nullptr, 0, nullptr); } -void KqueueEngine::disarm_write(int write_fd, Connection * /*conn*/) +void KqueueEngine::disarm_write(int write_fd, Connection *conn) { + const auto socket_use = conn->use_socket(); + if (!socket_use) { + return; + } + struct kevent change; EV_SET(&change, write_fd, EVFILT_WRITE, EV_DELETE, 0, 0, nullptr); ::kevent(kq_, &change, 1, nullptr, 0, nullptr); diff --git a/lib/crowdb-rpc/src/transport/socket_transport.cpp b/lib/crowdb-rpc/src/transport/socket_transport.cpp index f2ffe507e..2abf11acf 100644 --- a/lib/crowdb-rpc/src/transport/socket_transport.cpp +++ b/lib/crowdb-rpc/src/transport/socket_transport.cpp @@ -75,13 +75,63 @@ Worker::~Worker() void Worker::close_connections() { - std::scoped_lock lock(conns_mu_); - for (auto &[fd, conn] : connections_) { - (void)fd; - conn->close(); - close_connection_fds(engine_, conn.get(), transport_); + { + std::scoped_lock lock(conns_mu_); + for (auto &[fd, conn] : connections_) { + (void)fd; + conn->close(); + while (!conn->socket_cleanup_ready()) { + std::this_thread::yield(); + } + close_connection_fds(engine_, conn.get(), transport_); + } + connections_.clear(); + } + discard_closes(); +} + +void Worker::request_close(std::shared_ptr conn) +{ + enqueue_close(new CloseRequest{.connection = std::move(conn), .next = nullptr}); + engine_->notify_worker(); +} + +void Worker::enqueue_close(CloseRequest *request) +{ + auto *head = pending_closes_.load(std::memory_order_relaxed); + do { + request->next = head; + } while ( + !pending_closes_.compare_exchange_weak(head, request, std::memory_order_release, std::memory_order_relaxed)); +} + +void Worker::drain_closes(ClosedConnections &closed_fds) +{ + auto *request = pending_closes_.exchange(nullptr, std::memory_order_acquire); + while (request != nullptr) { + auto *next = request->next; + auto *conn = request->connection.get(); + const int fd = static_cast(conn->transport_handle); + if (!conn->socket_cleanup_ready()) { + enqueue_close(request); + request = next; + continue; + } + close_connection_fds(engine_, conn, transport_); + closed_fds.emplace_back(fd, conn->id()); + delete request; + request = next; + } +} + +void Worker::discard_closes() +{ + auto *request = pending_closes_.exchange(nullptr, std::memory_order_acquire); + while (request != nullptr) { + auto *next = request->next; + delete request; + request = next; } - connections_.clear(); } void Worker::start() @@ -203,6 +253,7 @@ void Worker::run_loop() } } + drain_closes(closed_fds); flush_responses(closed_fds); retire_closed(closed_fds); } @@ -297,6 +348,10 @@ void Worker::process_writable(const EngineEvent &ev, ClosedConnections &closed_f void Worker::process_error(const EngineEvent &ev, ClosedConnections &closed_fds) { if (ev.conn != nullptr && ev.conn->is_open()) { + const auto socket_use = ev.conn->use_socket(); + if (!socket_use) { + return; + } CRB_LOG_WARN("worker: socket error event fd={} conn_id={} name={}", ev.fd, static_cast(ev.conn->id()), ev.conn->name()); int rfd = static_cast(ev.conn->transport_handle); @@ -360,6 +415,11 @@ void Worker::retire_closed(const ClosedConnections &closed_fds) static void on_readable_impl(Connection *conn, int fd, uint8_t *recv_buf, size_t recv_buf_size, std::vector &pending_writes) { + const auto socket_use = conn->use_socket(); + if (!socket_use) { + return; + } + auto &parser = conn->parser(); auto on_frame_cb = [conn](Frame *frame) { conn->on_frame(frame); }; @@ -710,6 +770,9 @@ std::shared_ptr SocketTransport::create_connection(int fd, const std // safely resolve an event even after an explicit caller-side close. conn->set_on_frame(std::move(on_frame)); conn->set_on_close([this, on_close = std::move(on_close)](Connection *c) { + if (auto *worker = static_cast(c->io_worker); worker != nullptr) { + worker->request_close(c->shared_from_this()); + } if (on_close) { on_close(c); } diff --git a/lib/crowdb-rpc/tests/frame_revision_test.cpp b/lib/crowdb-rpc/tests/frame_revision_test.cpp new file mode 100644 index 000000000..d05674266 --- /dev/null +++ b/lib/crowdb-rpc/tests/frame_revision_test.cpp @@ -0,0 +1,181 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "crowdb-protocol/frame.h" + +#include + +#include +#include +#include +#include + +namespace crowdb::protocol +{ +namespace +{ + +constexpr FrameChunkId kChunk{.high = 7, .low = 11}; + +TEST(FrameRevisionTest, ReadsSharedVectorsAndEmitsCanonicalNewRevision) +{ + const auto path = std::filesystem::path(__FILE__).parent_path().parent_path().parent_path() / + "crowdb-protocol/tests/data/frame-vectors.txt"; + std::ifstream input(path); + ASSERT_TRUE(input.is_open()) << path; + std::string line; + size_t count = 0; + while (std::getline(input, line)) { + if (line.empty() || line.front() == '#') { + continue; + } + std::istringstream fields(line); + std::string name; + std::string hex; + fields >> name >> hex; + std::vector bytes; + for (size_t offset = 0; offset < hex.size(); offset += 2) { + bytes.push_back(static_cast(std::stoul(hex.substr(offset, 2), nullptr, 16))); + } + ParsedFrame parsed; + ASSERT_EQ(parse_frame(bytes, kChunk, &parsed), FrameError::Ok) << name; + EXPECT_EQ(parsed.header.write_time_ms, 42); + ASSERT_EQ(parsed.payload.size(), 3); + EXPECT_EQ(parsed.payload[0], 1); + EXPECT_EQ(parsed.payload[1], 2); + EXPECT_EQ(parsed.payload[2], 3); + std::vector encoded; + if (name.starts_with("ordinary")) { + ASSERT_EQ(encode_frame(parsed.header.magic, kChunk, parsed.payload, 42, &encoded), FrameError::Ok); + EXPECT_EQ(encoded, bytes) << name; + } + else if (name == "head" || name == "follower") { + ASSERT_EQ(encode_group_frame(kChunk, parsed.payload, 42, parsed.group, &encoded), FrameError::Ok); + EXPECT_EQ(encoded, bytes) << name; + } + ++count; + } + EXPECT_EQ(count, 6); +} + +TEST(FrameRevisionTest, OldMagicsAreUnsupported) +{ + std::vector bytes; + ASSERT_EQ(encode_frame(FrameMagic::RepoSmallV2, kChunk, std::array{1}, 42, &bytes), FrameError::Ok); + for (const uint16_t magic : {0x0101, 0x0201, 0x0301, 0x0401, 0x0501, 0x0103}) { + bytes[0] = static_cast(magic); + bytes[1] = static_cast(magic >> 8U); + EXPECT_EQ(parse_frame(bytes, kChunk, nullptr), FrameError::UnknownMagic); + } +} + +TEST(FrameRevisionTest, OrdinaryNewRevisionHasBoundedCanonicalFrames) +{ + for (const auto magic : + {FrameMagic::RepoSmallV2, FrameMagic::RepoLargeV2, FrameMagic::BtreePageV2, FrameMagic::PageIndexV2}) { + const std::vector payload(kMaxFrameV2PayloadBytes + 17, 0x65); + std::vector encoded; + ASSERT_EQ(encode_frames(magic, kChunk, payload, 42, &encoded), FrameError::Ok); + EXPECT_EQ(encoded.size(), framed_physical_length(payload.size(), kFrameV2HeaderBytes)); + ParsedFrame first; + ASSERT_EQ(parse_frame(encoded, kChunk, &first), FrameError::Ok); + EXPECT_EQ(first.physical_length, kMaxFrameBytes); + EXPECT_EQ(first.group.role, FrameRole::Ordinary); + std::vector decoded; + ASSERT_EQ(parse_frames(encoded, kChunk, magic, payload.size(), &decoded), FrameError::Ok); + EXPECT_EQ(decoded, payload); + EXPECT_EQ(validate_frame_sequence(encoded), FrameError::Ok); + } + std::vector encoded; + EXPECT_EQ(encode_frame(FrameMagic::StreamV2, kChunk, std::array{1}, 42, &encoded), + FrameError::InvalidRole); +} + +TEST(FrameRevisionTest, InterruptedAppendNeverBecomesACompleteGroup) +{ + for (const size_t length : {size_t{1}, kMaxFrameGroupPayloadBytes, kMaxFrameGroupPayloadBytes + 1, + (3 * kMaxFrameGroupPayloadBytes) + 17}) { + const std::vector payload(length, 0x51); + std::vector encoded; + ASSERT_EQ(encode_append_group(kChunk, 4096, payload, 42, &encoded), FrameError::Ok); + ParsedAppendGroup group; + ASSERT_EQ(parse_append_group(encoded, kChunk, 4096, &group), FrameError::Ok); + EXPECT_EQ(group.logical_length, length); + EXPECT_EQ(group.physical_length, encoded.size()); + EXPECT_EQ(group.frame_count, + (length / kMaxFrameGroupPayloadBytes) + (length % kMaxFrameGroupPayloadBytes != 0 ? 1 : 0)); + const std::span bytes(encoded); + for (size_t boundary = 0; boundary < encoded.size(); boundary += kMaxFrameBytes) { + for (const size_t end : + {boundary == 0 ? 0 : boundary - 1, boundary, std::min(boundary + 1, encoded.size() - 1)}) { + EXPECT_NE(parse_append_group(bytes.first(end), kChunk, 4096, &group), FrameError::Ok) + << length << '/' << end; + } + } + EXPECT_NE(parse_append_group(bytes.first(bytes.size() - 1), kChunk, 4096, &group), FrameError::Ok); + if (length > kMaxFrameGroupPayloadBytes) { + EXPECT_EQ(parse_append_group(bytes, kChunk, 4097, &group), FrameError::InvalidGroup); + } + } +} + +TEST(FrameRevisionTest, ValidFramesCannotBeReorderedOrBorrowedFromAnotherAppend) +{ + const std::vector payload(3 * kMaxFrameGroupPayloadBytes, 0x51); + std::vector encoded; + ASSERT_EQ(encode_append_group(kChunk, 4096, payload, 42, &encoded), FrameError::Ok); + auto reordered = encoded; + std::swap_ranges(reordered.begin() + kMaxFrameBytes, reordered.begin() + 2 * kMaxFrameBytes, + reordered.begin() + 2 * kMaxFrameBytes); + EXPECT_EQ(parse_append_group(reordered, kChunk, 4096, nullptr), FrameError::InvalidGroup); + std::vector other; + ASSERT_EQ(encode_append_group(kChunk, 8192, payload, 42, &other), FrameError::Ok); + std::copy(other.begin() + kMaxFrameBytes, other.begin() + 2 * kMaxFrameBytes, encoded.begin() + kMaxFrameBytes); + EXPECT_EQ(parse_append_group(encoded, kChunk, 4096, nullptr), FrameError::InvalidGroup); + encoded[kMaxFrameBytes + 15] ^= 1U; + EXPECT_NE(parse_append_group(encoded, kChunk, 4096, nullptr), FrameError::Ok); +} + +TEST(FrameRevisionTest, BoundedValidatorRequiresExactOffsetsAndEveryFollower) +{ + const std::vector payload(kMaxFrameGroupPayloadBytes + 1, 7); + std::vector encoded; + ASSERT_EQ(encode_append_group(kChunk, 4096, payload, 42, &encoded), FrameError::Ok); + ParsedFrame head; + ASSERT_EQ(parse_frame(encoded, kChunk, &head), FrameError::Ok); + AppendGroupValidator validator; + ASSERT_EQ(validator.start(4096, head), FrameError::Ok); + EXPECT_FALSE(validator.complete()); + EXPECT_EQ(validator.finish(nullptr), FrameError::InvalidGroup); + ParsedFrame tail; + ASSERT_EQ(parse_frame(std::span(encoded).subspan(head.physical_length), kChunk, &tail), + FrameError::Ok); + EXPECT_EQ(validator.push(4097 + head.physical_length, tail), FrameError::InvalidGroup); + ASSERT_EQ(validator.push(4096 + head.physical_length, tail), FrameError::Ok); + EXPECT_TRUE(validator.complete()); + EXPECT_EQ(validator.push(4096 + encoded.size(), tail), FrameError::InvalidGroup); + ParsedAppendGroup group; + ASSERT_EQ(validator.finish(&group), FrameError::Ok); + EXPECT_EQ(group.physical_length, encoded.size()); +} + +TEST(FrameRevisionTest, RejectsMalformedExtensionsAndAmbiguousCrc) +{ + std::vector encoded; + ASSERT_EQ(encode_frame(FrameMagic::RepoSmallV2, kChunk, std::array{1, 2, 3}, 42, &encoded), + FrameError::Ok); + const size_t footer = encoded.size() - kFrameFooterBytes; + const uint32_t prefix_crc = crowdb::common::crc32c(encoded.data(), footer); + const uint32_t footer_inclusive_crc = crowdb::common::crc32c_update(prefix_crc, encoded.data() + footer + 4, 16); + for (size_t index = 0; index < 4; ++index) { + encoded[footer + index] = static_cast(footer_inclusive_crc >> (index * 8U)); + } + EXPECT_EQ(parse_frame(encoded, kChunk, nullptr), FrameError::ChecksumMismatch); + encoded[14] = 1; + EXPECT_EQ(parse_frame(encoded, kChunk, nullptr), FrameError::InvalidRole); + encoded[2] = 14; + EXPECT_EQ(parse_frame(encoded, kChunk, nullptr), FrameError::InvalidPayloadOffset); +} + +} // namespace +} // namespace crowdb::protocol diff --git a/lib/crowdb-rpc/tests/frame_test.cpp b/lib/crowdb-rpc/tests/frame_test.cpp index 959b76883..679a313a6 100644 --- a/lib/crowdb-rpc/tests/frame_test.cpp +++ b/lib/crowdb-rpc/tests/frame_test.cpp @@ -17,30 +17,31 @@ TEST(FrameTest, RoundTripAndDetectsCorruption) const FrameChunkId chunk{.high = 7, .low = 11}; const std::array payload{1, 2, 3}; std::vector encoded; - ASSERT_EQ(encode_frame(FrameMagic::RepoSmallV1, chunk, payload, 42, &encoded), FrameError::Ok); + ASSERT_EQ(encode_frame(FrameMagic::RepoSmallV2, chunk, payload, 42, &encoded), FrameError::Ok); ParsedFrame decoded{}; ASSERT_EQ(parse_frame(encoded, chunk, &decoded), FrameError::Ok); - EXPECT_EQ(decoded.header.magic, FrameMagic::RepoSmallV1); + EXPECT_EQ(decoded.header.magic, FrameMagic::RepoSmallV2); EXPECT_EQ(decoded.header.write_time_ms, 42); ASSERT_EQ(decoded.payload.size(), payload.size()); EXPECT_TRUE(std::equal(decoded.payload.begin(), decoded.payload.end(), payload.begin())); - encoded[kFrameHeaderPrefixBytes] ^= 1U; + EXPECT_EQ(parse_frame(encoded, {.high = 7, .low = 12}, &decoded), FrameError::ChunkIdMismatch); + + encoded[kFrameV2HeaderBytes] ^= 1U; EXPECT_EQ(parse_frame(encoded, chunk, &decoded), FrameError::ChecksumMismatch); } -TEST(FrameTest, MatchesCrossLanguageVector) +TEST(FrameTest, MatchesCanonicalCrossLanguageVector) { constexpr FrameChunkId chunk{.high = 7, .low = 11}; constexpr std::array payload{1, 2, 3}; - constexpr std::array expected{ - 0x01, 0x01, 0x0E, 0x00, 0x03, 0x00, 0x2A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x01, 0x02, 0x03, 0x96, 0x16, 0xD6, 0x1E, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0B, - }; - std::vector encoded; - ASSERT_EQ(encode_frame(FrameMagic::RepoSmallV1, chunk, payload, 42, &encoded), FrameError::Ok); + constexpr std::array expected{0x02, 0x01, 0x0f, 0x00, 0x03, 0x00, 0x2a, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x02, 0x03, 0xaf, 0x5a, + 0xa0, 0xae, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b}; + std::vector encoded; + ASSERT_EQ(encode_frame(FrameMagic::RepoSmallV2, chunk, payload, 42, &encoded), FrameError::Ok); EXPECT_TRUE(std::equal(encoded.begin(), encoded.end(), expected.begin(), expected.end())); ParsedFrame decoded{}; @@ -54,13 +55,13 @@ TEST(FrameTest, ValidatesConcatenatedPublicFramesWithoutLocationMetadata) const std::array first{1, 2}; const std::array second{3, 4, 5}; std::vector encoded; - ASSERT_EQ(encode_frame(FrameMagic::StreamV1, chunk, first, 1, &encoded), FrameError::Ok); + ASSERT_EQ(encode_frame(FrameMagic::RepoLargeV2, chunk, first, 1, &encoded), FrameError::Ok); std::vector next; - ASSERT_EQ(encode_frame(FrameMagic::StreamV1, chunk, second, 2, &next), FrameError::Ok); + ASSERT_EQ(encode_frame(FrameMagic::RepoLargeV2, chunk, second, 2, &next), FrameError::Ok); encoded.insert(encoded.end(), next.begin(), next.end()); EXPECT_EQ(validate_frame_sequence(encoded), FrameError::Ok); - encoded.back() ^= 1U; + encoded[encoded.size() - kFrameFooterBytes - 1] ^= 1U; EXPECT_EQ(validate_frame_sequence(encoded), FrameError::ChecksumMismatch); } diff --git a/lib/crowdb-rpc/tests/transport_test.cpp b/lib/crowdb-rpc/tests/transport_test.cpp index 06e1a1e0d..2aed7818a 100644 --- a/lib/crowdb-rpc/tests/transport_test.cpp +++ b/lib/crowdb-rpc/tests/transport_test.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -427,3 +428,82 @@ TEST_F(TransportLoopbackTest, CrossThreadSubmitRejectsFreedPeerAfterClose) delete frame; transport.stop(); } + +TEST_F(TransportLoopbackTest, ExplicitIdleCloseReclaimsSocketsAcrossWorkersWhileHandlesAreRetained) +{ + SocketTransport transport(2, 4); + transport.start(); + std::vector> connections; + std::vector peers; + std::vector descriptors; + connections.reserve(128); + peers.reserve(128); + descriptors.reserve(256); + for (int i = 0; i < 128; ++i) { + int sockets[2]; + ASSERT_EQ(socketpair_nonblocking(sockets), 0); + auto connection = transport.create_connection(sockets[0], "idle-close"); + descriptors.push_back(sockets[0]); + descriptors.push_back(connection->write_fd); + connections.push_back(std::move(connection)); + peers.push_back(sockets[1]); + } + std::vector closers; + closers.reserve(8); + for (size_t worker = 0; worker < 8; ++worker) { + closers.emplace_back([&, worker] { + for (size_t i = worker; i < connections.size(); i += 8) { + connections[i]->close(); + } + }); + } + for (auto &closer : closers) { + closer.join(); + } + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); + const auto closed = [&] { + return transport.connection_count() == 0 && + std::all_of(descriptors.begin(), descriptors.end(), [](int fd) { return ::fcntl(fd, F_GETFD) == -1; }); + }; + while (!closed() && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + EXPECT_EQ(transport.connection_count(), 0U); + for (int fd : descriptors) { + EXPECT_EQ(::fcntl(fd, F_GETFD), -1); + EXPECT_EQ(errno, EBADF); + } + for (int peer : peers) { + char byte{}; + EXPECT_EQ(::read(peer, &byte, 1), 0); + ::close(peer); + } + transport.stop(); +} + +TEST_F(TransportLoopbackTest, ExplicitCloseDefersDescriptorReuseUntilActiveSocketIoRetires) +{ + SocketTransport transport(1, 2); + transport.start(); + int sockets[2]; + ASSERT_EQ(socketpair_nonblocking(sockets), 0); + auto connection = transport.create_connection(sockets[0], "active-socket-io"); + const int write_fd = connection->write_fd; + { + const auto socket_use = connection->use_socket(); + ASSERT_TRUE(socket_use); + connection->close(); + EXPECT_FALSE(connection->use_socket()); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_NE(::fcntl(sockets[0], F_GETFD), -1); + EXPECT_NE(::fcntl(write_fd, F_GETFD), -1); + } + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(3); + while (::fcntl(sockets[0], F_GETFD) != -1 && std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + EXPECT_EQ(::fcntl(sockets[0], F_GETFD), -1); + EXPECT_EQ(::fcntl(write_fd, F_GETFD), -1); + ::close(sockets[1]); + transport.stop(); +} diff --git a/lib/crowdb-tree/CMakeLists.txt b/lib/crowdb-tree/CMakeLists.txt index f6f2ba6fc..d8b2ebe2f 100644 --- a/lib/crowdb-tree/CMakeLists.txt +++ b/lib/crowdb-tree/CMakeLists.txt @@ -108,7 +108,7 @@ target_include_directories(crowdb_tree_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} target_include_directories(crowdb_tree_tests PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/crowdb-rpc-build/generated) target_include_directories(crowdb_tree_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../crowdb-rpc/include) target_include_directories(crowdb_tree_tests SYSTEM PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../../third-party/stdexec/include) -target_link_libraries(crowdb_tree_tests PRIVATE crowdb-tree GTest::gtest) +target_link_libraries(crowdb_tree_tests PRIVATE crowdb-tree crowdb-rpc GTest::gtest) # GCC 13 -Warray-bounds/-Wstringop-overflow false positives: buffer::alloc # returns heap memory, but GCC thinks b.data() points into the stack object b. # These flags are GCC-specific (clang doesn't know -Wstringop-overflow); guard diff --git a/lib/crowdb-tree/ffi/src/chunk/transport.rs b/lib/crowdb-tree/ffi/src/chunk/transport.rs index 50315af1b..9d32cdd06 100644 --- a/lib/crowdb-tree/ffi/src/chunk/transport.rs +++ b/lib/crowdb-tree/ffi/src/chunk/transport.rs @@ -33,6 +33,7 @@ pub struct ChunkRpcTransportOptions<'a> { pub rpc_timeout_ms: u64, pub completion_capacity: u32, pub mirror_copies: u32, + pub strip_capacity_bytes: u32, } #[cfg(feature = "chunk-rpc")] @@ -52,6 +53,7 @@ pub struct OwnedChunkRpcTransportOptions { pub rpc_timeout_ms: u64, pub completion_capacity: u32, pub mirror_copies: u32, + pub strip_capacity_bytes: u32, } #[cfg(feature = "chunk-rpc")] @@ -94,6 +96,7 @@ impl ChunkTransport { rpc_timeout_ms: options.rpc_timeout_ms, completion_capacity: options.completion_capacity, mirror_copies: options.mirror_copies, + strip_capacity_bytes: options.strip_capacity_bytes, chunkdb_resolver: sys::ct_chunk_rpc_resolver::default(), disk_resolver: sys::ct_chunk_rpc_resolver::default(), }; @@ -133,6 +136,7 @@ impl ChunkTransport { rpc_timeout_ms: options.rpc_timeout_ms, completion_capacity: options.completion_capacity, mirror_copies: options.mirror_copies, + strip_capacity_bytes: options.strip_capacity_bytes, chunkdb_resolver: sys::ct_chunk_rpc_resolver { context: Arc::as_ptr(&owners).cast_mut().cast(), resolve: Some(resolve_route), diff --git a/lib/crowdb-tree/ffi/src/sys.rs b/lib/crowdb-tree/ffi/src/sys.rs index 5565fe2e3..d3091deba 100644 --- a/lib/crowdb-tree/ffi/src/sys.rs +++ b/lib/crowdb-tree/ffi/src/sys.rs @@ -109,6 +109,7 @@ pub struct ct_chunk_rpc_transport_options { pub rpc_timeout_ms: u64, pub completion_capacity: u32, pub mirror_copies: u32, + pub strip_capacity_bytes: u32, pub chunkdb_resolver: ct_chunk_rpc_resolver, pub disk_resolver: ct_chunk_rpc_resolver, } diff --git a/lib/crowdb-tree/ffi/tests/ffi_test.rs b/lib/crowdb-tree/ffi/tests/ffi_test.rs index 9963c4e9a..8f9f8c5f8 100644 --- a/lib/crowdb-tree/ffi/tests/ffi_test.rs +++ b/lib/crowdb-tree/ffi/tests/ffi_test.rs @@ -153,6 +153,7 @@ fn owned_chunk_rpc_transport_retains_route_handles() { rpc_timeout_ms: 1_000, completion_capacity: 32, mirror_copies: 0, + strip_capacity_bytes: 0, }) .unwrap(); let store = PageStore::open_chunk( diff --git a/lib/crowdb-tree/include/crowdb-tree/c_api.h b/lib/crowdb-tree/include/crowdb-tree/c_api.h index 8c841d4a7..35e7c7a10 100644 --- a/lib/crowdb-tree/include/crowdb-tree/c_api.h +++ b/lib/crowdb-tree/include/crowdb-tree/c_api.h @@ -242,7 +242,8 @@ struct ct_chunk_rpc_transport_options uint64_t writer_lease_ms; uint64_t rpc_timeout_ms; // 0 => 30 seconds uint32_t completion_capacity; - uint32_t mirror_copies; // 0 => 2 + uint32_t mirror_copies; // 0 => 2 + uint32_t strip_capacity_bytes; // 0 => automatic ct_chunk_rpc_resolver chunkdb_resolver; ct_chunk_rpc_resolver disk_resolver; }; diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_catalog_checksum.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_catalog_checksum.cpp new file mode 100644 index 000000000..4444fc500 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_catalog_checksum.cpp @@ -0,0 +1,72 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "chunk_catalog_codec.h" +#include "crowdb-common/crc32c.h" + +namespace crowdb::tree::detail +{ +namespace +{ +uint32_t update_u64(uint32_t crc, uint64_t value) +{ + return crowdb::common::crc32c_update(crc, reinterpret_cast(&value), sizeof(value)); +} + +} // namespace + +uint32_t reference_segment_checksum(const ChunkReferenceSegmentImage &segment) +{ + uint32_t crc = update_u64(0, segment.object_id); + crc = update_u64(crc, segment.first_ordinal); + for (const ChunkPageRef &ref : segment.refs) { + crc = update_u64(crc, ref.chunk_id.high); + crc = update_u64(crc, ref.chunk_id.low); + crc = update_u64(crc, ref.offset); + const uint32_t encoded_length = ref.length; + crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&encoded_length), + sizeof(encoded_length)); + } + return crc; +} + +uint32_t manifest_checksum(const ChunkManifest &manifest) +{ + uint32_t crc = 0; + crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&manifest.format_version), + sizeof(manifest.format_version)); + crc = update_u64(crc, manifest.tree_id); + crc = update_u64(crc, manifest.generation); + crc = update_u64(crc, manifest.owner_epoch); + crc = update_u64(crc, manifest.logical_size); + crc = update_u64(crc, manifest.published_at_ms); + crc = update_u64(crc, manifest.packs_reused); + crc = update_u64(crc, manifest.pack_bytes_reused); + crc = update_u64(crc, manifest.wal_replay_offset); + for (const ChunkPagePack &pack : manifest.packs) { + crc = update_u64(crc, pack.owner_tree_id); + crc = update_u64(crc, pack.ordinal); + crc = update_u64(crc, pack.logical_offset); + crc = update_u64(crc, pack.ref.chunk_id.high); + crc = update_u64(crc, pack.ref.chunk_id.low); + crc = update_u64(crc, pack.ref.offset); + const uint32_t encoded_length = pack.ref.length; + crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&encoded_length), + sizeof(encoded_length)); + crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&pack.reused), sizeof(pack.reused)); + } + for (const ChunkReferenceSegment &segment : manifest.reference_segments) { + crc = update_u64(crc, segment.owner_tree_id); + crc = update_u64(crc, segment.object_id); + crc = update_u64(crc, segment.first_ordinal); + crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&segment.ref_count), + sizeof(segment.ref_count)); + crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&segment.checksum), + sizeof(segment.checksum)); + crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&segment.reused), + sizeof(segment.reused)); + } + return crc; +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_catalog_codec.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_catalog_codec.cpp new file mode 100644 index 000000000..823f00591 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_catalog_codec.cpp @@ -0,0 +1,200 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "chunk_catalog_codec.h" + +#include "chunk_frame.h" + +#include + +namespace crowdb::tree::detail +{ +namespace +{ +template void append_scalar(std::vector *out, T value) +{ + for (size_t i = 0; i < sizeof(T); ++i) { + out->push_back(static_cast((value >> (i * 8U)) & 0xffU)); + } +} + +class BlobReader +{ + public: + explicit BlobReader(const std::vector &bytes) : bytes_(bytes) + { + } + + template bool scalar(T *out) + { + if (out == nullptr || offset_ > bytes_.size() || sizeof(T) > bytes_.size() - offset_) { + return false; + } + T value = 0; + for (size_t i = 0; i < sizeof(T); ++i) { + value |= static_cast(bytes_[offset_ + i]) << (i * 8U); + } + offset_ += sizeof(T); + *out = value; + return true; + } + + [[nodiscard]] bool complete() const + { + return offset_ == bytes_.size(); + } + + [[nodiscard]] size_t remaining() const + { + return bytes_.size() - offset_; + } + + private: + const std::vector &bytes_; + size_t offset_ = 0; +}; + +} // namespace + +namespace +{ +bool valid_page_ref(const ChunkPageRef &ref) +{ + return ref.length > 0 && ref.length <= kMaxChunkPackBytes; +} +} // namespace + +std::vector encode_reference_segment(const ChunkReferenceSegmentImage &segment) +{ + if (std::any_of(segment.refs.begin(), segment.refs.end(), [](const auto &ref) { return !valid_page_ref(ref); })) { + return {}; + } + std::vector out; + out.reserve(24 + (segment.refs.size() * 28)); + append_scalar(&out, segment.object_id); + append_scalar(&out, segment.first_ordinal); + append_scalar(&out, static_cast(segment.refs.size())); + for (const auto &ref : segment.refs) { + append_scalar(&out, ref.chunk_id.high); + append_scalar(&out, ref.chunk_id.low); + append_scalar(&out, ref.offset); + append_scalar(&out, ref.length); + } + return out; +} + +std::shared_ptr decode_reference_segment(const std::vector &bytes) +{ + BlobReader reader(bytes); + auto segment = std::make_shared(); + uint64_t count = 0; + if (!reader.scalar(&segment->object_id) || !reader.scalar(&segment->first_ordinal) || !reader.scalar(&count) || + count > reader.remaining() / 28U) { + return nullptr; + } + segment->refs.reserve(static_cast(count)); + for (uint64_t i = 0; i < count; ++i) { + ChunkPageRef ref; + if (!reader.scalar(&ref.chunk_id.high) || !reader.scalar(&ref.chunk_id.low) || !reader.scalar(&ref.offset) || + !reader.scalar(&ref.length)) { + return nullptr; + } + if (ref.length == 0 || ref.length > kMaxChunkPackBytes) { + return nullptr; + } + segment->refs.push_back(ref); + } + return reader.complete() ? segment : nullptr; +} + +std::vector encode_manifest(const ChunkManifest &manifest) +{ + if (manifest.format_version != kChunkManifestFormat || + std::any_of(manifest.packs.begin(), manifest.packs.end(), + [&](const auto &pack) { return !valid_page_ref(pack.ref); })) { + return {}; + } + std::vector out; + append_scalar(&out, manifest.format_version); + append_scalar(&out, manifest.tree_id); + append_scalar(&out, manifest.generation); + append_scalar(&out, manifest.owner_epoch); + append_scalar(&out, manifest.logical_size); + append_scalar(&out, manifest.published_at_ms); + append_scalar(&out, manifest.packs_reused); + append_scalar(&out, manifest.pack_bytes_reused); + append_scalar(&out, manifest.wal_replay_offset); + append_scalar(&out, manifest.checksum); + append_scalar(&out, static_cast(manifest.reference_segments.size())); + for (const auto &segment : manifest.reference_segments) { + append_scalar(&out, segment.owner_tree_id); + append_scalar(&out, segment.object_id); + append_scalar(&out, segment.first_ordinal); + append_scalar(&out, segment.ref_count); + append_scalar(&out, segment.checksum); + append_scalar(&out, static_cast(segment.reused)); + } + append_scalar(&out, static_cast(manifest.packs.size())); + for (const auto &pack : manifest.packs) { + append_scalar(&out, pack.owner_tree_id); + append_scalar(&out, pack.ordinal); + append_scalar(&out, pack.logical_offset); + append_scalar(&out, pack.ref.chunk_id.high); + append_scalar(&out, pack.ref.chunk_id.low); + append_scalar(&out, pack.ref.offset); + append_scalar(&out, pack.ref.length); + append_scalar(&out, static_cast(pack.reused)); + } + return out; +} + +std::shared_ptr decode_manifest(const std::vector &bytes) +{ + BlobReader reader(bytes); + auto manifest = std::make_shared(); + uint64_t reference_count = 0; + if (!reader.scalar(&manifest->format_version) || manifest->format_version != kChunkManifestFormat || + !reader.scalar(&manifest->tree_id) || !reader.scalar(&manifest->generation) || + !reader.scalar(&manifest->owner_epoch) || !reader.scalar(&manifest->logical_size) || + !reader.scalar(&manifest->published_at_ms) || !reader.scalar(&manifest->packs_reused) || + !reader.scalar(&manifest->pack_bytes_reused) || !reader.scalar(&manifest->wal_replay_offset) || + !reader.scalar(&manifest->checksum) || !reader.scalar(&reference_count) || + reference_count > reader.remaining() / 33U) { + return nullptr; + } + manifest->reference_segments.reserve(static_cast(reference_count)); + for (uint64_t i = 0; i < reference_count; ++i) { + ChunkReferenceSegment segment; + uint8_t reused = 0; + if (!reader.scalar(&segment.owner_tree_id) || !reader.scalar(&segment.object_id) || + !reader.scalar(&segment.first_ordinal) || !reader.scalar(&segment.ref_count) || + !reader.scalar(&segment.checksum) || !reader.scalar(&reused) || reused > 1) { + return nullptr; + } + segment.reused = reused != 0; + manifest->reference_segments.push_back(segment); + } + uint64_t pack_count = 0; + if (!reader.scalar(&pack_count) || pack_count > reader.remaining() / 53U) { + return nullptr; + } + manifest->packs.reserve(static_cast(pack_count)); + for (uint64_t i = 0; i < pack_count; ++i) { + ChunkPagePack pack; + uint8_t reused = 0; + if (!reader.scalar(&pack.owner_tree_id) || !reader.scalar(&pack.ordinal) || + !reader.scalar(&pack.logical_offset) || !reader.scalar(&pack.ref.chunk_id.high) || + !reader.scalar(&pack.ref.chunk_id.low) || !reader.scalar(&pack.ref.offset) || + !reader.scalar(&pack.ref.length) || !reader.scalar(&reused) || reused > 1) { + return nullptr; + } + if (!valid_page_ref(pack.ref)) { + return nullptr; + } + pack.reused = reused != 0; + manifest->packs.push_back(pack); + } + return reader.complete() ? manifest : nullptr; +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_catalog_codec.h b/lib/crowdb-tree/src/backend/chunk/chunk_catalog_codec.h new file mode 100644 index 000000000..9b7ca83ec --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_catalog_codec.h @@ -0,0 +1,20 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "chunk_page_store.h" + +namespace crowdb::tree::detail +{ + +[[nodiscard]] uint32_t reference_segment_checksum(const ChunkReferenceSegmentImage &segment); +[[nodiscard]] uint32_t manifest_checksum(const ChunkManifest &manifest); + +[[nodiscard]] std::vector encode_reference_segment(const ChunkReferenceSegmentImage &segment); +[[nodiscard]] std::shared_ptr +decode_reference_segment(const std::vector &bytes); +[[nodiscard]] std::vector encode_manifest(const ChunkManifest &manifest); +[[nodiscard]] std::shared_ptr decode_manifest(const std::vector &bytes); + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_frame.h b/lib/crowdb-tree/src/backend/chunk/chunk_frame.h new file mode 100644 index 000000000..749913e39 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_frame.h @@ -0,0 +1,25 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "chunk_purpose.h" +#include "crowdb-protocol/frame.h" + +namespace crowdb::tree::detail +{ + +inline constexpr uint32_t kMaxChunkPackBytes = 256U * 1024U * 1024U; + +[[nodiscard]] inline crowdb::protocol::FrameMagic chunk_page_frame_magic(PagePurpose purpose) +{ + return purpose == PagePurpose::kPageIndex ? crowdb::protocol::FrameMagic::PageIndexV2 + : crowdb::protocol::FrameMagic::BtreePageV2; +} + +[[nodiscard]] inline size_t chunk_framed_length(size_t logical) +{ + return crowdb::protocol::framed_physical_length(logical, crowdb::protocol::kFrameV2HeaderBytes); +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp index 49e2e546e..d1ed67003 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_pack_pipeline.cpp @@ -4,6 +4,8 @@ #include "chunk_pack_pipeline.h" #include "c_api_internal.h" +#include "chunk_catalog_codec.h" +#include "chunk_frame.h" #include "chunk_page_store.h" #include "crowdb-common/crc32c.h" #include "crowdb-protocol/frame.h" @@ -17,11 +19,6 @@ namespace crowdb::tree::detail { -static crowdb::protocol::FrameMagic page_frame_magic(PagePurpose purpose) -{ - return purpose == PagePurpose::kPageIndex ? crowdb::protocol::FrameMagic::PageIndexV1 - : crowdb::protocol::FrameMagic::BtreePageV1; -} uint64_t monotonic_millis() { @@ -205,8 +202,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable } const size_t length = store->staged_pack_length(offset, reuse_base.get()); const auto purpose = store->staged_purposes_.at(offset); - if (reuse_base != nullptr && reuse_base->format_version >= 3 && - !ChunkPageStore::range_has_pack(*reuse_base, offset, length) && + if (reuse_base != nullptr && !ChunkPageStore::range_has_pack(*reuse_base, offset, length) && !store->range_was_written(offset, length)) { offset += length; continue; @@ -219,7 +215,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable } if (reused != nullptr && chunk_purpose(reused->ref.chunk_id) == purpose) { manifest->packs.push_back({ - .owner_tree_id = chunk_pack_owner(*reuse_base, *reused), + .owner_tree_id = (*reused).owner_tree_id, .ordinal = manifest->packs.size(), .logical_offset = offset, .ref = reused->ref, @@ -233,7 +229,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable } if (!chunk.empty() && (chunk_purpose(chunk) != purpose || length > store->config_.max_chunk_bytes - logical_bytes || - crowdb::protocol::framed_physical_length(length) > store->chunk_allocation_bytes() - cursor)) { + chunk_framed_length(length) > store->chunk_allocation_bytes() - cursor)) { if (writes.empty()) { initial_rotated_chunk = chunk; initial_rotated_cursor = cursor; @@ -245,8 +241,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable if (chunk.empty()) { const uint64_t packs_per_chunk = (store->config_.max_chunk_bytes + store->config_.pack_bytes - 1) / store->config_.pack_bytes; - const uint64_t physical_pack_bytes = - crowdb::protocol::framed_physical_length(store->config_.pack_bytes); + const uint64_t physical_pack_bytes = chunk_framed_length(store->config_.pack_bytes); if (physical_pack_bytes == 0 || packs_per_chunk > std::numeric_limits::max() / physical_pack_bytes) { return Status::resource_exhausted("chunk page framing exceeds address space"); @@ -267,7 +262,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable .offset = cursor, .length = static_cast(length), }; - job->physical_length = crowdb::protocol::framed_physical_length(length); + job->physical_length = chunk_framed_length(length); job->source_offset = offset; manifest->packs.push_back(job->pack); logical_bytes += length; @@ -361,7 +356,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable remember_orphan_segments(); return segment_status; } - manifest->checksum = ChunkPageStore::manifest_checksum(*manifest); + manifest->checksum = manifest_checksum(*manifest); Status validation_status = crowdb::tree::detail::ChunkPageStore::validate_manifest(*manifest, *store->catalog_); if (!validation_status.ok()) { remember_orphan_segments(); @@ -407,7 +402,7 @@ class ChunkPackPipelineImpl final : public ChunkPackPipeline, public std::enable auto &job = *writes[index]; orphan_pack_bytes.fetch_add(job.pack.ref.length, std::memory_order_relaxed); const auto frame_error = crowdb::protocol::encode_frames( - page_frame_magic(chunk_purpose(job.pack.ref.chunk_id)), + chunk_page_frame_magic(chunk_purpose(job.pack.ref.chunk_id)), {.high = job.pack.ref.chunk_id.high, .low = job.pack.ref.chunk_id.low}, std::span(store->staged_.data() + job.source_offset, job.pack.ref.length), monotonic_millis(), &job.framed); diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_pack_read.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_pack_read.cpp new file mode 100644 index 000000000..9dd013451 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/chunk_pack_read.cpp @@ -0,0 +1,80 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "chunk_frame.h" +#include "chunk_page_store.h" +#include "crowdb-common/log.h" + +#include + +namespace crowdb::tree::detail +{ +namespace +{ +uint64_t monotonic_nanos() +{ + return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} +} // namespace + +Status ChunkPageStore::read_pack(const ChunkPageRef &ref, std::shared_ptr> *out, + ChunkCancellation cancellation) const +{ + if (cancellation.cancelled()) { + return Status::unavailable("chunk page read cancelled"); + } + const bool use_cache = cancellation.cancelled_id != nullptr; + auto cached = use_cache ? cached_pack_ : nullptr; + if (cached != nullptr && cached->ref.chunk_id == ref.chunk_id && cached->ref.offset == ref.offset && + cached->ref.length == ref.length) { + if (cancellation.cancelled()) { + return Status::unavailable("chunk page read cancelled"); + } + *out = cached->bytes; + coalesced_reads_.fetch_add(1, std::memory_order_relaxed); + coalesced_read_bytes_.fetch_add(ref.length, std::memory_order_relaxed); + return Status::Ok(); + } + pack_reads_.fetch_add(1, std::memory_order_relaxed); + const size_t physical_length = chunk_framed_length(ref.length); + if (physical_length == 0) { + return Status::corruption("chunk page frame length overflows"); + } + auto framed = std::make_shared>(physical_length); + bool mirror_responded = false; + for (uint32_t mirror = 0; mirror < config_.mirror_copies; ++mirror) { + if (cancellation.cancelled()) { + return Status::unavailable("chunk page read cancelled"); + } + const uint64_t started = monotonic_nanos(); + Status read_status = transport_->read_mirror(ref.chunk_id, mirror, ref.offset, framed->data(), framed->size()); + diskio_operations_.fetch_add(1, std::memory_order_relaxed); + diskio_latency_ns_.fetch_add(monotonic_nanos() - started, std::memory_order_relaxed); + if (!read_status.ok()) { + CRB_LOG_ERROR("chunk page store tree={} mirror={} read failed: chunk={}:{} offset={} length={} status={}", + config_.tree_id, mirror, ref.chunk_id.high, ref.chunk_id.low, ref.offset, physical_length, + read_status.to_string()); + continue; + } + mirror_responded = true; + auto payload = std::make_shared>(); + const auto frame_error = crowdb::protocol::parse_frames( + *framed, {.high = ref.chunk_id.high, .low = ref.chunk_id.low}, + chunk_page_frame_magic(chunk_purpose(ref.chunk_id)), ref.length, payload.get()); + if (frame_error == crowdb::protocol::FrameError::Ok) { + auto entry = std::make_shared(CachedPack{.ref = ref, .bytes = payload}); + if (use_cache) { + cached_pack_ = std::move(entry); + } + *out = std::move(payload); + return cancellation.cancelled() ? Status::unavailable("chunk page read cancelled") : Status::Ok(); + } + CRB_LOG_ERROR("chunk page store tree={} mirror={} frame verification failed: chunk={}:{} offset={} length={}", + config_.tree_id, mirror, ref.chunk_id.high, ref.chunk_id.low, ref.offset, physical_length); + } + return mirror_responded ? Status::corruption("chunk page frame verification failed on every mirror") + : Status::unavailable("chunk page pack mirrors are unavailable"); +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp index 365c8f498..59a710bfb 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_purpose.cpp @@ -1,6 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#include "chunk_frame.h" #include "chunk_page_store.h" #include "crowdb-protocol/frame.h" @@ -92,7 +93,7 @@ bool ChunkPageStore::range_has_pack(const ChunkManifest &base, uint64_t offset, uint64_t ChunkPageStore::chunk_allocation_bytes() const { const auto packs = (config_.max_chunk_bytes + config_.pack_bytes - 1) / config_.pack_bytes; - return packs * crowdb::protocol::framed_physical_length(config_.pack_bytes); + return packs * chunk_framed_length(config_.pack_bytes); } } // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp index 76aa29cfc..6f1b8d4d6 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.cpp @@ -8,6 +8,8 @@ #include "c_api_internal.h" #include "chunk_async_executor.h" #include "chunk_c_api_internal.h" +#include "chunk_catalog_codec.h" +#include "chunk_frame.h" #include "chunk_pack_pipeline.h" #include "crowdb-common/crc32c.h" #include "crowdb-common/log.h" @@ -21,23 +23,12 @@ namespace crowdb::tree::detail { -static crowdb::protocol::FrameMagic page_frame_magic(PagePurpose purpose) -{ - return purpose == PagePurpose::kPageIndex ? crowdb::protocol::FrameMagic::PageIndexV1 - : crowdb::protocol::FrameMagic::BtreePageV1; -} - namespace { constexpr size_t kReferencesPerSegment = 256; constexpr uint64_t kMaxChunkBytes = 256U * 1024U * 1024U; -uint32_t update_u64(uint32_t crc, uint64_t value) -{ - return crowdb::common::crc32c_update(crc, reinterpret_cast(&value), sizeof(value)); -} - uint64_t monotonic_millis() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) @@ -50,170 +41,6 @@ uint64_t monotonic_nanos() .count(); } -template void append_scalar(std::vector *out, T value) -{ - for (size_t i = 0; i < sizeof(T); ++i) { - out->push_back(static_cast((value >> (i * 8U)) & 0xffU)); - } -} - -class BlobReader -{ - public: - explicit BlobReader(const std::vector &bytes) : bytes_(bytes) - { - } - - template bool scalar(T *out) - { - if (out == nullptr || offset_ > bytes_.size() || sizeof(T) > bytes_.size() - offset_) { - return false; - } - T value = 0; - for (size_t i = 0; i < sizeof(T); ++i) { - value |= static_cast(bytes_[offset_ + i]) << (i * 8U); - } - offset_ += sizeof(T); - *out = value; - return true; - } - - [[nodiscard]] bool complete() const - { - return offset_ == bytes_.size(); - } - - [[nodiscard]] size_t remaining() const - { - return bytes_.size() - offset_; - } - - private: - const std::vector &bytes_; - size_t offset_ = 0; -}; - -std::vector encode_reference_segment(const ChunkReferenceSegmentImage &segment) -{ - std::vector out; - out.reserve(24 + (segment.refs.size() * 28)); - append_scalar(&out, segment.object_id); - append_scalar(&out, segment.first_ordinal); - append_scalar(&out, static_cast(segment.refs.size())); - for (const auto &ref : segment.refs) { - append_scalar(&out, ref.chunk_id.high); - append_scalar(&out, ref.chunk_id.low); - append_scalar(&out, ref.offset); - append_scalar(&out, ref.length); - } - return out; -} - -std::shared_ptr decode_reference_segment(const std::vector &bytes) -{ - BlobReader reader(bytes); - auto segment = std::make_shared(); - uint64_t count = 0; - if (!reader.scalar(&segment->object_id) || !reader.scalar(&segment->first_ordinal) || !reader.scalar(&count) || - count > reader.remaining() / 28U) { - return nullptr; - } - segment->refs.reserve(static_cast(count)); - for (uint64_t i = 0; i < count; ++i) { - ChunkPageRef ref; - if (!reader.scalar(&ref.chunk_id.high) || !reader.scalar(&ref.chunk_id.low) || !reader.scalar(&ref.offset) || - !reader.scalar(&ref.length)) { - return nullptr; - } - segment->refs.push_back(ref); - } - return reader.complete() ? segment : nullptr; -} - -std::vector encode_manifest(const ChunkManifest &manifest) -{ - std::vector out; - append_scalar(&out, manifest.format_version); - append_scalar(&out, manifest.tree_id); - append_scalar(&out, manifest.generation); - append_scalar(&out, manifest.owner_epoch); - append_scalar(&out, manifest.logical_size); - append_scalar(&out, manifest.published_at_ms); - append_scalar(&out, manifest.packs_reused); - append_scalar(&out, manifest.pack_bytes_reused); - if (manifest.format_version >= 4) { - append_scalar(&out, manifest.wal_replay_offset); - } - append_scalar(&out, manifest.checksum); - append_scalar(&out, static_cast(manifest.reference_segments.size())); - for (const auto &segment : manifest.reference_segments) { - append_scalar(&out, segment.owner_tree_id); - append_scalar(&out, segment.object_id); - append_scalar(&out, segment.first_ordinal); - append_scalar(&out, segment.ref_count); - append_scalar(&out, segment.checksum); - append_scalar(&out, static_cast(segment.reused)); - } - append_scalar(&out, static_cast(manifest.packs.size())); - for (const auto &pack : manifest.packs) { - append_scalar(&out, pack.owner_tree_id); - append_scalar(&out, pack.ordinal); - append_scalar(&out, pack.logical_offset); - append_scalar(&out, pack.ref.chunk_id.high); - append_scalar(&out, pack.ref.chunk_id.low); - append_scalar(&out, pack.ref.offset); - append_scalar(&out, pack.ref.length); - append_scalar(&out, static_cast(pack.reused)); - } - return out; -} - -std::shared_ptr decode_manifest(const std::vector &bytes) -{ - BlobReader reader(bytes); - auto manifest = std::make_shared(); - uint64_t reference_count = 0; - if (!reader.scalar(&manifest->format_version) || !reader.scalar(&manifest->tree_id) || - !reader.scalar(&manifest->generation) || !reader.scalar(&manifest->owner_epoch) || - !reader.scalar(&manifest->logical_size) || !reader.scalar(&manifest->published_at_ms) || - !reader.scalar(&manifest->packs_reused) || !reader.scalar(&manifest->pack_bytes_reused) || - (manifest->format_version >= 4 && !reader.scalar(&manifest->wal_replay_offset)) || - !reader.scalar(&manifest->checksum) || !reader.scalar(&reference_count) || - reference_count > reader.remaining() / 33U) { - return nullptr; - } - manifest->reference_segments.reserve(static_cast(reference_count)); - for (uint64_t i = 0; i < reference_count; ++i) { - ChunkReferenceSegment segment; - uint8_t reused = 0; - if (!reader.scalar(&segment.owner_tree_id) || !reader.scalar(&segment.object_id) || - !reader.scalar(&segment.first_ordinal) || !reader.scalar(&segment.ref_count) || - !reader.scalar(&segment.checksum) || !reader.scalar(&reused) || reused > 1) { - return nullptr; - } - segment.reused = reused != 0; - manifest->reference_segments.push_back(segment); - } - uint64_t pack_count = 0; - if (!reader.scalar(&pack_count) || pack_count > reader.remaining() / 53U) { - return nullptr; - } - manifest->packs.reserve(static_cast(pack_count)); - for (uint64_t i = 0; i < pack_count; ++i) { - ChunkPagePack pack; - uint8_t reused = 0; - if (!reader.scalar(&pack.owner_tree_id) || !reader.scalar(&pack.ordinal) || - !reader.scalar(&pack.logical_offset) || !reader.scalar(&pack.ref.chunk_id.high) || - !reader.scalar(&pack.ref.chunk_id.low) || !reader.scalar(&pack.ref.offset) || - !reader.scalar(&pack.ref.length) || !reader.scalar(&reused) || reused > 1) { - return nullptr; - } - pack.reused = reused != 0; - manifest->packs.push_back(pack); - } - return reader.complete() ? manifest : nullptr; -} - struct CallbackBlob { ct_status status = 0; @@ -303,11 +130,6 @@ struct ReferenceSegmentIdentityHash } }; -uint64_t reference_segment_owner(const ChunkManifest &manifest, const ChunkReferenceSegment &segment) -{ - return manifest.format_version == 0 ? manifest.tree_id : segment.owner_tree_id; -} - PackIdentity pack_identity(const ChunkPagePack &pack) { return { @@ -497,7 +319,7 @@ void MemoryRootCatalog::corrupt_active_reference_segment(size_t segment_index, s return; } const ChunkReferenceSegment &descriptor = manifest->reference_segments[segment_index]; - const uint64_t owner_tree_id = reference_segment_owner(*manifest, descriptor); + const uint64_t owner_tree_id = descriptor.owner_tree_id; const uint64_t object_id = descriptor.object_id; auto current = reference_segment_store_.load(std::memory_order_acquire); for (;;) { @@ -533,10 +355,10 @@ void MemoryRootCatalog::corrupt_active_pack_layout_for_tests(uint64_t tree_id) if (found == state->current.end() || (*found)->packs.empty()) { return; } - auto corrupt = std::make_shared(**found); - corrupt->packs.front().logical_offset = std::numeric_limits::max(); - corrupt->checksum = ChunkPageStore::manifest_checksum(*corrupt); - auto next = std::make_shared(*state); + auto corrupt = std::make_shared(**found); + corrupt->packs.front().logical_offset = std::numeric_limits::max(); + corrupt->checksum = manifest_checksum(*corrupt); + auto next = std::make_shared(*state); next->current[static_cast(found - state->current.begin())] = std::move(corrupt); if (state_.compare_exchange_weak(state, next, std::memory_order_release, std::memory_order_acquire)) { return; @@ -544,7 +366,7 @@ void MemoryRootCatalog::corrupt_active_pack_layout_for_tests(uint64_t tree_id) } } -void MemoryRootCatalog::downgrade_active_manifest_for_tests(uint64_t tree_id) +void MemoryRootCatalog::corrupt_active_manifest_version_for_tests(uint64_t tree_id) { auto state = state_.load(std::memory_order_acquire); for (;;) { @@ -556,15 +378,11 @@ void MemoryRootCatalog::downgrade_active_manifest_for_tests(uint64_t tree_id) if (found == state->current.end()) { return; } - auto legacy = std::make_shared(**found); - legacy->format_version = 0; - for (ChunkReferenceSegment &segment : legacy->reference_segments) { - segment.owner_tree_id = 0; - segment.reused = false; - } - legacy->checksum = ChunkPageStore::manifest_checksum(*legacy); + auto corrupt = std::make_shared(**found); + corrupt->format_version = 0; + corrupt->checksum = manifest_checksum(*corrupt); auto next = std::make_shared(*state); - next->current[static_cast(found - state->current.begin())] = std::move(legacy); + next->current[static_cast(found - state->current.begin())] = std::move(corrupt); if (state_.compare_exchange_weak(state, next, std::memory_order_release, std::memory_order_acquire)) { return; } @@ -707,8 +525,7 @@ uint64_t MemoryRootCatalog::reclaim_before(uint64_t tree_id, uint64_t generation candidate_packs.insert(pack_identity(pack)); } for (const ChunkReferenceSegment &segment : manifest->reference_segments) { - candidate_segments.push_back( - {.owner_tree_id = reference_segment_owner(*manifest, segment), .object_id = segment.object_id}); + candidate_segments.push_back({.owner_tree_id = segment.owner_tree_id, .object_id = segment.object_id}); } } uint64_t reclaimed = 0; @@ -722,8 +539,7 @@ uint64_t MemoryRootCatalog::reclaim_before(uint64_t tree_id, uint64_t generation live_packs.insert(pack_identity(pack)); } for (const ChunkReferenceSegment &segment : manifest->reference_segments) { - live_segments.insert( - {.owner_tree_id = reference_segment_owner(*manifest, segment), .object_id = segment.object_id}); + live_segments.insert({.owner_tree_id = segment.owner_tree_id, .object_id = segment.object_id}); } } }; @@ -830,7 +646,7 @@ ChunkPageStore::ChunkPageStore(Config config, std::shared_ptr catal transport_(std::move(transport)) { if (config_.pack_bytes == 0) { - config_.pack_bytes = crowdb::protocol::kMaxFramePayloadBytes; + config_.pack_bytes = crowdb::protocol::kMaxFrameV2PayloadBytes; } if (config_.iu_size == 0) { config_.iu_size = 64U * 1024U; @@ -857,7 +673,7 @@ ChunkPageStore::ChunkPageStore(Config config, std::shared_ptr catal transport_ = std::make_shared(); } if (config_.open_generation == 0) { - if (auto current = catalog_->load(config_.tree_id); current != nullptr && current->format_version >= 4) { + if (auto current = catalog_->load(config_.tree_id); current != nullptr) { wal_replay_offset_.store(current->wal_replay_offset, std::memory_order_relaxed); } } @@ -867,9 +683,7 @@ ChunkPageStore::ChunkPageStore(Config config, std::shared_ptr catal cached_layout_.store(opened, std::memory_order_release); layout_valid_until_ms_.store(monotonic_millis() + config_.layout_validity_ms, std::memory_order_release); bootstrap_layout_.store(opened, std::memory_order_release); - if (opened->format_version >= 4) { - wal_replay_offset_.store(opened->wal_replay_offset, std::memory_order_relaxed); - } + wal_replay_offset_.store(opened->wal_replay_offset, std::memory_order_relaxed); } } else { @@ -877,7 +691,7 @@ ChunkPageStore::ChunkPageStore(Config config, std::shared_ptr catal } config_.max_chunk_bytes = std::min(config_.max_chunk_bytes, kMaxChunkBytes); config_.pack_bytes = std::min(config_.pack_bytes, config_.max_chunk_bytes); - const uint64_t physical_pack_bytes = crowdb::protocol::framed_physical_length(config_.pack_bytes); + const uint64_t physical_pack_bytes = chunk_framed_length(config_.pack_bytes); if (physical_pack_bytes != 0) { const uint64_t packs_per_chunk = std::max(1, config_.max_chunk_bytes / physical_pack_bytes); config_.max_chunk_bytes = packs_per_chunk * config_.pack_bytes; @@ -1018,9 +832,8 @@ bool ChunkPageStore::has_shared_ownership() const if (manifest == nullptr) { return false; } - return std::any_of(manifest->packs.begin(), manifest->packs.end(), [this, &manifest](const ChunkPagePack &pack) { - return chunk_pack_owner(*manifest, pack) != config_.tree_id; - }); + return std::any_of(manifest->packs.begin(), manifest->packs.end(), + [this, &manifest](const ChunkPagePack &pack) { return pack.owner_tree_id != config_.tree_id; }); } bool ChunkPageStore::inherited_snapshot_matches(const PageStore &source_store) const @@ -1095,7 +908,7 @@ Status ChunkPageStore::validate_manifest(const ChunkManifest &manifest, const Ro if (segment.first_ordinal != first || segment.ref_count != count) { return Status::corruption("chunk reference segment directory coverage is invalid"); } - auto image = catalog.load_reference_segment(reference_segment_owner(manifest, segment), segment.object_id); + auto image = catalog.load_reference_segment(segment.owner_tree_id, segment.object_id); if (image == nullptr || image->object_id != segment.object_id || image->first_ordinal != first || image->refs.size() != count || reference_segment_checksum(*image) != segment.checksum) { return Status::corruption("chunk reference segment image is missing or corrupt"); @@ -1109,7 +922,7 @@ Status ChunkPageStore::validate_manifest(const ChunkManifest &manifest, const Ro if (pack.ordinal != ordinal || bad_layout || pack.ref.chunk_id.empty() || pack.ref.length == 0 || pack.logical_offset > manifest.logical_size || pack.ref.length > manifest.logical_size - pack.logical_offset || ref.chunk_id != pack.ref.chunk_id || - ref.offset != pack.ref.offset || ref.length != pack.ref.length) { + ref.offset != pack.ref.offset || ref.length != pack.ref.length || ref.length > kMaxChunkPackBytes) { return Status::corruption("chunk manifest page-pack coverage or reference is invalid"); } logical_offset = pack.logical_offset + pack.ref.length; @@ -1119,8 +932,7 @@ Status ChunkPageStore::validate_manifest(const ChunkManifest &manifest, const Ro } } } - if ((manifest.format_version < 3 && logical_offset != manifest.logical_size) || - logical_offset > manifest.logical_size || reused_packs != manifest.packs_reused || + if (logical_offset > manifest.logical_size || reused_packs != manifest.packs_reused || reused_pack_bytes != manifest.pack_bytes_reused) { return Status::corruption("chunk manifest coverage or reuse counters are inconsistent"); } @@ -1164,65 +976,6 @@ Status ChunkPageStore::read_at(uint64_t off, uint8_t *buf, size_t len) const return read_at_cancellable(off, buf, len, {}); } -Status ChunkPageStore::read_pack(const ChunkPageRef &ref, std::shared_ptr> *out, - ChunkCancellation cancellation) const -{ - if (cancellation.cancelled()) { - return Status::unavailable("chunk page read cancelled"); - } - const bool use_cache = cancellation.cancelled_id != nullptr; - auto cached = use_cache ? cached_pack_ : nullptr; - if (cached != nullptr && cached->ref.chunk_id == ref.chunk_id && cached->ref.offset == ref.offset && - cached->ref.length == ref.length) { - if (cancellation.cancelled()) { - return Status::unavailable("chunk page read cancelled"); - } - *out = cached->bytes; - coalesced_reads_.fetch_add(1, std::memory_order_relaxed); - coalesced_read_bytes_.fetch_add(ref.length, std::memory_order_relaxed); - return Status::Ok(); - } - pack_reads_.fetch_add(1, std::memory_order_relaxed); - const size_t physical_length = crowdb::protocol::framed_physical_length(ref.length); - if (physical_length == 0) { - return Status::corruption("chunk page frame length overflows"); - } - auto framed = std::make_shared>(physical_length); - bool mirror_responded = false; - for (uint32_t mirror = 0; mirror < config_.mirror_copies; ++mirror) { - if (cancellation.cancelled()) { - return Status::unavailable("chunk page read cancelled"); - } - const uint64_t started = monotonic_nanos(); - Status read_status = transport_->read_mirror(ref.chunk_id, mirror, ref.offset, framed->data(), framed->size()); - diskio_operations_.fetch_add(1, std::memory_order_relaxed); - diskio_latency_ns_.fetch_add(monotonic_nanos() - started, std::memory_order_relaxed); - if (!read_status.ok()) { - CRB_LOG_ERROR("chunk page store tree={} mirror={} read failed: chunk={}:{} offset={} length={} status={}", - config_.tree_id, mirror, ref.chunk_id.high, ref.chunk_id.low, ref.offset, physical_length, - read_status.to_string()); - continue; - } - mirror_responded = true; - auto payload = std::make_shared>(); - const auto frame_error = - crowdb::protocol::parse_frames(*framed, {.high = ref.chunk_id.high, .low = ref.chunk_id.low}, - page_frame_magic(chunk_purpose(ref.chunk_id)), ref.length, payload.get()); - if (frame_error == crowdb::protocol::FrameError::Ok) { - auto entry = std::make_shared(CachedPack{.ref = ref, .bytes = payload}); - if (use_cache) { - cached_pack_ = std::move(entry); - } - *out = std::move(payload); - return cancellation.cancelled() ? Status::unavailable("chunk page read cancelled") : Status::Ok(); - } - CRB_LOG_ERROR("chunk page store tree={} mirror={} frame verification failed: chunk={}:{} offset={} length={}", - config_.tree_id, mirror, ref.chunk_id.high, ref.chunk_id.low, ref.offset, physical_length); - } - return mirror_responded ? Status::corruption("chunk page frame verification failed on every mirror") - : Status::unavailable("chunk page pack mirrors are unavailable"); -} - Status ChunkPageStore::read_at_cancellable(uint64_t off, uint8_t *buf, size_t len, ChunkCancellation cancellation) const { if (unavailable_.load(std::memory_order_acquire)) { @@ -1324,67 +1077,6 @@ Status ChunkPageStore::publication_base(std::shared_ptr *ou return Status::Ok(); } -uint32_t ChunkPageStore::reference_segment_checksum(const ChunkReferenceSegmentImage &segment) -{ - uint32_t crc = update_u64(0, segment.object_id); - crc = update_u64(crc, segment.first_ordinal); - for (const ChunkPageRef &ref : segment.refs) { - crc = update_u64(crc, ref.chunk_id.high); - crc = update_u64(crc, ref.chunk_id.low); - crc = update_u64(crc, ref.offset); - crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&ref.length), sizeof(ref.length)); - } - return crc; -} - -uint32_t ChunkPageStore::manifest_checksum(const ChunkManifest &manifest) -{ - uint32_t crc = 0; - if (manifest.format_version != 0) { - crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&manifest.format_version), - sizeof(manifest.format_version)); - } - crc = update_u64(crc, manifest.tree_id); - crc = update_u64(crc, manifest.generation); - crc = update_u64(crc, manifest.owner_epoch); - crc = update_u64(crc, manifest.logical_size); - crc = update_u64(crc, manifest.published_at_ms); - crc = update_u64(crc, manifest.packs_reused); - crc = update_u64(crc, manifest.pack_bytes_reused); - if (manifest.format_version >= 4) { - crc = update_u64(crc, manifest.wal_replay_offset); - } - for (const ChunkPagePack &pack : manifest.packs) { - if (manifest.format_version >= 2) { - crc = update_u64(crc, pack.owner_tree_id); - } - crc = update_u64(crc, pack.ordinal); - crc = update_u64(crc, pack.logical_offset); - crc = update_u64(crc, pack.ref.chunk_id.high); - crc = update_u64(crc, pack.ref.chunk_id.low); - crc = update_u64(crc, pack.ref.offset); - crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&pack.ref.length), - sizeof(pack.ref.length)); - crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&pack.reused), sizeof(pack.reused)); - } - for (const ChunkReferenceSegment &segment : manifest.reference_segments) { - if (manifest.format_version != 0) { - crc = update_u64(crc, reference_segment_owner(manifest, segment)); - } - crc = update_u64(crc, segment.object_id); - crc = update_u64(crc, segment.first_ordinal); - crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&segment.ref_count), - sizeof(segment.ref_count)); - crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&segment.checksum), - sizeof(segment.checksum)); - if (manifest.format_version != 0) { - crc = crowdb::common::crc32c_update(crc, reinterpret_cast(&segment.reused), - sizeof(segment.reused)); - } - } - return crc; -} - Status ChunkPageStore::persist_reference_segments(ChunkManifest *manifest, const ChunkManifest *reuse_base) { for (size_t first = 0; first < manifest->packs.size(); first += kReferencesPerSegment) { @@ -1403,7 +1095,7 @@ Status ChunkPageStore::persist_reference_segments(ChunkManifest *manifest, const }); if (same) { ChunkReferenceSegment shared = candidate; - shared.owner_tree_id = reference_segment_owner(*reuse_base, candidate); + shared.owner_tree_id = candidate.owner_tree_id; shared.reused = true; manifest->reference_segments.push_back(shared); continue; @@ -1477,7 +1169,7 @@ Status ChunkPageStore::build_manifest(uint64_t expected_generation, std::shared_ } const size_t length = staged_pack_length(offset, reuse_base.get()); const auto purpose = staged_purposes_.at(offset); - if (reuse_base != nullptr && reuse_base->format_version >= 3 && !range_has_pack(*reuse_base, offset, length) && + if (reuse_base != nullptr && !range_has_pack(*reuse_base, offset, length) && !range_was_written(offset, length)) { offset += length; continue; @@ -1487,7 +1179,7 @@ Status ChunkPageStore::build_manifest(uint64_t expected_generation, std::shared_ find_reusable_pack(*reuse_base, offset, static_cast(length), cancellation); if (reused != nullptr && chunk_purpose(reused->ref.chunk_id) == purpose) { manifest->packs.push_back({ - .owner_tree_id = chunk_pack_owner(*reuse_base, *reused), + .owner_tree_id = reused->owner_tree_id, .ordinal = manifest->packs.size(), .logical_offset = offset, .ref = reused->ref, @@ -1501,7 +1193,7 @@ Status ChunkPageStore::build_manifest(uint64_t expected_generation, std::shared_ } if (!active_chunk_id_.empty() && (chunk_purpose(active_chunk_id_) != purpose || length > config_.max_chunk_bytes - active_chunk_bytes_ || - crowdb::protocol::framed_physical_length(length) > chunk_allocation_bytes() - active_chunk_cursor_)) { + chunk_framed_length(length) > chunk_allocation_bytes() - active_chunk_cursor_)) { Status seal_status = transport_->seal_chunk(active_chunk_id_, config_.owner_epoch, active_chunk_cursor_); if (!seal_status.ok()) { return seal_status; @@ -1512,7 +1204,7 @@ Status ChunkPageStore::build_manifest(uint64_t expected_generation, std::shared_ } if (active_chunk_id_.empty()) { const uint64_t packs_per_chunk = (config_.max_chunk_bytes + config_.pack_bytes - 1) / config_.pack_bytes; - const uint64_t physical_pack_bytes = crowdb::protocol::framed_physical_length(config_.pack_bytes); + const uint64_t physical_pack_bytes = chunk_framed_length(config_.pack_bytes); if (physical_pack_bytes == 0) { return Status::resource_exhausted("chunk page framing exceeds address space"); } @@ -1533,10 +1225,10 @@ Status ChunkPageStore::build_manifest(uint64_t expected_generation, std::shared_ pack.ref.chunk_id = active_chunk_id_; pack.ref.offset = active_chunk_cursor_; pack.ref.length = static_cast(length); - const size_t physical_length = crowdb::protocol::framed_physical_length(length); + const size_t physical_length = chunk_framed_length(length); std::vector framed; if (physical_length == 0 || - crowdb::protocol::encode_frames(page_frame_magic(purpose), + crowdb::protocol::encode_frames(chunk_page_frame_magic(purpose), {.high = active_chunk_id_.high, .low = active_chunk_id_.low}, std::span(staged_.data() + offset, length), monotonic_millis(), &framed) != crowdb::protocol::FrameError::Ok) { @@ -1670,12 +1362,12 @@ Status ChunkPageStore::materialize_ownership(uint64_t *bytes_written, bool *comp removed_packs = true; continue; } - const bool shared = chunk_pack_owner(*base, base_pack) != config_.tree_id; + const bool shared = base_pack.owner_tree_id != config_.tree_id; const bool within_budget = base_pack.ref.length <= config_.materialization_bytes_per_pass - copied_bytes; if (!shared || !within_budget) { ChunkPagePack reused = base_pack; reused.ordinal = manifest->packs.size(); - reused.owner_tree_id = chunk_pack_owner(*base, base_pack); + reused.owner_tree_id = base_pack.owner_tree_id; reused.reused = true; manifest->packs.push_back(reused); ++manifest->packs_reused; @@ -1691,7 +1383,7 @@ Status ChunkPageStore::materialize_ownership(uint64_t *bytes_written, bool *comp return fail(std::move(read_status)); } const auto purpose = chunk_purpose(base_pack.ref.chunk_id); - const size_t physical_length = crowdb::protocol::framed_physical_length(bytes->size()); + const size_t physical_length = chunk_framed_length(bytes->size()); if (!active_chunk_id_.empty() && (chunk_purpose(active_chunk_id_) != purpose || base_pack.ref.length > config_.max_chunk_bytes - active_chunk_bytes_ || physical_length > chunk_allocation_bytes() - active_chunk_cursor_)) { @@ -1709,7 +1401,7 @@ Status ChunkPageStore::materialize_ownership(uint64_t *bytes_written, bool *comp } if (active_chunk_id_.empty()) { const uint64_t packs_per_chunk = (config_.max_chunk_bytes + config_.pack_bytes - 1) / config_.pack_bytes; - const uint64_t physical_pack_bytes = crowdb::protocol::framed_physical_length(config_.pack_bytes); + const uint64_t physical_pack_bytes = chunk_framed_length(config_.pack_bytes); if (physical_pack_bytes == 0) { orphan_bytes_.fetch_add(copied_bytes, std::memory_order_relaxed); return fail(Status::resource_exhausted("chunk materialization framing exceeds address space")); @@ -1728,7 +1420,7 @@ Status ChunkPageStore::materialize_ownership(uint64_t *bytes_written, bool *comp std::vector framed; if (physical_length == 0 || - crowdb::protocol::encode_frames(page_frame_magic(purpose), + crowdb::protocol::encode_frames(chunk_page_frame_magic(purpose), {.high = active_chunk_id_.high, .low = active_chunk_id_.low}, *bytes, monotonic_millis(), &framed) != crowdb::protocol::FrameError::Ok) { orphan_bytes_.fetch_add(copied_bytes, std::memory_order_relaxed); @@ -2028,7 +1720,7 @@ Status ChunkPageStore::set_wal_replay_offset(uint64_t offset) status.to_string()); return status; } - if (current->format_version >= 4 && offset < current->wal_replay_offset) { + if (offset < current->wal_replay_offset) { return Status::invalid_argument("WAL replay offset cannot regress"); } } @@ -2058,7 +1750,7 @@ Status ChunkPageStore::wal_replay_offset(uint64_t *offset) const if (!status.ok()) { return status; } - *offset = current->format_version >= 4 ? current->wal_replay_offset : 0; + *offset = current->wal_replay_offset; return Status::Ok(); } @@ -2095,14 +1787,11 @@ ChunkPageStoreStats ChunkPageStore::stats() const uint64_t shared_packs = 0; uint64_t shared_metadata_segments = 0; if (manifest != nullptr) { - shared_packs = std::count_if(manifest->packs.begin(), manifest->packs.end(), [&](const ChunkPagePack &pack) { - return chunk_pack_owner(*manifest, pack) != config_.tree_id; - }); - shared_metadata_segments = - std::count_if(manifest->reference_segments.begin(), manifest->reference_segments.end(), - [&](const ChunkReferenceSegment &segment) { - return reference_segment_owner(*manifest, segment) != config_.tree_id; - }); + shared_packs = std::count_if(manifest->packs.begin(), manifest->packs.end(), + [&](const ChunkPagePack &pack) { return pack.owner_tree_id != config_.tree_id; }); + shared_metadata_segments = std::count_if( + manifest->reference_segments.begin(), manifest->reference_segments.end(), + [&](const ChunkReferenceSegment &segment) { return segment.owner_tree_id != config_.tree_id; }); } return { .generations_published = generations_published_.load(std::memory_order_relaxed), @@ -2188,8 +1877,11 @@ Status CallbackRootCatalog::publish(uint64_t tree_id, uint64_t expected_generati return Status::invalid_argument("callback root catalog publish is unavailable"); } const auto bytes = encode_manifest(*manifest); - const auto code = callbacks_.publish(context_, tree_id, expected_generation, owner_epoch, manifest->generation, - bytes.data(), bytes.size()); + if (bytes.empty()) { + return Status::corruption("invalid manifest encoding"); + } + const auto code = callbacks_.publish(context_, tree_id, expected_generation, owner_epoch, manifest->generation, + bytes.data(), bytes.size()); return callback_status(code, "callback root catalog publish failed"); } @@ -2200,8 +1892,11 @@ Status CallbackRootCatalog::persist_reference_segment(uint64_t return Status::invalid_argument("callback root catalog store is unavailable"); } const auto bytes = encode_reference_segment(*segment); - const auto code = callbacks_.store(context_, CT_ROOT_CATALOG_REFERENCE_SEGMENT, tree_id, segment->object_id, - bytes.data(), bytes.size()); + if (bytes.empty()) { + return Status::corruption("invalid reference segment encoding"); + } + const auto code = callbacks_.store(context_, CT_ROOT_CATALOG_REFERENCE_SEGMENT, tree_id, segment->object_id, + bytes.data(), bytes.size()); return callback_status(code, "callback reference segment store failed"); } diff --git a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h index 8fbfb193a..8957f4b34 100644 --- a/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h +++ b/lib/crowdb-tree/src/backend/chunk/chunk_page_store.h @@ -23,7 +23,7 @@ namespace crowdb::tree::detail { -inline constexpr uint32_t kChunkManifestFormat = 6; +inline constexpr uint32_t kChunkManifestFormat = 7; class ChunkAsyncExecutor; class ChunkPackPipeline; @@ -66,7 +66,7 @@ struct ChunkReferenceSegmentImage struct ChunkManifest { - uint32_t format_version = 0; + uint32_t format_version = kChunkManifestFormat; uint64_t tree_id = 0; uint64_t generation = 0; uint64_t owner_epoch = 0; @@ -80,16 +80,6 @@ struct ChunkManifest std::vector packs; }; -inline uint64_t chunk_pack_owner(const ChunkManifest &manifest, const ChunkPagePack &pack) -{ - if (manifest.format_version >= 2) { - return pack.owner_tree_id; - } - // Older manifests did not distinguish same-lineage reuse from inherited - // reuse. Conservatively materialize every reused pack on the next pass. - return pack.reused ? UINT64_MAX : manifest.tree_id; -} - class RootCatalog { public: @@ -153,7 +143,7 @@ class MemoryRootCatalog final : public RootCatalog [[nodiscard]] uint64_t reference_segment_count(uint64_t tree_id) const; void corrupt_active_reference_segment(size_t segment_index, size_t ref_index); void corrupt_active_pack_layout_for_tests(uint64_t tree_id); - void downgrade_active_manifest_for_tests(uint64_t tree_id); + void corrupt_active_manifest_version_for_tests(uint64_t tree_id); void block_next_publish_for_tests(); void wait_for_blocked_publish_for_tests() const; void release_blocked_publish_for_tests(); @@ -191,9 +181,9 @@ class MemoryRootCatalog final : public RootCatalog using ReferenceSegmentStore = std::vector; AtomicSharedPtr reference_segment_store_; - std::atomic block_next_publish_{false}; - mutable std::atomic publish_blocked_{false}; - std::atomic release_publish_{false}; + std::atomic block_next_publish_{false}; + mutable std::atomic publish_blocked_{false}; + std::atomic release_publish_{false}; }; class CallbackRootCatalog final : public RootCatalog @@ -285,7 +275,7 @@ class ChunkPageStore final : public PageStore, public AsyncPageStore uint64_t tree_id = 0; uint64_t owner_epoch = 0; uint64_t open_generation = 0; - size_t pack_bytes = (64U * 1024U) - 34U; + size_t pack_bytes = (64U * 1024U) - 35U; uint64_t max_chunk_bytes = 256U * 1024U * 1024U; uint32_t mirror_copies = 2; uint32_t page_alignment = 64U * 1024U; @@ -395,65 +385,63 @@ class ChunkPageStore final : public PageStore, public AsyncPageStore uint32_t length, ChunkCancellation cancellation = {}) const; [[nodiscard]] static const ChunkPagePack *find_pack_at(const ChunkManifest &base, uint64_t logical_offset, uint32_t length); - static uint32_t reference_segment_checksum(const ChunkReferenceSegmentImage &segment); - static uint32_t manifest_checksum(const ChunkManifest &manifest); static std::vector reference_segment_ids(const ChunkManifest &manifest); [[nodiscard]] bool pack_is_live(const ChunkPagePack &pack) const; [[nodiscard]] bool range_was_written(uint64_t offset, uint64_t length) const; void record_completion_wakeup(); - Config config_; - std::shared_ptr catalog_; - std::shared_ptr transport_; - std::vector staged_; - PagePurposeRanges staged_purposes_; - std::vector> dirty_ranges_; - std::vector> materialization_live_extents_; - bool staged_initialized_ = false; - bool data_durable_ = false; - bool anchor_dirty_ = false; - std::atomic unavailable_{false}; - std::atomic mirror_write_failure_mask_{0}; + Config config_; + std::shared_ptr catalog_; + std::shared_ptr transport_; + std::vector staged_; + PagePurposeRanges staged_purposes_; + std::vector> dirty_ranges_; + std::vector> materialization_live_extents_; + bool staged_initialized_ = false; + bool data_durable_ = false; + bool anchor_dirty_ = false; + std::atomic unavailable_{false}; + std::atomic mirror_write_failure_mask_{0}; mutable AtomicSharedPtr cached_layout_; mutable AtomicSharedPtr bootstrap_layout_; - mutable std::shared_ptr cached_pack_; - mutable std::atomic layout_valid_until_ms_{0}; - std::atomic generations_published_{0}; - std::atomic wal_replay_offset_{0}; - std::atomic packs_written_{0}; - std::atomic pack_bytes_written_{0}; - std::atomic packs_reused_{0}; - std::atomic pack_bytes_reused_{0}; - mutable std::atomic pack_reads_{0}; - mutable std::atomic cache_hits_{0}; - mutable std::atomic layout_queries_{0}; - std::atomic mirror_write_attempts_{0}; - std::atomic mirror_write_failures_{0}; - std::atomic orphan_bytes_{0}; - std::atomic materialization_passes_{0}; - std::atomic materialization_failures_{0}; - std::atomic materialization_packs_written_{0}; - std::atomic materialization_bytes_written_{0}; - mutable std::atomic rpc_operations_{0}; - mutable std::atomic rpc_latency_ns_{0}; - mutable std::atomic diskio_operations_{0}; - mutable std::atomic diskio_latency_ns_{0}; - mutable std::atomic coalesced_reads_{0}; - mutable std::atomic coalesced_read_bytes_{0}; - std::atomic completion_wakeups_{0}; - std::atomic materialization_scan_bytes_{0}; - std::atomic materialized_metadata_segments_{0}; - std::atomic manifest_publication_latency_ns_{0}; - mutable std::atomic recovery_latency_ns_{0}; - mutable std::atomic recovery_recorded_{false}; - std::vector orphan_reference_segments_; - ChunkId active_chunk_id_; - uint64_t active_chunk_bytes_ = 0; - uint64_t active_chunk_cursor_ = 0; - std::shared_ptr inherited_manifest_; - std::shared_ptr inherited_catalog_; - std::unique_ptr async_executor_; - Status open_status_; + mutable std::shared_ptr cached_pack_; + mutable std::atomic layout_valid_until_ms_{0}; + std::atomic generations_published_{0}; + std::atomic wal_replay_offset_{0}; + std::atomic packs_written_{0}; + std::atomic pack_bytes_written_{0}; + std::atomic packs_reused_{0}; + std::atomic pack_bytes_reused_{0}; + mutable std::atomic pack_reads_{0}; + mutable std::atomic cache_hits_{0}; + mutable std::atomic layout_queries_{0}; + std::atomic mirror_write_attempts_{0}; + std::atomic mirror_write_failures_{0}; + std::atomic orphan_bytes_{0}; + std::atomic materialization_passes_{0}; + std::atomic materialization_failures_{0}; + std::atomic materialization_packs_written_{0}; + std::atomic materialization_bytes_written_{0}; + mutable std::atomic rpc_operations_{0}; + mutable std::atomic rpc_latency_ns_{0}; + mutable std::atomic diskio_operations_{0}; + mutable std::atomic diskio_latency_ns_{0}; + mutable std::atomic coalesced_reads_{0}; + mutable std::atomic coalesced_read_bytes_{0}; + std::atomic completion_wakeups_{0}; + std::atomic materialization_scan_bytes_{0}; + std::atomic materialized_metadata_segments_{0}; + std::atomic manifest_publication_latency_ns_{0}; + mutable std::atomic recovery_latency_ns_{0}; + mutable std::atomic recovery_recorded_{false}; + std::vector orphan_reference_segments_; + ChunkId active_chunk_id_; + uint64_t active_chunk_bytes_ = 0; + uint64_t active_chunk_cursor_ = 0; + std::shared_ptr inherited_manifest_; + std::shared_ptr inherited_catalog_; + std::unique_ptr async_executor_; + Status open_status_; }; } // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocation.h b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocation.h new file mode 100644 index 000000000..8f033b356 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_allocation.h @@ -0,0 +1,31 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include + +namespace crowdb::tree::detail +{ + +struct RpcChunkAllocation +{ + uint32_t granularity_kb; + uint32_t strip_count; +}; + +[[nodiscard]] inline RpcChunkAllocation rpc_chunk_allocation(uint64_t capacity, uint32_t strip_bytes, uint32_t copies) +{ + constexpr uint64_t kMiB = 1024U * 1024U; + uint64_t bytes = strip_bytes; + if (bytes == 0 && copies == 1) { + bytes = kMiB; + } + if (bytes != 0) { + return {.granularity_kb = static_cast(bytes / 1024), + .strip_count = static_cast((capacity + bytes - 1) / bytes)}; + } + return {.granularity_kb = static_cast((capacity + 1023) / 1024), .strip_count = 1}; +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_result.h b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_result.h new file mode 100644 index 000000000..e84b13c34 --- /dev/null +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_result.h @@ -0,0 +1,77 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#pragma once + +#include "crowdb-rpc/c_api.h" + +#include +#include + +namespace crowdb::tree::detail +{ + +struct RpcResult +{ + crowdb_rpc_status status = CROWDB_RPC_ERR_CONN_ERROR; + crowdb_rpc_buffer_t control = nullptr; + crowdb_rpc_buffer_t data = nullptr; + + RpcResult() = default; + RpcResult(const RpcResult &) = delete; + RpcResult &operator=(const RpcResult &) = delete; + + RpcResult(RpcResult &&other) noexcept + : status(other.status), + control(std::exchange(other.control, nullptr)), + data(std::exchange(other.data, nullptr)) + { + } + + RpcResult &operator=(RpcResult &&other) noexcept + { + if (this != &other) { + release(); + status = other.status; + control = std::exchange(other.control, nullptr); + data = std::exchange(other.data, nullptr); + } + return *this; + } + + ~RpcResult() + { + release(); + } + + void release() + { + if (control != nullptr) { + crowdb_rpc_buffer_release(control); + control = nullptr; + } + if (data != nullptr) { + crowdb_rpc_buffer_release(data); + data = nullptr; + } + } +}; + +struct RpcCallState +{ + std::atomic done{false}; + RpcResult result; +}; + +inline void rpc_complete(uint64_t /*unused*/, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, + crowdb_rpc_status status, void *context) +{ + auto *state = static_cast(context); + state->result.status = status; + state->result.control = control; + state->result.data = data; + state->done.store(true, std::memory_order_release); + state->done.notify_one(); +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.cpp b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.cpp index 372321bee..67b1f2861 100644 --- a/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.cpp +++ b/lib/crowdb-tree/src/backend/chunk/rpc_chunk_transport.cpp @@ -9,6 +9,8 @@ #include "crowdb-rpc/c_api.h" #include "diskio_generated.h" #include "msg_type_generated.h" +#include "rpc_chunk_allocation.h" +#include "rpc_chunk_result.h" #include "rpc_chunk_route.h" #include @@ -36,58 +38,6 @@ using crowdb::chunkdb::proto::FBChunkState_Sealed; using crowdb::chunkdb::proto::FBStripType_Mirror; using crowdb::rpc::proto::FBInt128; -struct RpcResult -{ - crowdb_rpc_status status = CROWDB_RPC_ERR_CONN_ERROR; - crowdb_rpc_buffer_t control = nullptr; - crowdb_rpc_buffer_t data = nullptr; - - RpcResult() = default; - RpcResult(const RpcResult &) = delete; - RpcResult &operator=(const RpcResult &) = delete; - - RpcResult(RpcResult &&other) noexcept - : status(other.status), - control(std::exchange(other.control, nullptr)), - data(std::exchange(other.data, nullptr)) - { - } - - RpcResult &operator=(RpcResult &&other) noexcept - { - if (this != &other) { - release(); - status = other.status; - control = std::exchange(other.control, nullptr); - data = std::exchange(other.data, nullptr); - } - return *this; - } - - ~RpcResult() - { - release(); - } - - void release() - { - if (control != nullptr) { - crowdb_rpc_buffer_release(control); - control = nullptr; - } - if (data != nullptr) { - crowdb_rpc_buffer_release(data); - data = nullptr; - } - } -}; - -struct RpcCallState -{ - std::atomic done{false}; - RpcResult result; -}; - uint64_t monotonic_nanos() { return std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()) @@ -109,17 +59,6 @@ uint64_t monotonic_millis() inline constexpr uint64_t kLivenessRenewalMs = 12U * 60U * 1000U; inline constexpr uint64_t kLivenessSelfFenceMs = 13U * 60U * 1000U; -void rpc_complete(uint64_t /*unused*/, crowdb_rpc_buffer_t control, crowdb_rpc_buffer_t data, crowdb_rpc_status status, - void *context) -{ - auto *state = static_cast(context); - state->result.status = status; - state->result.control = control; - state->result.data = data; - state->done.store(true, std::memory_order_release); - state->done.notify_one(); -} - Status call_rpc(const ct_chunk_rpc_route &route, uint64_t request_id, uint16_t message_type, const std::vector &control, const uint8_t *data, size_t data_length, RpcResult *out) { @@ -279,8 +218,8 @@ struct RpcChunkTransport::Impl [[nodiscard]] bool valid() const { - return options.mirror_copies <= kMaxMirrorCopies && chunkdb_routes.valid() && - (disk_resolver.valid() || !disk_routes.empty()); + return options.mirror_copies <= kMaxMirrorCopies && options.strip_capacity_bytes % 1024 == 0 && + chunkdb_routes.valid() && (disk_resolver.valid() || !disk_routes.empty()); } template @@ -357,9 +296,6 @@ struct RpcChunkTransport::Impl if (chunk->state() == crowdb::chunkdb::proto::FBChunkState_Deleted) { return Status::unavailable("tree chunk is deleted"); } - if (chunk->strips() == nullptr) { - return Status::corruption("ChunkDB returned malformed tree chunk metadata"); - } RemoteChunk parsed; parsed.layout = { .chunk_id = ChunkId(chunk->id()->high(), chunk->id()->low()), @@ -374,6 +310,13 @@ struct RpcChunkTransport::Impl } parsed.owner_epoch = chunk->writer_epoch(); parsed.modify_ts = chunk->modify_ts(); + if (chunk->strips() == nullptr || chunk->strips()->empty()) { + if (!parsed.layout.sealed || parsed.layout.logical_capacity != 0 || parsed.layout.acknowledged_bytes != 0) { + return Status::corruption("ChunkDB tree chunk has no valid physical layout"); + } + *out = std::move(parsed); + return Status::Ok(); + } for (const auto *wire_strip : *chunk->strips()) { const auto *mirror = wire_strip == nullptr ? nullptr : wire_strip->strip_body_as_FBMirrorStrip(); if (wire_strip == nullptr || wire_strip->strip_type() != FBStripType_Mirror || mirror == nullptr || @@ -396,9 +339,6 @@ struct RpcChunkTransport::Impl } parsed.strips.push_back(strip); } - if (parsed.strips.empty()) { - return Status::corruption("ChunkDB returned an empty tree chunk layout"); - } *out = std::move(parsed); return Status::Ok(); } @@ -532,13 +472,13 @@ struct RpcChunkTransport::Impl return status; } - RpcChunkRouteResolver chunkdb_routes; - RpcChunkRouteResolver disk_resolver; - ct_chunk_rpc_transport_options options; - std::vector disk_routes; - inline static std::atomic request_ids{1}; + RpcChunkRouteResolver chunkdb_routes; + RpcChunkRouteResolver disk_resolver; + ct_chunk_rpc_transport_options options; + std::vector disk_routes; + inline static std::atomic request_ids{1}; mutable AtomicSharedPtr chunks; - std::jthread liveness_thread; + std::jthread liveness_thread; }; struct RpcChunkTransport::Impl::AsyncWrite @@ -686,15 +626,13 @@ Status RpcChunkTransport::allocate_mirror_chunk(uint64_t logical_capacity, uint6 logical_capacity > std::numeric_limits::max()) { return Status::invalid_argument("tree chunk RPC allocation arguments are invalid"); } - constexpr uint64_t kMiB = 1024U * 1024U; - const uint64_t request_id = impl_->next_request_id(); - const bool single_copy = impl_->options.mirror_copies == 1; - const auto granularity_kb = single_copy ? 1024U : static_cast((logical_capacity + 1023) / 1024); - const auto strip_count = single_copy ? static_cast((logical_capacity + kMiB - 1) / kMiB) : 1U; + const uint64_t request_id = impl_->next_request_id(); + const auto geometry = + rpc_chunk_allocation(logical_capacity, impl_->options.strip_capacity_bytes, impl_->options.mirror_copies); flatbuffers::FlatBufferBuilder builder; auto request = crowdb::chunkdb::proto::CreateFBAllocateChunkRequest( - builder, request_id, monotonic_nanos(), nullptr, granularity_kb, strip_count, FBStripType_Mirror, 0, 0, - impl_->options.mirror_copies == 0 ? 2 : impl_->options.mirror_copies, + builder, request_id, monotonic_nanos(), nullptr, geometry.granularity_kb, geometry.strip_count, + FBStripType_Mirror, 0, 0, impl_->options.mirror_copies == 0 ? 2 : impl_->options.mirror_copies, static_cast(purpose), owner_epoch, impl_->options.writer_lease_ms); builder.Finish(request); std::vector control(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); @@ -942,17 +880,15 @@ Status RpcChunkTransport::seal_chunk(ChunkId chunk_id, uint64_t owner_epoch, uin if (!status.ok()) { return status; } - constexpr uint64_t kKiB = 1024; if (chunk.owner_epoch != owner_epoch || chunk.layout.acknowledged_bytes != acknowledged_bytes || - acknowledged_bytes > static_cast(std::numeric_limits::max()) * kKiB) { + acknowledged_bytes > chunk.layout.logical_capacity) { return Status::unavailable("tree chunk RPC seal is fenced by owner epoch or cursor"); } - const auto seal_length_kib = static_cast((acknowledged_bytes + kKiB - 1) / kKiB); - const uint64_t request_id = impl_->next_request_id(); + const uint64_t request_id = impl_->next_request_id(); const FBInt128 id(chunk_id.high, chunk_id.low); flatbuffers::FlatBufferBuilder builder; - auto request = - crowdb::chunkdb::proto::CreateFBSealChunkRequest(builder, request_id, monotonic_nanos(), &id, seal_length_kib); + auto request = crowdb::chunkdb::proto::CreateFBSealChunkRequest(builder, request_id, monotonic_nanos(), &id, + acknowledged_bytes); builder.Finish(request); std::vector control(builder.GetBufferPointer(), builder.GetBufferPointer() + builder.GetSize()); RpcResult result; @@ -970,6 +906,10 @@ Status RpcChunkTransport::seal_chunk(ChunkId chunk_id, uint64_t owner_epoch, uin if (status.ok()) { status = crowdb::tree::detail::RpcChunkTransport::Impl::parse_chunk(response->chunk(), &updated); } + if (status.ok() && (updated.layout.chunk_id != chunk_id || updated.owner_epoch != owner_epoch || + updated.layout.acknowledged_bytes != acknowledged_bytes || !updated.layout.sealed)) { + return Status::corruption("ChunkDB seal response does not match the exact sealed boundary"); + } if (status.ok()) { updated.valid_until_ms = std::numeric_limits::max(); impl_->cache(std::move(updated)); diff --git a/lib/crowdb-tree/tests/common/test_chunk_route_service.h b/lib/crowdb-tree/tests/common/test_chunk_route_service.h index 1ea4efb5b..49a11395e 100644 --- a/lib/crowdb-tree/tests/common/test_chunk_route_service.h +++ b/lib/crowdb-tree/tests/common/test_chunk_route_service.h @@ -10,11 +10,14 @@ #include #include +#include #include namespace crowdb::tree::detail { +enum class TestSealReply : uint8_t { Exact, Active, WrongCursor }; + struct TestChunkRouteService { crowdb_rpc_server_t server = nullptr; @@ -23,6 +26,9 @@ struct TestChunkRouteService std::atomic queries{0}; std::atomic advances{0}; std::atomic seals{0}; + std::atomic sealed_bytes{0}; + std::atomic sealed{false}; + TestSealReply seal_reply = TestSealReply::Exact; std::atomic acknowledged{0}; bool drop_allocation = false; @@ -70,12 +76,18 @@ struct TestChunkRouteService return; } low = flatbuffers::GetRoot(control)->chunk_id()->low(); + sealed_bytes.store(flatbuffers::GetRoot(control)->seal_bytes()); + acknowledged.store(sealed_bytes.load()); + sealed.store(true); seals.fetch_add(1); } const auto code = low % 2 == parity ? FBChunkdbRetCode_Success : FBChunkdbRetCode_NotMyRange; flatbuffers::FlatBufferBuilder builder; - const auto chunk = make_chunk(builder, low, acknowledged.load()); - uint16_t response_type = 0; + const bool is_seal = message == FBMsgType_ESealChunkRequest; + const auto cursor = acknowledged.load() + uint64_t{is_seal && seal_reply == TestSealReply::WrongCursor}; + const auto chunk = + make_chunk(builder, low, cursor, sealed.load() && (!is_seal || seal_reply != TestSealReply::Active)); + uint16_t response_type = 0; if (message == FBMsgType_EAllocateChunkRequest) { builder.Finish(CreateFBAllocateChunkResponse(builder, request_id, 0, code, 0, 0, 0, chunk)); response_type = FBMsgType_EAllocateChunkResponse; @@ -96,12 +108,15 @@ struct TestChunkRouteService builder.GetSize(), nullptr, 0, response_type, request_id)); } - static flatbuffers::Offset make_chunk(flatbuffers::FlatBufferBuilder &builder, - uint64_t low, uint64_t acknowledged_bytes) + static flatbuffers::Offset + make_chunk(flatbuffers::FlatBufferBuilder &builder, uint64_t low, uint64_t acknowledged_bytes, bool is_sealed) { using namespace crowdb::chunkdb::proto; - const crowdb::rpc::proto::FBInt128 id(0x0200'0000'0000'0042ULL, low); - const crowdb::rpc::proto::FBInt128 disk(9, 10); + const crowdb::rpc::proto::FBInt128 id(0x0200'0000'0000'0042ULL, low); + const crowdb::rpc::proto::FBInt128 disk(9, 10); + if (is_sealed && acknowledged_bytes == 0) { + return CreateFBChunk(builder, &id, 3, FBChunkState_Sealed, 1, 2, 0, 0, 0, FBChunkType_BtreePage, 17, 0); + } const std::vector segments{ {disk, id, 0, 1, 2, 4096} }; @@ -109,8 +124,8 @@ struct TestChunkRouteService const auto strip = CreateFBChunkStrip(builder, 0, 0, 256U * 1024U, 256U * 1024U, 1, 0, 0, FBStripType_Mirror, FBStripBody_FBMirrorStrip, mirror.Union()); const auto strips = builder.CreateVector(std::vector>{strip}); - return CreateFBChunk(builder, &id, 3, FBChunkState_Active, 1, 0, 256U * 1024U, 0, strips, FBChunkType_BtreePage, - 17, acknowledged_bytes); + return CreateFBChunk(builder, &id, 3, is_sealed ? FBChunkState_Sealed : FBChunkState_Active, 1, 0, 256U * 1024U, + 0, strips, FBChunkType_BtreePage, 17, acknowledged_bytes); } }; @@ -137,7 +152,8 @@ struct TestChunkRouteFixture static_cast(crowdb_rpc_server_listen(service.server, "127.0.0.1", 0)); for (auto message : {FBMsgType_EAllocateChunkRequest, FBMsgType_EQueryChunkRequest, FBMsgType_EAdvanceChunkWriteRequest, FBMsgType_ESealChunkRequest}) { - crowdb_rpc_server_register_handler(service.server, message, TestChunkRouteService::handle, &service); + crowdb_rpc_server_register_handler(service.server, message, TestChunkRouteService::handle, &service, + nullptr); } crowdb_rpc_server_start(service.server); auto *connection = crowdb_rpc_connect(service.server, "127.0.0.1", crowdb_rpc_server_port(service.server)); diff --git a/lib/crowdb-tree/tests/integration/chunk_frame_revision_test.cpp b/lib/crowdb-tree/tests/integration/chunk_frame_revision_test.cpp new file mode 100644 index 000000000..0a0c19d67 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/chunk_frame_revision_test.cpp @@ -0,0 +1,128 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "backend/chunk/chunk_catalog_codec.h" +#include "backend/chunk/chunk_frame.h" +#include "backend/chunk/chunk_page_store.h" + +#include + +#include +#include +#include + +namespace crowdb::tree::detail +{ + +TEST(ChunkFrameRevision, DefaultPackFitsOnePhysicalFrame) +{ + const ChunkPageStore::Config config; + EXPECT_EQ(config.pack_bytes, crowdb::protocol::kMaxFrameV2PayloadBytes); + EXPECT_EQ(chunk_framed_length(config.pack_bytes), crowdb::protocol::kMaxFrameBytes); +} + +TEST(ChunkFrameRevision, CompactReferenceHasOneCheckedGeometry) +{ + ChunkReferenceSegmentImage segment{ + .object_id = 7, .first_ordinal = 0, .refs = {{.chunk_id = {3, 5}, .offset = 4096, .length = 9000}}}; + const auto bytes = encode_reference_segment(segment); + ASSERT_EQ(bytes.size(), 24 + 28); + EXPECT_EQ(bytes.back(), 0); + auto decoded = decode_reference_segment(bytes); + ASSERT_NE(decoded, nullptr); + ASSERT_EQ(decoded->refs.size(), 1); + EXPECT_EQ(decoded->refs[0].length, 9000); + auto damaged = bytes; + damaged.pop_back(); + EXPECT_EQ(decode_reference_segment(damaged), nullptr); + damaged = bytes; + damaged.back() = 0x80; + EXPECT_EQ(decode_reference_segment(damaged), nullptr); + const uint32_t checksum = reference_segment_checksum(segment); + ++segment.refs[0].length; + EXPECT_NE(reference_segment_checksum(segment), checksum); +} + +TEST(ChunkFrameRevision, ManifestAcceptsOnlyCurrentVersion) +{ + ChunkManifest manifest; + manifest.packs.push_back({ + .ref = {.chunk_id = {3, 5}, .length = 9000} + }); + auto bytes = encode_manifest(manifest); + ASSERT_FALSE(bytes.empty()); + auto decoded = decode_manifest(bytes); + ASSERT_NE(decoded, nullptr); + EXPECT_EQ(decoded->packs[0].ref.length, 9000); + for (const uint8_t version : {0, 6, 8}) { + bytes[0] = version; + EXPECT_EQ(decode_manifest(bytes), nullptr); + manifest.format_version = version; + EXPECT_TRUE(encode_manifest(manifest).empty()); + } +} + +TEST(ChunkFrameRevision, ReopenedCatalogRetainsBytesAndUsesOrdinaryFrames) +{ + auto catalog = std::make_shared(1); + auto transport = std::make_shared(); + ChunkId chunk; + ASSERT_TRUE(transport->allocate_mirror_chunk(16384, 1, &chunk, PagePurpose::kBtreePage).ok()); + const std::vector payload(9000, 0x57); + std::vector framed; + ASSERT_EQ(crowdb::protocol::encode_frames(crowdb::protocol::FrameMagic::BtreePageV2, + {.high = chunk.high, .low = chunk.low}, payload, 42, &framed), + crowdb::protocol::FrameError::Ok); + for (uint32_t mirror = 0; mirror < 3; ++mirror) { + ASSERT_TRUE(transport->write_mirror(chunk, mirror, 0, framed.data(), framed.size()).ok()); + } + ASSERT_TRUE(transport->advance_write(chunk, 0, framed.size()).ok()); + ASSERT_TRUE(transport->seal_chunk(chunk, 1, framed.size()).ok()); + auto segment = std::make_shared(); + segment->object_id = catalog->allocate_reference_segment_id(777); + segment->refs.push_back({.chunk_id = chunk, .length = 9000}); + ASSERT_TRUE(catalog->persist_reference_segment(777, segment).ok()); + auto manifest = std::make_shared(); + manifest->format_version = kChunkManifestFormat; + manifest->tree_id = 777; + manifest->owner_epoch = 1; + manifest->generation = 1; + manifest->logical_size = payload.size(); + manifest->packs.push_back({.owner_tree_id = 777, .ref = segment->refs[0]}); + manifest->reference_segments.push_back({.owner_tree_id = 777, + .object_id = segment->object_id, + .ref_count = 1, + .checksum = reference_segment_checksum(*segment)}); + manifest->checksum = manifest_checksum(*manifest); + ASSERT_TRUE(catalog->publish(777, 0, 1, manifest).ok()); + ChunkPageStore store({.tree_id = 777, .owner_epoch = 1, .pack_bytes = 9000, .iu_size = 1}, catalog, transport); + std::vector actual(payload.size()); + ASSERT_TRUE(store.read_at(0, actual.data(), actual.size()).ok()); + EXPECT_EQ(actual, payload); + const uint8_t value = 0x61; + ASSERT_TRUE(store.write_at(9000, &value, 1).ok()); + ASSERT_TRUE(store.sync().ok()); + ASSERT_TRUE(store.write_at(0, &value, 1).ok()); + ASSERT_TRUE(store.sync().ok()); + auto current = catalog->load(777); + ASSERT_NE(current, nullptr); + EXPECT_EQ(current->format_version, 7); + bool saw_new = false; + for (const auto &pack : current->packs) { + saw_new = true; + std::vector bytes(chunk_framed_length(pack.ref.length)); + ASSERT_TRUE(transport->read_mirror(pack.ref.chunk_id, 0, pack.ref.offset, bytes.data(), bytes.size()).ok()); + crowdb::protocol::ParsedFrame frame; + ASSERT_EQ(crowdb::protocol::parse_frame(bytes, {.high = pack.ref.chunk_id.high, .low = pack.ref.chunk_id.low}, + &frame), + crowdb::protocol::FrameError::Ok); + EXPECT_EQ(frame.header.magic, crowdb::protocol::FrameMagic::BtreePageV2); + EXPECT_EQ(frame.group.role, crowdb::protocol::FrameRole::Ordinary); + } + EXPECT_TRUE(saw_new); + std::vector retained(framed.size()); + ASSERT_TRUE(transport->read_mirror(chunk, 0, 0, retained.data(), retained.size()).ok()); + EXPECT_EQ(retained, framed); +} + +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp b/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp index 15686d3e7..86de65b90 100644 --- a/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp +++ b/lib/crowdb-tree/tests/integration/chunk_page_store_test.cpp @@ -1018,14 +1018,14 @@ TEST(ChunkPageStore, StaleMaterializationCannotReplaceForegroundCheckpoint) EXPECT_GT(materializer.stats().orphan_bytes, 0U); } -TEST(ChunkPageStore, RejectsLegacyManifestWithoutPublicFrameLocations) +TEST(ChunkPageStore, RejectsUnsupportedManifestVersion) { auto catalog = std::make_shared(1); auto transport = std::make_shared(); ChunkPageStore store({.tree_id = 52, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); publish_raw_generation(&store, 6); - catalog->downgrade_active_manifest_for_tests(52); + catalog->corrupt_active_manifest_version_for_tests(52); ChunkPageStore reopened({.tree_id = 52, .owner_epoch = 1, .pack_bytes = 4096, .page_alignment = 1, .iu_size = 1}, catalog, transport); @@ -1333,7 +1333,7 @@ TEST(ChunkPageStore, RotatesWholePacksAndReopenAllocatesFreshChunk) EXPECT_LE(pack.ref.length, config.pack_bytes); ChunkLayout layout; ASSERT_TRUE(transport->query_chunk(pack.ref.chunk_id, &layout).ok()); - EXPECT_LE(pack.ref.offset + pack.ref.length + 34U, layout.logical_capacity); + EXPECT_LE(pack.ref.offset + pack.ref.length + 35U, layout.logical_capacity); if (std::find(chunk_ids.begin(), chunk_ids.end(), pack.ref.chunk_id) == chunk_ids.end()) { chunk_ids.push_back(pack.ref.chunk_id); } @@ -1428,7 +1428,7 @@ TEST(ChunkPageStore, FramesPackTailWithoutAlignmentPadding) EXPECT_EQ(first->packs[0].ref.length, bytes.size()); ChunkLayout layout; ASSERT_TRUE(transport->query_chunk(first->packs[0].ref.chunk_id, &layout).ok()); - EXPECT_EQ(layout.acknowledged_bytes, first->packs[0].ref.length + 34U); + EXPECT_EQ(layout.acknowledged_bytes, first->packs[0].ref.length + 35U); transport->corrupt_mirror(first->packs[0].ref.chunk_id, 0, first->packs[0].ref.length); std::array out{}; @@ -1461,10 +1461,10 @@ TEST(ChunkPageStore, LargePackUsesOneLocationAndMultipleVerifiedFrames) ASSERT_NE(manifest, nullptr); ASSERT_EQ(manifest->packs.size(), 1U); const ChunkPageRef &ref = manifest->packs.front().ref; - EXPECT_GT(ref.length, (64U * 1024U) - 34U); + EXPECT_GT(ref.length, (64U * 1024U) - 35U); ChunkLayout layout; ASSERT_TRUE(transport->query_chunk(ref.chunk_id, &layout).ok()); - EXPECT_EQ(layout.acknowledged_bytes, ref.length + 68U); + EXPECT_EQ(layout.acknowledged_bytes, ref.length + 70U); std::vector read(payload.size()); ASSERT_TRUE(store.read_at(8192, read.data(), read.size()).ok()); diff --git a/lib/crowdb-tree/tests/integration/rpc_chunk_seal_test.cpp b/lib/crowdb-tree/tests/integration/rpc_chunk_seal_test.cpp new file mode 100644 index 000000000..0c1e01235 --- /dev/null +++ b/lib/crowdb-tree/tests/integration/rpc_chunk_seal_test.cpp @@ -0,0 +1,62 @@ +// Copyright 2026-present Gian +// Licensed under the Apache License, Version 2.0. + +#include "backend/chunk/rpc_chunk_transport.h" +#include "common/test_chunk_route_service.h" + +#include + +#include + +namespace crowdb::tree::detail +{ +namespace +{ +TEST(RpcChunkTransport, SealsEmptyAndPartialKibBoundariesWithExactBytes) +{ + for (const uint64_t bytes : {uint64_t{0}, uint64_t{1025}}) { + TestChunkRouteFixture fixture; + const ct_chunk_rpc_disk_route disk{.disk_id_high = 9, .disk_id_low = 10, .route = fixture.routes[0]}; + ct_chunk_rpc_transport_options options{}; + options.disk_routes = &disk; + options.disk_route_count = 1; + options.chunkdb_resolver = fixture.resolver(); + RpcChunkTransport transport(options); + ChunkId allocated; + ASSERT_TRUE( + transport.allocate_mirror_chunk(256U * 1024U * 1024U, 17, &allocated, PagePurpose::kBtreePage).ok()); + if (bytes != 0) { + ASSERT_TRUE(transport.advance_write(allocated, 0, bytes).ok()); + } + ASSERT_TRUE(transport.seal_chunk(allocated, 17, bytes).ok()); + EXPECT_EQ(fixture.services[0].sealed_bytes.load(), bytes); + ChunkLayout sealed; + ASSERT_TRUE(transport.query_chunk(allocated, &sealed).ok()); + EXPECT_TRUE(sealed.sealed); + EXPECT_EQ(sealed.acknowledged_bytes, bytes); + if (bytes == 0) { + EXPECT_EQ(sealed.logical_capacity, 0); + } + } +} + +TEST(RpcChunkTransport, RejectsASealReplyWithoutTheExactSealedBoundary) +{ + for (const auto fault : {TestSealReply::Active, TestSealReply::WrongCursor}) { + TestChunkRouteFixture fixture; + fixture.services[0].seal_reply = fault; + const ct_chunk_rpc_disk_route disk{.disk_id_high = 9, .disk_id_low = 10, .route = fixture.routes[0]}; + ct_chunk_rpc_transport_options options{}; + options.disk_routes = &disk; + options.disk_route_count = 1; + options.chunkdb_resolver = fixture.resolver(); + RpcChunkTransport transport(options); + ChunkId allocated; + ASSERT_TRUE( + transport.allocate_mirror_chunk(256U * 1024U * 1024U, 17, &allocated, PagePurpose::kBtreePage).ok()); + ASSERT_TRUE(transport.advance_write(allocated, 0, 4).ok()); + EXPECT_FALSE(transport.seal_chunk(allocated, 17, 4).ok()); + } +} +} // namespace +} // namespace crowdb::tree::detail diff --git a/lib/crowdb-tree/tests/integration/rpc_chunk_transport_test.cpp b/lib/crowdb-tree/tests/integration/rpc_chunk_transport_test.cpp index f9cfcc7f9..622161d48 100644 --- a/lib/crowdb-tree/tests/integration/rpc_chunk_transport_test.cpp +++ b/lib/crowdb-tree/tests/integration/rpc_chunk_transport_test.cpp @@ -1,6 +1,7 @@ // Copyright 2026-present Gian // Licensed under the Apache License, Version 2.0. +#include "backend/chunk/rpc_chunk_allocation.h" #include "backend/chunk/rpc_chunk_transport.h" #include "chunkdb_generated.h" #include "common/test_chunk_route_service.h" @@ -50,6 +51,19 @@ struct AllocateHandlerState std::atomic read_error{false}; }; +TEST(RpcChunkTransport, ExplicitStripCapacityIsIndependentOfMirrorCopies) +{ + constexpr uint32_t kMiB = 1024U * 1024U; + for (uint32_t copies : {1U, 2U, 3U}) { + const auto geometry = rpc_chunk_allocation((256U * kMiB) - 4096, kMiB, copies); + EXPECT_EQ(geometry.granularity_kb, 1024U); + EXPECT_EQ(geometry.strip_count, 256U); + } + const auto automatic = rpc_chunk_allocation(256U * kMiB, 0, 2); + EXPECT_EQ(automatic.granularity_kb, 256U * 1024U); + EXPECT_EQ(automatic.strip_count, 1U); +} + extern "C" void handle_allocate(uint64_t request_id, uint64_t /*unused*/, uint16_t /*unused*/, const uint8_t *control, uint32_t control_len, const uint8_t * /*unused*/, uint32_t /*unused*/, void *connection, void *frame, void *user_data) @@ -173,9 +187,9 @@ TEST(RpcChunkTransport, AllocatesOneFullMirrorStripAndPreserves128BitChunkId) ASSERT_NE(server, nullptr); ASSERT_EQ(crowdb_rpc_server_listen(server, "127.0.0.1", 0), CROWDB_RPC_OK); AllocateHandlerState handler{.server = server}; - crowdb_rpc_server_register_handler(server, FBMsgType_EAllocateChunkRequest, handle_allocate, &handler); - crowdb_rpc_server_register_handler(server, FBMsgType_EDiskReadRequest, handle_disk_read, &handler); - crowdb_rpc_server_register_handler(server, FBMsgType_EDiskWriteRequest, handle_disk_write, &handler); + crowdb_rpc_server_register_handler(server, FBMsgType_EAllocateChunkRequest, handle_allocate, &handler, nullptr); + crowdb_rpc_server_register_handler(server, FBMsgType_EDiskReadRequest, handle_disk_read, &handler, nullptr); + crowdb_rpc_server_register_handler(server, FBMsgType_EDiskWriteRequest, handle_disk_write, &handler, nullptr); crowdb_rpc_server_start(server); crowdb_rpc_client_t client = crowdb_rpc_client_create(); @@ -191,15 +205,16 @@ TEST(RpcChunkTransport, AllocatesOneFullMirrorStripAndPreserves128BitChunkId) .route = {.client = client, .server = server, .connection = connection}, }; ct_chunk_rpc_transport_options options{ - .chunkdb = {.client = client, .server = server, .connection = connection}, - .disk_routes = &disk_route, - .disk_route_count = 1, - .writer_lease_ms = 5000, - .rpc_timeout_ms = 1000, - .completion_capacity = 16, - .mirror_copies = 3, - .chunkdb_resolver = {}, - .disk_resolver = {}, + .chunkdb = {.client = client, .server = server, .connection = connection}, + .disk_routes = &disk_route, + .disk_route_count = 1, + .writer_lease_ms = 5000, + .rpc_timeout_ms = 1000, + .completion_capacity = 16, + .mirror_copies = 3, + .strip_capacity_bytes = 0, + .chunkdb_resolver = {}, + .disk_resolver = {}, }; RpcChunkTransport transport(options); ChunkId allocated; @@ -258,6 +273,7 @@ TEST(RpcChunkTransport, ResolvesEachChunkOwnerAndReleasesRouteLeases) EXPECT_EQ(fixture.services[0].allocations.load(), 1U); EXPECT_EQ(fixture.services[0].advances.load(), 1U); EXPECT_EQ(fixture.services[0].seals.load(), 1U); + EXPECT_EQ(fixture.services[0].sealed_bytes.load(), 4U); EXPECT_EQ(fixture.services[1].queries.load(), 1U); EXPECT_EQ(fixture.retains.load(), 1U); EXPECT_EQ(fixture.leases.load(), 0U); diff --git a/pixi.toml b/pixi.toml index b74e0de31..4f95c28b4 100644 --- a/pixi.toml +++ b/pixi.toml @@ -74,9 +74,6 @@ boto3 = "==1.43.92" awscli = "==2.36.47" rclone = "==1.75.1" -[feature.s3-e2e.tasks] -test-boto3-copy-error = { cmd = "python tools/pixi-tasks/test.py unit boto3" } - [environments] s3-e2e = { features = ["s3-e2e"], no-default-feature = true } iceberg-e2e = { features = ["iceberg-e2e"], no-default-feature = true } @@ -119,23 +116,45 @@ duckdb = "==1.4.3" openjdk = "17.*" maven = "3.9.*" -[feature.iceberg-ecosystem.tasks] -test-iceberg-container-ecosystem = "bash e2e-test/clients/ecosystem/run.sh" +[pypi-dependencies] +maturin = ">=1.15, <2" +pip = ">=25, <27" [tasks] -image-build = "bash container/oci-image/image-build.sh" -image-inspect = "pixi run --manifest-path container/oci-image/pixi.toml inspect" -image-import = "pixi run --manifest-path container/oci-image/pixi.toml import" -image-ctl = "pixi run --manifest-path container/oci-image/pixi.toml -e runtime control" -image-run = "pixi run --manifest-path container/oci-image/pixi.toml -e runtime run" -test-container-e2e = "python e2e-test/run.py" + +################################################################################ +# Build — install dependencies, clean outputs and compile +################################################################################ +install-deps = { cmd = "bash tools/pixi-tasks/install-deps.sh" } +# Frontend dependencies only; avoids installing profiling tools for UI tests. +install-ui-deps = "test -d app/crowdb-web/ui/node_modules || (cd app/crowdb-web/ui && npm ci)" +# Remove build products and the complete workspace runtime tree. +clean = { cmd = "bash tools/pixi-tasks/clean-build.sh" } +# Stop recorded ephemeral processes and remove disposable runtime state. +clean-env = "bash tools/pixi-tasks/clean-env.sh" +# Standalone C++ libraries for ctest; Rust FFI builds its own C++ libraries. +build-cpp = { cmd = "bash tools/pixi-tasks/build-cpp.sh" } +# Full release build: C++ libraries, Rust binaries and frontend. +build = { cmd = "bash tools/pixi-tasks/build-release.sh", depends-on = ["build-cpp", "install-ui-deps"] } +# Optional compile-only check for all workspace test targets. +build-tests = "cargo test --workspace --all-targets --no-run" +default = { depends-on = ["build"] } + +################################################################################ +# Test — component, SDK and container acceptance suites +################################################################################ test = "python tools/pixi-tasks/test.py" +test-suite = { cmd = "python tools/pixi-tasks/test.py all", depends-on = ["build-cpp", "install-ui-deps"] } +test-cpp = { cmd = "python tools/pixi-tasks/test.py unit cpp", depends-on = ["build-cpp"] } +test-core = { cmd = "python tools/pixi-tasks/test.py unit core" } +test-storage = { cmd = "python tools/pixi-tasks/test.py unit storage" } +test-access = { cmd = "python tools/pixi-tasks/test.py unit access" } +test-console = { cmd = "python tools/pixi-tasks/test.py unit console" } +test-console-ui = { cmd = "python tools/pixi-tasks/test.py unit ui", depends-on = ["install-ui-deps"] } test-access-sdk = "python tools/pixi-tasks/test.py unit sdk" -test-dataset-sdk-python = "python tools/sdk/package.py" test-dataset-sdk = "cargo test --manifest-path sdk/Cargo.toml --workspace --locked" -sdk-fmt-check = "cargo fmt --manifest-path sdk/Cargo.toml --all -- --check" -sdk-lint = "cargo clippy --manifest-path sdk/Cargo.toml --workspace --all-targets --locked -- -D warnings" -check-dataset-sdk-boundary = "python tools/ci-checks/check-dataset-sdk-boundary.py" +test-dataset-sdk-python = "python tools/sdk/package.py" +test-tree-link-isolation = "bash tools/cpp-checks/tree-link-isolation.sh" test-e2e-kv = "python tools/pixi-tasks/test.py e2e kv" test-e2e-diskdb = "python tools/pixi-tasks/test.py e2e diskdb" test-e2e-diskio = "python tools/pixi-tasks/test.py e2e diskio" @@ -148,116 +167,60 @@ test-e2e-iceberg = "python tools/pixi-tasks/test.py e2e iceberg" test-e2e-dataset = "python tools/pixi-tasks/test.py e2e dataset" test-e2e-console = "python tools/pixi-tasks/test.py e2e console" test-e2e-deployment = "python tools/pixi-tasks/test.py e2e deployment" +test-container-e2e = "python e2e-test/run.py" test-container-e2e-fixture = "env PYTHONPATH=e2e-test python -m unittest discover -s e2e-test/tests -p '*_test.py' -v" test-oci-image = "pixi run --manifest-path container/oci-image/pixi.toml test" +test-single-node-container = { cmd = "bash container/single-node-container/tests/release-policy.sh && bash container/single-node-container/tests/image-smoke.sh && bash container/single-node-container/tests/container-e2e.sh", depends-on = ["install-ui-deps"] } -# ── Build ──────────────────────────────────────────────────────── -# C++ libraries only (crowdb-tree, crowdb-rpc, crowdb-diskio). Needed by C++ -# ctest tasks. Rust FFI crates compile their own C++ via cc::Build in -# build.rs, so Rust tests do NOT depend on this. -build-cpp = { cmd = "bash tools/pixi-tasks/build-cpp.sh" } -# npm deps only (for UI tests and ts-lint). Separated from install-deps -# so test-console-ui doesn't trigger cargo installs of profiling tools. -install-ui-deps = "test -d app/crowdb-web/ui/node_modules || (cd app/crowdb-web/ui && npm ci)" -install-deps = { cmd = "bash tools/pixi-tasks/install-deps.sh" } +################################################################################ +# Tools — format, lint, architecture checks and source line counts +################################################################################ tree-fmt = ''' find lib/crowdb-tree/src lib/crowdb-tree/include lib/crowdb-tree/tests lib/crowdb-tree/bench lib/crowdb-common/cpp lib/crowdb-rpc/src lib/crowdb-rpc/include lib/crowdb-rpc/tests -type f \( -name '*.cpp' -o -name '*.h' \) -print0 | xargs -0 clang-format -i ''' -# CPATH lets clang-tidy find the conda env headers (spdlog, gtest, folly, -# absl, flatbuffers, zlib, isa-l, liburing, infiniband, ...) at -# $CONDA_PREFIX/include. The conda gcc has this as a built-in default include -# path, but clang-tidy (a separate compiler) does not, and compile_commands.json -# does not list it explicitly because gcc finds those headers automatically. -# clang treats CPATH entries as system includes, so warnings from third-party -# headers are suppressed. -tree-lint = "CPATH=$CONDA_PREFIX/include python3 tools/cpp-checks/clang-tidy.py" -test-tree-link-isolation = "bash tools/cpp-checks/tree-link-isolation.sh" rs-fmt = '''cargo metadata --format-version 1 --no-deps | jq -r '.workspace_members as $members | .packages[] | select(.id as $package_id | $members | index($package_id)) | .name' | xargs sh -c 'cargo fmt --package "$@"' _''' rs-fmt-check = '''cargo metadata --format-version 1 --no-deps | jq -r '.workspace_members as $members | .packages[] | select(.id as $package_id | $members | index($package_id)) | .name' | xargs sh -c 'cargo fmt --package "$@" -- --check' _''' -check-production-dashmap = "python3 tools/ci-checks/check-production-dashmap.py" -check-version = "python3 tools/ci-checks/check-version.py" +sdk-fmt-check = "cargo fmt --manifest-path sdk/Cargo.toml --all -- --check" +# CPATH supplies conda headers omitted from GCC compile_commands.json. +tree-lint = "CPATH=$CONDA_PREFIX/include python3 tools/cpp-checks/clang-tidy.py" rs-lint = { cmd = "cargo clippy --all-targets -- -D warnings", depends-on = ["check-production-dashmap", "check-version"] } +sdk-lint = "cargo clippy --manifest-path sdk/Cargo.toml --workspace --all-targets --locked -- -D warnings" ts-lint = { cmd = "python tools/pixi-tasks/check-console-types.py", depends-on = ["install-ui-deps"] } -# Full build: C++ libs + release Rust binaries + frontend. Used by -# run/bench/profile. Tests use build-cpp (C++ tests) or nothing (Rust tests). -build = { cmd = "bash tools/pixi-tasks/build-release.sh", depends-on = ["build-cpp", "install-ui-deps"] } -default = { depends-on = ["build"] } -stage-single-node-container = "bash container/single-node-container/build.sh stage" -build-single-node-container = "bash container/single-node-container/build.sh image" -start-container = { cmd = "bash container/single-node-container/dev-container.sh start", depends-on = ["build-single-node-container"] } -inject-container = "bash container/single-node-container/dev-container.sh inject" -clean-container = "bash container/single-node-container/dev-container.sh clean" -test-single-node-container = { cmd = "bash container/single-node-container/tests/release-policy.sh && bash container/single-node-container/tests/image-smoke.sh && bash container/single-node-container/tests/container-e2e.sh", depends-on = ["build-single-node-container", "install-ui-deps"] } - -# ── Run ────────────────────────────────────────────────────────── -web = { cmd = "echo 'CROWDB console starting at: http://127.0.0.1:9090/' && cargo run --release -p crowdb-web", depends-on = ["build"] } - -# ── Benchmark ──────────────────────────────────────────────────── -bench = { cmd = "cargo run --release -p crowdb-cli -- bench kv --mode mem --duration-secs 12 --loader-num 2 --connections 1 --workload write", depends-on = ["build"] } -bench-mem = { cmd = "cargo run --release -p crowdb-cli -- bench kv --mode mem --duration-secs 30", depends-on = ["build"] } -bench-file = { cmd = "cargo run --release -p crowdb-cli -- bench kv --mode file --duration-secs 30", depends-on = ["build"] } -bench-block = { cmd = "cargo run --release -p crowdb-cli -- bench kv --mode block --duration-secs 30", depends-on = ["build"] } -bench-burst = { cmd = "cargo run --release -p crowdb-cli -- bench kv --mode mem --duration-secs 10 --loader-num 64 --connections 8 --key-space 10000 --value-size 256 --workload mix", depends-on = ["build"] } -bench-soak = { cmd = "cargo run --release -p crowdb-cli -- bench kv --mode mem --duration-secs 30 --loader-num 8 --connections 4 --key-space 100000 --value-size 1024 --workload write", depends-on = ["build"] } -bench-hotread = { cmd = "cargo run --release -p crowdb-cli -- bench kv --mode mem --duration-secs 15 --loader-num 32 --connections 4 --key-space 64 --value-size 64 --workload read", depends-on = ["build"] } -bench-compare = { cmd = "cargo run --release -p crowdb-cli -- bench compare", depends-on = ["build"] } - -# ── Profile ─────────────────────────────────────────────────────── -profile-write = { cmd = "bash tools/profiling/profile-write.sh", depends-on = ["build"] } - -# ── Test ───────────────────────────────────────────────────────── -# Tasks are grouped by CI job (see doc/working/test.md § "Current CI -# Test Design"). Each group maps 1:1 to a parallel CI job. Run -# Each test group builds only its own Rust targets. C++ ctest tasks need -# `pixi run build-cpp`. -clean-env = "bash tools/pixi-tasks/clean-env.sh" -# Optional full-workspace compile check. CI test groups use their component -# tasks to avoid building unrelated test targets on each runner. -build-tests = "cargo test --workspace --all-targets --no-run" -# Verify every workspace package is assigned to a test task reachable from CI. +check-production-dashmap = "python3 tools/ci-checks/check-production-dashmap.py" +check-version = "python3 tools/ci-checks/check-version.py" +check-dataset-sdk-boundary = "python tools/ci-checks/check-dataset-sdk-boundary.py" check-ci-test-tasks = "python tools/ci-checks/check-ci-test-tasks.py" - -# ── Lint job ── -# (uses rs-fmt / rs-lint / tree-lint defined above, no separate test-* tasks) - -test-cpp = { cmd = "python tools/pixi-tasks/test.py unit cpp", depends-on = ["build-cpp"] } - -# ── Component tests ───────────────────────────────────────────── -test-core = { cmd = "python tools/pixi-tasks/test.py unit core" } -test-storage = { cmd = "python tools/pixi-tasks/test.py unit storage" } -test-access = { cmd = "python tools/pixi-tasks/test.py unit access" } - -# ── ConsoleTests job: deterministic component tests ── -test-console = { cmd = "python tools/pixi-tasks/test.py unit console" } - -# ── UITests job: deterministic UI component tests ── -test-console-ui = { cmd = "python tools/pixi-tasks/test.py unit ui", depends-on = ["install-ui-deps"] } - -# ── Full local suite (not a CI job; runs everything in one invocation) ── -# Grouped by CI job order. Each group task owns its component list; -# Component tasks run first; one root container aggregate follows. -test-suite = { cmd = "python tools/pixi-tasks/test.py all", depends-on = ["build-cpp", "install-ui-deps"] } - -# ── Coverage & LOC ─────────────────────────────────────────────── -coverage = { cmd = "cargo tarpaulin --workspace --out Html --output-dir target/coverage --exclude-files '*/tests/*'", depends-on = ["install-deps"] } loc = "tokei . .github --exclude swagger-ui --exclude third-party --exclude 'lib/crowdb-tree/build-*' --exclude target --exclude dist --exclude node_modules --exclude package-lock.json" -# ── Clean ──────────────────────────────────────────────────────── -# Removes build products and the complete workspace runtime tree. -clean = { cmd = "bash tools/pixi-tasks/clean-build.sh" } - -# ── lib/crowdb-tree sanitizers ────────────────────────────────────────── - -# AddressSanitizer build + tests. setarch -R disables ASLR (Linux only); -# on macOS we run ctest directly. +################################################################################ +# Tools — sanitizers, coverage and profiling +################################################################################ tree-asan = { cmd = "bash tools/pixi-tasks/tree-asan.sh" } - -# ThreadSanitizer build + tests. tree-tsan = { cmd = "bash tools/pixi-tasks/tree-tsan.sh" } - -# UndefinedBehaviorSanitizer build + tests. tree-ubsan = { cmd = "bash tools/pixi-tasks/tree-ubsan.sh" } +coverage = { cmd = "cargo tarpaulin --workspace --out Html --output-dir target/coverage --exclude-files '*/tests/*'", depends-on = ["install-deps"] } +profile-write = { cmd = "bash tools/profiling/profile-write.sh", depends-on = ["build"] } -[pypi-dependencies] -maturin = ">=1.15, <2" -pip = ">=25, <27" +################################################################################ +# OCI containers — build once, run isolated clusters, stop or clean owned resources +################################################################################ +# Build an OCI archive. Add --load to load crowdb-node:dev into local Docker. +oci-build = "bash container/oci-image/image-build.sh" +# Run uses the existing image; monitors discover peers and prepare the cluster. +oci-run = "python container/oci-image/docker-local.py run" +oci-run-1c = "python container/oci-image/docker-local.py run --nodes 1" +oci-run-3c = "python container/oci-image/docker-local.py run --nodes 3" +# Scope defaults to --cluster local. Global scope requires --all explicitly. +oci-stop = "python container/oci-image/docker-local.py stop" +# Remove owned containers and bridges; preserve data unless --delete-data is set. +oci-clean = "python container/oci-image/docker-local.py clean" +oci-host-clean = "python container/oci-image/host-clean.py" + +################################################################################ +# Test environments — isolated client dependencies; run with pixi run -e +################################################################################ +[feature.s3-e2e.tasks] +test-boto3-copy-error = { cmd = "python tools/pixi-tasks/test.py unit boto3" } + +[feature.iceberg-ecosystem.tasks] +test-iceberg-container-ecosystem = "bash e2e-test/clients/ecosystem/run.sh" diff --git a/tools/README.md b/tools/README.md index ac7cc738a..8214f28ca 100644 --- a/tools/README.md +++ b/tools/README.md @@ -10,9 +10,9 @@ Run commands through `pixi run`. Read only the directory relevant to the task. `unit GROUP`, `e2e DOMAIN`, `profile NAME`, or `all`. Container lifecycle remains owned by `e2e-test/run.py`. Component runs retain phase logs and wall times under `.crowdb-runtime/artifacts/unit/`. - `clean` stops all CROWDB services owned by the current user, then removes - the complete runtime tree and build output. `clean-env` stops only recorded - ephemeral processes and preserves persistent clusters. + `clean-env` stops only recorded ephemeral processes and preserves persistent + clusters. `oci-clean --cluster NAME` removes owned containers and unused + bridges; data deletion requires explicit `--delete-data`. - **`ci-checks/`** — version consistency, production concurrency-container policy and package-to-Pixi-to-CI coverage. `task_graph.py` follows task calls through shell scripts; it is shared with timing collection. @@ -24,6 +24,8 @@ Run commands through `pixi run`. Read only the directory relevant to the task. isolation probes. The `tree-link-isolation/` sources belong to that probe. - **`benchmark/`** — repeatable performance regressions, shared result handling, leak inspection and KV write sentinel checks. +- **[`rpc-test/`](rpc-test/README.md)** — real echo transport measurements and + owned server fixtures, including both client schedulers and sanitizer runs. - **`profiling/`** — perf setup and write-path flamegraphs. Common entry points: @@ -41,6 +43,31 @@ update the package groups in `pixi-tasks/test.py` and the checked CI map, then r `check-ci-test-tasks`. Feature-gated or ignored client tests need explicit task selection. +Benchmark and profiling scripts create an explicit native measurement fixture +through `benchmark/fixture.py`. The fixture is a `test-util` example, built by +the scripts, and owns its services by PID plus process start identity. It stops +them on explicit teardown or when the measurement owner exits. Its private +descriptor and retained logs live beside the selected measurement state file. +Memory backends, server transport tuning and group reset belong to this fixture; +the installed CLI attaches to its target using the normal client contract. +ChunkDB measurements retain protected placement over three logical native nodes +so EC workloads remain covered, including in release builds. These processes +share one physical host and do not qualify host-failure tolerance. Read/replay +fixtures use memory disks that retain bytes; allocation and write-throughput +fixtures may use NullDisk. +These native measurements do not certify OCI startup or runtime acceptance. + +KV measurements keep the human console report and `report.json` in the latest +CLI benchmark directory. The fixture saves every invocation under `measurements/` +before another invocation replaces it. Reports contain average, p50 and p99 +latency; scripts print `unavailable` for the unsupported p999 field. Failed +commands clear prior reports and preserve complete diagnostics. The Console +component task explicitly builds and runs `benchmark/fixture-smoke.py`, checking +real read/scan/write, reset, foreign control rejection, owner exit and scoped +cleanup. `bench-leak-inspect.sh` remains alive during inspection; Ctrl-C ends its +ownership and stops its services. Profiling records the attached client; fixture +server PIDs are retained in its state for separate server sampling. + Container packaging and its acceptance scripts live under [`container/single-node-container/`](../container/single-node-container/README.md). Generated artifacts belong under `target/` or `.crowdb-runtime/`, not here. diff --git a/tools/benchmark/bench-chunk-kv-regression.sh b/tools/benchmark/bench-chunk-kv-regression.sh index 8768c2579..80becca99 100755 --- a/tools/benchmark/bench-chunk-kv-regression.sh +++ b/tools/benchmark/bench-chunk-kv-regression.sh @@ -23,32 +23,29 @@ REGRESSION_LOG_ROOT="$LOG_ROOT" source tools/benchmark/bench-regression-common.sh CHUNK_KV_PIDS=() +CHUNK_KV_STARTS=() -stop_auxiliary_processes() { - local pattern="$1" pid - local pids=() - mapfile -t pids < <(pgrep -f "$pattern" || true) - for pid in "${pids[@]}"; do - kill -TERM "$pid" 2>/dev/null || true - done - sleep 0.1 - for pid in "${pids[@]}"; do - if kill -0 "$pid" 2>/dev/null; then - kill -KILL "$pid" 2>/dev/null || true - fi - done +process_start() { + pixi run python - "$1" <<'PY' +import sys +sys.path.insert(0, "tools/benchmark") +from fixture import process_start +print(process_start(int(sys.argv[1])) or "stopped") +PY } + cleanup() { - local pid - for pid in "${CHUNK_KV_PIDS[@]}"; do - if kill -0 "$pid" 2>/dev/null; then - kill -TERM "$pid" 2>/dev/null || true + local pid index status=0 + for index in "${!CHUNK_KV_PIDS[@]}"; do + pid=${CHUNK_KV_PIDS[$index]} + if [ "$(process_start "$pid")" = "${CHUNK_KV_STARTS[$index]}" ]; then + kill -TERM "$pid" || status=1 wait "$pid" 2>/dev/null || true fi done - regression_destroy || true - stop_auxiliary_processes "^$LOG_ROOT/cli-cluster-local-deploy-.*/deploy/.*bin/crowdb-(chunkdb|diskdb|diskio)" + regression_destroy || status=1 + return "$status" } trap cleanup EXIT @@ -73,22 +70,9 @@ fi mkdir -p "$LOG_ROOT" regression_init -stop_auxiliary_processes "^${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/chunk-kv-regression-.*/cli-cluster-local-deploy-.*/deploy/.*bin/crowdb-(chunkdb|diskdb|diskio)" -DEPLOYED=0 -for attempt in 1 2 3; do - REGRESSION_CONFIG="$LOG_ROOT/console-attempt-$attempt.toml" - if regression_cli cluster local-deploy -t combined --metrics-interval 1 --allow-unsafe-ec \ - --kv-backend mem-block --wal-backend mem-block --no-fsync \ - --diskio-dummy-disk-type mem; then - DEPLOYED=1 - break - fi - regression_destroy || true -done -if [[ "$DEPLOYED" -ne 1 ]]; then - echo "chunk KV local deployment failed after 3 attempts" >&2 - exit 1 -fi +regression_fixture setup -- --kind combined --metrics-interval 1 \ + --kv-backend mem-block --wal-backend mem-block --no-fsync \ + --diskio-dummy-disk-type mem MGMT_SEED=$(sed -n 's/^[[:space:]]*url = "\([^"]*\)"/\1/p' "$REGRESSION_CONFIG" | head -n 1) if [ -z "$MGMT_SEED" ]; then @@ -132,30 +116,41 @@ write_config() { } start_server() { - local instance="$1" config="$LOG_ROOT/chunk-kv-$instance.toml" + local instance="$1" start + local config="$LOG_ROOT/chunk-kv-$instance.toml" write_config "$instance" "$((15200 + instance))" "$((15100 + instance))" "$config" - pixi run -- ./target/release/crowdb-chunk-kv-server --config "$config" \ + # The caller runs this script through Pixi. Keep the native child PID, + # rather than the PID of another Pixi command wrapper. + ./target/release/crowdb-chunk-kv-server --config "$config" \ --log-dir "$LOG_ROOT/chunk-kv-$instance-log" \ >"$LOG_ROOT/chunk-kv-$instance.console.log" 2>&1 & SERVER_PID=$! + start=$(process_start "$SERVER_PID") CHUNK_KV_PIDS+=("$SERVER_PID") + CHUNK_KV_STARTS+=("$start") + printf '%s\t%s\n' "$SERVER_PID" "$start" >>"$LOG_ROOT/owned-processes.tsv" } wait_ready() { - local instance="$1" pid="$2" deadline=$((SECONDS + READY_TIMEOUT_SECS)) + local instance="$1" pid="$2" deadline=$((SECONDS + READY_TIMEOUT_SECS)) code while [ "$SECONDS" -lt "$deadline" ]; do - if curl --silent --fail "http://127.0.0.1:$((15100 + instance))/ready" \ - >"$LOG_ROOT/chunk-kv-$instance.ready.json"; then + code=$(curl --silent --output "$LOG_ROOT/chunk-kv-$instance.ready.json" \ + --write-out '%{http_code}' "http://127.0.0.1:$((15100 + instance))/ready") || code=000 + if [ "$code" = 200 ]; then return 0 fi if ! kill -0 "$pid" 2>/dev/null || [ "$(awk '{ print $3 }' "/proc/$pid/stat" 2>/dev/null || true)" = Z ]; then wait "$pid" 2>/dev/null || true echo "chunk-KV server $instance exited before becoming ready" >&2 + cat "$LOG_ROOT/chunk-kv-$instance.console.log" >&2 return 1 fi sleep 1 done echo "ERROR: chunk-KV server $instance did not become ready" >&2 + cat "$LOG_ROOT/chunk-kv-$instance.ready.json" >&2 + regression_fixture inspect >"$LOG_ROOT/failed-recovery-records.json" + cat "$LOG_ROOT/failed-recovery-records.json" >&2 return 1 } @@ -229,6 +224,7 @@ if ! LOAD_OUTPUT=$(timeout "$LOAD_TIMEOUT_SECS" pixi run -- ./target/release/cro --key-prefix "$HOT_KEY_PREFIX" --key-offset 0 --read-percent "$READ_PERCENT"); then printf '%s\n' "$LOAD_OUTPUT" >"$LOG_ROOT/load-continuous.log" capture_failure_metrics + printf '%s\n' "$LOAD_OUTPUT" >&2 echo "ERROR: continuous routed load failed; retained metrics: $LOG_ROOT/load-failure-metrics.log" >&2 exit 1 fi @@ -308,6 +304,7 @@ if [ "$OWNERS_WITH_PARTITIONS" -lt 2 ]; then exit 1 fi +regression_fixture inspect >"$LOG_ROOT/before-restart-records.json" OLD_PID=${CHUNK_KV_PIDS[0]} kill -TERM "$OLD_PID" # Keep the intentional restart inside the registry dead-owner window. The @@ -322,20 +319,10 @@ if kill -0 "$OLD_PID" 2>/dev/null; then fi wait "$OLD_PID" 2>/dev/null || true REPLAY_STARTED_MS=$(date +%s%3N) -RESTARTED_PID=0 -for attempt in 1 2 3; do - start_server 1 - RESTARTED_PID=$SERVER_PID - if wait_ready 1 "$RESTARTED_PID"; then - break - fi - RESTARTED_PID=0 - sleep 6 -done -if [ "$RESTARTED_PID" -eq 0 ]; then - echo "ERROR: restarted chunk-KV server did not become ready after 3 attempts" >&2 - exit 1 -fi +start_server 1 +RESTARTED_PID=$SERVER_PID +wait_ready 1 "$RESTARTED_PID" +regression_fixture inspect >"$LOG_ROOT/after-restart-records.json" REPLAY_MS=$(($(date +%s%3N) - REPLAY_STARTED_MS)) REPLAY_PARTITIONS=$(sed -n 's/.*"hosted_partitions":\([0-9][0-9]*\).*/\1/p' \ "$LOG_ROOT/chunk-kv-1.ready.json") diff --git a/tools/benchmark/bench-chunk-stream-regression.sh b/tools/benchmark/bench-chunk-stream-regression.sh index 62a10ac09..27945a821 100755 --- a/tools/benchmark/bench-chunk-stream-regression.sh +++ b/tools/benchmark/bench-chunk-stream-regression.sh @@ -2,229 +2,497 @@ # Copyright 2026-present Gian # Licensed under the Apache License, Version 2.0. # -# Production chunk-stream regression over the three-node local NullDisk stack. +# Current baseline: 2026-10-24, Intel Core i9-7960X (16 cores / 32 logical CPUs). +# Retained MemDisk, mem-block KV/WAL, 2 synchronized mirrors, 32 Tokio workers. +# All fixtures/workloads/builds ran serially. No CPU power policy change. +# Affinity applies to the client AND every fixture service, sharing the same CPUs. +# default=0-31 (16 physical cores); taskset=0-3 (4 physical cores). +# +# Placement Case Seconds ops/s avg/p50/p99 us CPU client/services % +# default append_4k_1t 60.002 1641.33 608/594/1149 92.90/171.00 +# default append_4k_32t 60.016 15640.45 2044/2017/3818 111.11/206.81 +# cpus_0_3 append_4k_1t 60.002 6110.25 163/149/338 92.91/170.88 +# cpus_0_3 append_4k_32t 60.015 51258.14 623/513/3638 101.94/163.38 +# CPU %: 100%=one logical CPU. Process CPU time covers the whole benchmark +# command, including stream/dataset preparation and teardown; workload latency +# and throughput exclude that preparation. Per-process affinities are verified +# from /proc; CPU timing is captured before/after, without hot-path sampling. +# +# Default-placement remaining cases: throughput, p99 us: +# append_1m_8t 265.71 MiB/s, p99 67035 us +# gc_576m 10599.53 MiB/s, p99 23790 us +# random_4k_32t 46395.38 ops/s, p99 1707 us +# replay_320m_1t 495.56 MiB/s, p99 665975 us +# rollover_4m 307.26 MiB/s, p99 240149 us +# All nine cases pass original throughput/p99, queue/RSS and cleanup gates. +# Ordinary append: zero errors/watchdogs/cursor checkpoints; record publication +# count equals initial attachment plus rollovers. Trim may checkpoint its boundary. +# Benchmark SHA-256: 0ae700e68147f616ef613f993e0130e95274c5668bbd9d37c9311453f8235f29 +# DiskIO SHA-256: a41b81ccf358f51e917b3c9ea6e79164347183b5341f7a6cef675eb162bdf4bf +# Full raw phase/CPU/process/cleanup receipts under .crowdb-runtime/artifacts/bench/: +# chunk-stream-regression-20261024T064043Z (default, seven cases) +# chunk-stream-regression-20261024T064825Z (taskset 0-3, two 60-second cases) +# The measured tree includes uncommitted common C6 instrumentation; binary hashes +# identify the measured artifacts. This baseline is the current implementation. +# +# Retained-MemDisk Stream regression; fresh native fixture for each case. +# Run through: pixi run bash tools/benchmark/bench-chunk-stream-regression.sh # # Optional environment variables: -# CHUNK_STREAM_BENCH_CASES space-separated case labels -# CHUNK_STREAM_BENCH_DURATION timed workload seconds (default: 10) -# CHUNK_STREAM_BENCH_TIMEOUT seconds allowed per case (default: 180) -# CHUNK_STREAM_BENCH_LOG_ROOT retained run root +# CHUNK_STREAM_BENCH_CASES space-separated labels (default: all seven) +# CHUNK_STREAM_BENCH_DURATION large-write/read/replay seconds (default: 10) +# CHUNK_STREAM_BENCH_APPEND_DURATION one-/32-producer 4 KiB seconds (60) +# CHUNK_STREAM_BENCH_TIMEOUT per-case seconds, including setup data (180) +# CHUNK_STREAM_BENCH_WORKERS Tokio workers for client and services (32) +# CHUNK_STREAM_BENCH_CPUS optional taskset CPU list for both sides +# CHUNK_STREAM_BENCH_BIN current release binary; required with skip-build +# CHUNK_STREAM_BENCH_SKIP_BUILD 1 reuses all release artifacts (default: 0) +# CHUNK_STREAM_BENCH_LOG_ROOT new/empty retained run directory # CHUNK_STREAM_BENCH_RESULTS result TSV path -# CHUNK_STREAM_BENCH_SKIP_BUILD set to 1 to reuse release artifacts +# +# This runs the current implementation as the regression baseline. Default +# CPU placement and optional affinity diagnostics are distinct conditions; +# record them with the results instead of treating their throughput as equal. set -euo pipefail cd "$(dirname "$0")/../.." +report() { + pixi run python - "$@" <<'PY_STREAM_REGRESSION' +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Strict result checks and retained reports for the Stream regression script.""" + +import argparse +import csv +import hashlib +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import time + +CASES = { + "append_4k_1t": ("append", 4096, 1, 0, "ops_s", 250, 10000), + "append_4k_32t": ("append", 4096, 32, 0, "ops_s", 3000, 20000), + "append_1m_8t": ("append", 1048576, 8, 0, "mib_s", 125, 150000), + "rollover_4m": ("append", 4194304, 8, 0, "mib_s", 200, 500000), + "random_4k_32t": ("random-read", 4096, 32, 335544320, "ops_s", 30000, 5000), + "replay_320m_1t": ("replay", 4096, 1, 335544320, "mib_s", 300, 2000000), + "gc_576m": ("gc", 4096, 1, 603979776, "mib_s", 1000, 100000), +} +INTEGER_FIELDS = """operations bytes errors avg_us p50_us p99_us rss_start_kib +rss_end_kib rss_peak_kib max_queue_requests max_queue_bytes batches batch_requests +rollovers metadata_publications cache_hits cache_misses physical_reads reclaimed_bytes +watchdogs""".split() +FLOAT_FIELDS = ("seconds", "ops_s", "mib_s") +PHASE_FIELDS = ("count", "avg_us", "p50_us", "p99_us", "max_us", "total_us") +REQUIRED_PHASES = { + "stream.append.e2e.lh", "stream.append.queue.lh", "stream.append.batch.lh", + "stream.append.mirror.lh", "stream.capacity.wait.lh", "stream.successor.wait.lh", + "stream.checkpoint.cursor.lh", "stream.metadata.publish.lh", + "stream.append.capacity.lh", "stream.capacity.prepare.lh", "stream.successor.prepare.lh", + "stream.capacity.grow.lh", "stream.capacity.initialize.lh", "stream.chunk.seal.lh", + "stream.chunk.seal_drain.lh", "stream.append.encode.lh", "stream.append.local_view.lh", + "stream.checkpoint.lh", "stream.metadata.record_build.lh", "stream.metadata.record_get.lh", + "stream.metadata.record_scan.lh", "stream.metadata.record_cas.lh", "stream.read.e2e.lh", + "stream.read.plan.lh", "stream.read.io.lh", "chunkio.diskio.write.e2e.lh", + "rpc.request.e2e", "rpc.response.schedule.lh", +} + + +def fields(line, integers, floats, strings=()): + value = {} + for item in line.split(): + key, separator, raw = item.partition("=") + if not separator or key in value: + raise ValueError(f"malformed or duplicate field: {item}") + value[key] = raw + for key in (*integers, *floats, *strings): + if key not in value: + raise ValueError(f"missing field: {key}") + for key in (*integers, *floats): + number = int(value[key]) if key in integers else float(value[key]) + if number < 0 or not math.isfinite(number): + raise ValueError(f"invalid nonnegative number: {key}={value[key]}") + value[key] = number + return value + + +def analyze(text, label, duration, status=0): + lines = [line.removeprefix("chunk-stream: ") for line in text.splitlines() + if line.startswith("chunk-stream: ")] + if len(lines) != 1: + raise ValueError(f"expected one completed result, found {len(lines)}") + result = fields(lines[0], INTEGER_FIELDS, FLOAT_FIELDS, ("workload",)) + runtime_lines = [line.removeprefix("bench-runtime: ") for line in text.splitlines() + if line.startswith("bench-runtime: ")] + if len(runtime_lines) != 1: + raise ValueError("expected one benchmark runtime/affinity record") + runtime = fields(runtime_lines[0], ("pid", "tokio_workers"), (), ("allowed_cpus",)) + phases = {} + for line in text.splitlines(): + prefix = next((p for p in ("stream-phase: ", "client-phase: ") if line.startswith(p)), None) + if prefix is None: + continue + phase = fields(line[len(prefix):], ("count", "p50_us", "p99_us", "max_us"), + ("avg_us",), ("name",)) + name = phase.pop("name") + if name in phases: + raise ValueError(f"duplicate phase: {name}") + # Stages overlap; this cumulative time must not be summed as if every + # histogram described an exclusive portion of each request. + phase["total_us"] = phase["count"] * phase["avg_us"] + phases[name] = phase + missing = REQUIRED_PHASES - phases.keys() + if missing: + raise ValueError(f"missing phases: {sorted(missing)}") + workload, size, _, _, rate, minimum, p99 = CASES[label] + checks = { + "completed": status == 0 and result["operations"] > 0 and result["seconds"] > 0, + "workload": result["workload"] == workload, + "no_errors": result["errors"] == 0 and result["watchdogs"] == 0, + "full_duration": workload == "gc" or result["seconds"] >= duration - 0.001, + } + checks.update( + queue=result["max_queue_requests"] <= 1024 and result["max_queue_bytes"] <= 67108864, + rss=0 < result["rss_start_kib"] <= result["rss_peak_kib"] + and 0 < result["rss_end_kib"] <= result["rss_peak_kib"] + and result["rss_peak_kib"] - result["rss_start_kib"] <= 262144, + throughput=result[rate] >= minimum, + p99=result["p99_us"] <= p99, + publication_accounting=phases["stream.metadata.publish.lh"]["count"] + == result["metadata_publications"], + ) + if workload == "append": + checks.update( + publications=result["metadata_publications"] == result["rollovers"] + 1, + no_cursor_checkpoint=phases["stream.checkpoint.cursor.lh"]["count"] == 0, + append_bytes=result["bytes"] == result["operations"] * size, + completed_requests=result["batch_requests"] == result["operations"], + batch_accounting=phases["stream.append.batch.lh"]["count"] == result["batches"], + mirror_batches=phases["stream.append.mirror.lh"]["count"] == result["batches"], + mirror_writes=phases["chunkio.diskio.write.e2e.lh"]["count"] >= 2 * result["batches"], + ) + if label == "rollover_4m": + checks["rollover"] = result["rollovers"] > 0 + elif workload in ("random-read", "replay"): + multiplier = 32 if workload == "random-read" else 1 + checks["record_lookups"] = (result["physical_reads"] > 0 and result["cache_misses"] + <= multiplier * (result["rollovers"] + 1)) + if workload == "random-read": + checks["cache_hits"] = result["cache_hits"] > 0 + else: + checks["reclaimed_chunk"] = result["reclaimed_bytes"] > 0 + return dict(case=label, runtime=runtime, result=result, phases=phases, checks=checks, + passed=all(checks.values())) + + +def digest(path): + with Path(path).open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def initialize(args): + root = Path.cwd() + binaries = [args.binary, os.environ["CROWDB_REGRESSION_FIXTURE_BIN"], + *(root / "target/release" / name for name in + ("crowdb-cli", "crowdb-kv-server", "crowdb-diskdb", "crowdb-chunkdb")), + root / "app/crowdb-diskio/build/crowdb-diskio"] + value = dict(commit=subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + dirty_paths=subprocess.check_output(["git", "status", "--short"], text=True).splitlines(), + binaries={str(Path(p).resolve()): digest(p) for p in binaries}, + cpus=args.cpus or "default", inherited_cpus=sorted(os.sched_getaffinity(0)), + tokio_workers=int(os.environ["TOKIO_WORKER_THREADS"]), duration_secs=args.duration, + append_duration_secs=args.append_duration, mirror_copies=2, cases=args.cases.split(), + media="retained-MemDisk", kv_backend="mem-block", wal_backend="mem-block", + optional_checkpoints=False, fresh_fixture_per_case=True) + topology = {} + for cpu in os.sched_getaffinity(0): + path = Path(f"/sys/devices/system/cpu/cpu{cpu}/topology") + topology[cpu] = {name: (path / name).read_text().strip() + for name in ("core_id", "physical_package_id", "thread_siblings_list")} + value["cpu_topology"] = topology + (args.root / "conditions.json").write_text(json.dumps(value, indent=2) + "\n") + + +def record(args): + try: + value = analyze(args.log.read_text(), args.label, args.duration, args.status) + value["cpu"] = json.loads((args.log.parent / "cpu.json").read_text()) + receipt = json.loads((args.log.parent / "console.fixture-stopped.json").read_text()) + value["checks"]["cleanup"] = receipt.get("ok") is True and receipt.get("error") is None + value["passed"] = all(value["checks"].values()) + except (OSError, ValueError) as error: + value = dict(case=args.label, result={}, phases={}, checks={}, + passed=False, error=str(error)) + (args.log.parent / "result.json").write_text(json.dumps(value, indent=2) + "\n") + if not value["passed"]: + print(f"FAILED {args.label}: " + value.get("error", str(value["checks"])), file=sys.stderr) + return 0 if value["passed"] else 1 + + +def summarize(args): + cases = [json.loads(path.read_text()) for path in sorted(args.root.glob("*/result.json"))] + conditions = json.loads((args.root / "conditions.json").read_text()) + expected = set(conditions["cases"]) + complete = len(cases) == len(expected) and {c["case"] for c in cases} == expected + if not complete: + print(f"FAILED case inventory: expected {sorted(expected)}, received " + f"{[c['case'] for c in cases]}", file=sys.stderr) + value = dict(conditions=conditions, cases=cases, + complete=complete, passed=bool(cases) and complete and all(c["passed"] for c in cases)) + (args.root / "results.json").write_text(json.dumps(value, indent=2) + "\n") + with args.results.open("w") as stream, (args.root / "phases.tsv").open("w") as phase_stream: + results = csv.writer(stream, delimiter="\t") + phases = csv.writer(phase_stream, delimiter="\t") + columns = ["workload", *INTEGER_FIELDS, *FLOAT_FIELDS] + results.writerow(["case", "passed", *columns]) + phases.writerow(["case", "name", *PHASE_FIELDS]) + for case in cases: + results.writerow([case["case"], case["passed"], + *(case["result"].get(field, "") for field in columns)]) + for name, phase in case["phases"].items(): + phases.writerow([case["case"], name, + *(phase[field] for field in PHASE_FIELDS)]) + print(json.dumps(dict(passed=value["passed"], results=str(args.results)), indent=2)) + return 0 if value["passed"] else 1 + + +def cpu_snapshot(pid, start): + proc = Path(f"/proc/{pid}") + fields = (proc / "stat").read_text().rsplit(")", 1)[1].split() + if fields[19] != start or fields[0] == "Z": + raise ValueError(f"recorded benchmark service {pid} was replaced or exited") + affinities = {} + for task in (proc / "task").iterdir(): + try: + mask = ",".join(map(str, sorted(os.sched_getaffinity(int(task.name))))) + except ProcessLookupError: + continue + affinities[mask] = affinities.get(mask, 0) + 1 + return dict(pid=pid, start=start, executable=(proc / "exe").resolve().name, + cpu_seconds=(int(fields[11]) + int(fields[12])) / os.sysconf("SC_CLK_TCK"), + allowed_cpus=sorted(os.sched_getaffinity(pid)), thread_affinities=affinities) + + +def cpu_start(state): + descriptor = json.loads(state.with_suffix(".fixture.json").read_text()) + namespace = json.loads((Path(descriptor["root"]) / "namespace.json").read_text()) + services = [cpu_snapshot(p["pid"], p["start"]) for p in namespace["processes"]] + if not services: + raise ValueError("fixture has no recorded service processes") + (state.parent / "cpu-start.json").write_text(json.dumps( + dict(monotonic=time.monotonic(), services=services), indent=2) + "\n") + + +def cpu_finish(state): + before = json.loads((state.parent / "cpu-start.json").read_text()) + elapsed = time.monotonic() - before["monotonic"] + services = [] + for previous in before["services"]: + current = cpu_snapshot(previous["pid"], previous["start"]) + delta = current["cpu_seconds"] - previous["cpu_seconds"] + if delta < 0: + raise ValueError("service CPU accounting regressed") + current.update(cpu_seconds=delta, average_cpu_percent=100 * delta / elapsed) + services.append(current) + client = json.loads((state.parent / "cpu-client.json").read_text()) + client["cpu_seconds"] = client["user_seconds"] + client["system_seconds"] + client["average_cpu_percent"] = 100 * client["cpu_seconds"] / client["wall_seconds"] + value = dict(window="whole benchmark command, including stream/dataset preparation and teardown", + percent_unit="100% is one logical CPU; services are measured across the same command window", + server_window_seconds=elapsed, services=services, client=client, + service_cpu_seconds=sum(p["cpu_seconds"] for p in services)) + (state.parent / "cpu.json").write_text(json.dumps(value, indent=2) + "\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="action", required=True) + sub.add_parser("cases").add_argument("selection") + sub.add_parser("arguments").add_argument("label", choices=CASES) + sub.add_parser("binary").add_argument("messages", type=Path) + init = sub.add_parser("init") + for flag in ("binary", "cpus"): + init.add_argument("--" + flag, required=True) + init.add_argument("--duration", type=int, required=True) + init.add_argument("--append-duration", type=int, required=True) + init.add_argument("--cases", required=True) + init.add_argument("--root", type=Path, required=True) + case = sub.add_parser("case") + case.add_argument("--label", choices=CASES, required=True) + case.add_argument("--duration", type=int, required=True) + case.add_argument("--status", type=int, required=True) + case.add_argument("--log", type=Path, required=True) + summary = sub.add_parser("summary") + summary.add_argument("--root", type=Path, required=True) + summary.add_argument("--results", type=Path, required=True) + for action in ("cpu-start", "cpu-finish"): + sub.add_parser(action).add_argument("--state", type=Path, required=True) + args = parser.parse_args() + if args.action == "cases": + labels = args.selection.split() or list(CASES) + if len(set(labels)) != len(labels) or any(label not in CASES for label in labels): + raise ValueError("case selection has unknown or duplicate labels") + print("\n".join(labels)) + elif args.action == "arguments": + workload, size, concurrency, dataset, *_ = CASES[args.label] + values = ["--workload", workload, "--object-size", str(size), "--concurrency", str(concurrency)] + if dataset: + values += ["--dataset-bytes", str(dataset)] + print("\n".join(values)) + elif args.action == "binary": + paths = {m["executable"] for line in args.messages.read_text().splitlines() + if (m := json.loads(line)).get("reason") == "compiler-artifact" + and m["target"]["name"] == "production_bounds" and m.get("executable")} + if len(paths) != 1: + raise ValueError("build did not identify one production_bounds executable") + print(paths.pop()) + elif args.action == "cpu-start": + cpu_start(args.state) + elif args.action == "cpu-finish": + cpu_finish(args.state) + elif args.action == "init": + initialize(args) + elif args.action == "case": + return record(args) + else: + return summarize(args) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except (OSError, ValueError, subprocess.SubprocessError) as error: + print(f"Stream regression failed: {error}", file=sys.stderr) + sys.exit(2) +PY_STREAM_REGRESSION +} + unset CROWDB_ASAN -CASES="${CHUNK_STREAM_BENCH_CASES:-}" -DURATION_SECS="${CHUNK_STREAM_BENCH_DURATION:-10}" +DURATION="${CHUNK_STREAM_BENCH_DURATION:-10}" +APPEND_DURATION="${CHUNK_STREAM_BENCH_APPEND_DURATION:-60}" TIMEOUT_SECS="${CHUNK_STREAM_BENCH_TIMEOUT:-180}" SKIP_BUILD="${CHUNK_STREAM_BENCH_SKIP_BUILD:-0}" -RUN_STAMP=$(date +%Y%m%d-%H%M%S) -LOG_ROOT="${CHUNK_STREAM_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/chunk-stream-regression-$RUN_STAMP}" -RESULTS_FILE="${CHUNK_STREAM_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" -REGRESSION_LOG_ROOT="$LOG_ROOT" -source tools/benchmark/bench-regression-common.sh -FAILURES=0 -CASE_NUMBER=0 - -for value in "$DURATION_SECS" "$TIMEOUT_SECS"; do +BENCH_BIN="${CHUNK_STREAM_BENCH_BIN:-}" +CPUS="${CHUNK_STREAM_BENCH_CPUS:-}" +export TOKIO_WORKER_THREADS="${CHUNK_STREAM_BENCH_WORKERS:-32}" +CASES="${CHUNK_STREAM_BENCH_CASES:-}" +for value in "$DURATION" "$APPEND_DURATION" "$TIMEOUT_SECS" "$TOKIO_WORKER_THREADS"; do if ! [[ "$value" =~ ^[1-9][0-9]*$ ]]; then - echo "ERROR: benchmark duration and timeout must be positive integers" >&2 + echo 'ERROR: duration, timeout and worker counts must be positive integers' >&2 exit 2 fi done if [[ "$SKIP_BUILD" != 0 && "$SKIP_BUILD" != 1 ]]; then - echo "ERROR: CHUNK_STREAM_BENCH_SKIP_BUILD must be 0 or 1" >&2 + echo 'ERROR: CHUNK_STREAM_BENCH_SKIP_BUILD must be 0 or 1' >&2 + exit 2 +fi +if [ "$TIMEOUT_SECS" -le "$DURATION" ] || [ "$TIMEOUT_SECS" -le "$APPEND_DURATION" ]; then + echo 'ERROR: per-case timeout must exceed workload duration' >&2 + exit 2 +fi +labels=$(report cases "$CASES") +mapfile -t labels <<<"$labels" +affinity=() +if [ -n "$CPUS" ]; then + affinity=(taskset -c "$CPUS") + pixi run "${affinity[@]}" true +fi +if [ -n "$BENCH_BIN" ] && [ ! -x "$BENCH_BIN" ]; then + echo "ERROR: benchmark binary is not executable: $BENCH_BIN" >&2 + exit 2 +fi +if [ "$SKIP_BUILD" -eq 1 ] && [ -z "$BENCH_BIN" ]; then + echo 'ERROR: skip-build requires CHUNK_STREAM_BENCH_BIN; no stale artifact guessing' >&2 exit 2 fi -destroy_cluster() { - if [ -f "$REGRESSION_CONFIG" ]; then - regression_destroy - fi -} -trap destroy_cluster EXIT - -field() { - local line="$1" name="$2" - awk -v name="$name" '{ - for (i = 1; i <= NF; i++) { - split($i, pair, "=") - if (pair[1] == name) { - print pair[2] - exit - } - } - }' <<<"$line" -} - -selected() { - local label="$1" - [ -z "$CASES" ] || [[ " $CASES " == *" $label "* ]] -} - -record_result() { - local label="$1" line="$2" - printf '%s' "$label" >>"$RESULTS_FILE" - local name - for name in workload operations bytes errors seconds ops_s mib_s avg_us p50_us p99_us \ - rss_start_kib rss_end_kib rss_peak_kib max_queue_requests max_queue_bytes \ - batches batch_requests rollovers metadata_publications cache_hits cache_misses \ - physical_reads reclaimed_bytes watchdogs; do - printf '\t%s' "$(field "$line" "$name")" >>"$RESULTS_FILE" - done - printf '\n' >>"$RESULTS_FILE" +RUN_STAMP=$(date -u +%Y%m%dT%H%M%SZ) +LOG_ROOT="${CHUNK_STREAM_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/chunk-stream-regression-$RUN_STAMP}" +mkdir -p "$LOG_ROOT" +if [ -n "$(ls -A "$LOG_ROOT")" ]; then + echo "ERROR: run directory must be empty: $LOG_ROOT" >&2 + exit 2 +fi +LOG_ROOT=$(realpath "$LOG_ROOT") +RESULTS_FILE="${CHUNK_STREAM_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" +mkdir -p "$(dirname "$RESULTS_FILE")" "$LOG_ROOT/bin" +REGRESSION_LOG_ROOT="$LOG_ROOT" +source tools/benchmark/bench-regression-common.sh +cleanup() { + local status=$? + trap - EXIT + regression_destroy || status=1 + exit "$status" } +trap cleanup EXIT -validate_common() { - local label="$1" status="$2" line="$3" - local operations errors queue_requests queue_bytes watchdogs rss_start rss_peak - operations=$(field "$line" operations) - errors=$(field "$line" errors) - queue_requests=$(field "$line" max_queue_requests) - queue_bytes=$(field "$line" max_queue_bytes) - watchdogs=$(field "$line" watchdogs) - rss_start=$(field "$line" rss_start_kib) - rss_peak=$(field "$line" rss_peak_kib) - if [ "$status" -ne 0 ] || [ -z "$line" ] || [ -z "$operations" ] \ - || [ "$operations" -eq 0 ] || [ "$errors" != 0 ] || [ "$watchdogs" != 0 ] \ - || [ "$queue_requests" -gt 1024 ] || [ "$queue_bytes" -gt 67108864 ] \ - || [ $((rss_peak - rss_start)) -gt 262144 ]; then - echo "ERROR: $label violated completion, queue, or watchdog bounds" >&2 - return 1 +# Finish every build before starting fixtures or measurements. Invoke the +# identified executable directly, so a case cannot trigger a Cargo rebuild. +if [ "$SKIP_BUILD" -eq 0 ]; then + pixi run build-cpp + pixi run cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb + if [ -z "$BENCH_BIN" ]; then + pixi run cargo bench -p crowdb-chunk-stream --bench production_bounds --no-run \ + --message-format=json >"$LOG_ROOT/build.jsonl" + BENCH_BIN=$(report binary "$LOG_ROOT/build.jsonl") fi -} - -at_least() { - local actual="$1" minimum="$2" - awk -v actual="$actual" -v minimum="$minimum" 'BEGIN { exit !(actual >= minimum) }' -} - -at_most() { - local actual="$1" maximum="$2" - awk -v actual="$actual" -v maximum="$maximum" 'BEGIN { exit !(actual <= maximum) }' -} - -validate_case() { - local label="$1" line="$2" - local workload rollovers publications misses hits reads reclaimed - workload=$(field "$line" workload) - rollovers=$(field "$line" rollovers) - publications=$(field "$line" metadata_publications) - misses=$(field "$line" cache_misses) - hits=$(field "$line" cache_hits) - reads=$(field "$line" physical_reads) - reclaimed=$(field "$line" reclaimed_bytes) - case "$workload" in - append) - if [ "$publications" -gt $((rollovers + 1)) ]; then - echo "ERROR: $label published metadata on the append hot path" >&2 - return 1 - fi - if [[ "$label" == rollover_* && "$rollovers" -eq 0 ]]; then - echo "ERROR: $label did not cross a chunk boundary" >&2 - return 1 - fi - ;; - random-read) - if [ "$reads" -eq 0 ] || [ "$misses" -gt $((32 * (rollovers + 1))) ] || [ "$hits" -eq 0 ]; then - echo "ERROR: $label exceeded extent lookup bounds" >&2 - return 1 - fi - ;; - replay) - if [ "$reads" -eq 0 ] || [ "$misses" -gt $((rollovers + 1)) ]; then - echo "ERROR: $label exceeded replay lookup bounds" >&2 - return 1 - fi - ;; - gc) - if [ "$reclaimed" -eq 0 ]; then - echo "ERROR: $label reclaimed no complete stream chunk" >&2 - return 1 - fi - ;; - *) - echo "ERROR: $label returned unknown workload $workload" >&2 - return 1 - ;; - esac - local ops_s mib_s p99 - ops_s=$(field "$line" ops_s) - mib_s=$(field "$line" mib_s) - p99=$(field "$line" p99_us) - case "$label" in - append_4k_1t) at_least "$ops_s" 250 && at_most "$p99" 10000 ;; - append_4k_32t) at_least "$ops_s" 3000 && at_most "$p99" 20000 ;; - append_1m_8t) at_least "$mib_s" 125 && at_most "$p99" 150000 ;; - rollover_4m) at_least "$mib_s" 200 && at_most "$p99" 500000 ;; - random_4k_32t) at_least "$ops_s" 30000 && at_most "$p99" 5000 ;; - replay_320m_1t) at_least "$mib_s" 300 && at_most "$p99" 2000000 ;; - gc_576m) at_least "$mib_s" 1000 && at_most "$p99" 100000 ;; - esac || { - echo "ERROR: $label violated its production throughput or p99 bound" >&2 - return 1 - } -} +fi +regression_init +export CROWDB_REGRESSION_FIXTURE_BIN="${CROWDB_REGRESSION_FIXTURE_BIN:-$(pwd)/target/release/examples/regression-fixture}" +cp -- "$BENCH_BIN" "$LOG_ROOT/bin/current" +BENCH_BIN="$LOG_ROOT/bin/current" +report init --root "$LOG_ROOT" --binary "$BENCH_BIN" \ + --cpus "$CPUS" --duration "$DURATION" --append-duration "$APPEND_DURATION" --cases "${labels[*]}" run_case() { - local label="$1" - shift - if ! selected "$label"; then - return - fi - if [ "$CASE_NUMBER" -gt 0 ]; then - regression_reset_stack 1 - fi - CASE_NUMBER=$((CASE_NUMBER + 1)) - echo ">>> $label" - local output status line case_log="$LOG_ROOT/$label.log" + local label="$1" duration="$2" name="$1" arguments seed status + local case_args=() statuses=() + REGRESSION_CONFIG="$LOG_ROOT/$name/console.toml" + mkdir -p "$LOG_ROOT/$name" + echo ">>> $name (${duration}s, workers=$TOKIO_WORKER_THREADS, cpus=${CPUS:-default})" + pixi run "${affinity[@]}" python tools/benchmark/fixture.py --state "$REGRESSION_CONFIG" \ + --owner-pid "$$" setup -- --kind combined --metrics-interval 1 \ + --kv-backend mem-block --wal-backend mem-block --no-fsync --diskio-dummy-disk-type mem + seed=$(pixi run python tools/benchmark/fixture.py --state "$REGRESSION_CONFIG" seed) + arguments=$(report arguments "$label") + mapfile -t case_args <<<"$arguments" + report cpu-start --state "$REGRESSION_CONFIG" + date -u --iso-8601=ns >"$LOG_ROOT/$name/started.txt" set +e - output=$(timeout --signal=INT --kill-after=10 "$TIMEOUT_SECS" \ - pixi run -- cargo bench -p crowdb-chunk-stream --bench production_bounds -- \ - --duration-secs "$DURATION_SECS" "$@" 2>&1) - status=$? + /usr/bin/time --quiet --format='{"user_seconds":%U,"system_seconds":%S,"wall_seconds":%e,"rss_peak_kib":%M}' \ + --output="$LOG_ROOT/$name/cpu-client.json" \ + timeout --signal=INT --kill-after=10 "$TIMEOUT_SECS" pixi run "${affinity[@]}" "$BENCH_BIN" \ + --management-seed "$seed" --duration-secs "$duration" "${case_args[@]}" \ + 2>&1 | tee "$LOG_ROOT/$name/workload.log" + statuses=("${PIPESTATUS[@]}") set -e - printf '%s\n' "$output" | tee "$case_log" - line=$(sed -n '/^chunk-stream:/p' <<<"$output" | tail -n 1) - if [ -n "$line" ]; then - record_result "$label" "$line" + status=${statuses[0]} + if [ "${statuses[1]}" -ne 0 ]; then + echo "ERROR: could not retain $name output" >&2 + exit 1 fi - if ! validate_common "$label" "$status" "$line" || ! validate_case "$label" "$line"; then - FAILURES=$((FAILURES + 1)) + date -u --iso-8601=ns >"$LOG_ROOT/$name/finished.txt" + report cpu-finish --state "$REGRESSION_CONFIG" + regression_destroy + if ! report case --label "$label" --duration "$duration" --status "$status" \ + --log "$LOG_ROOT/$name/workload.log"; then + echo "ERROR: $name failed; continuing with a fresh fixture for the next case" >&2 fi } -if [ "$SKIP_BUILD" -eq 0 ]; then - echo "=== building production benchmark and local stack ===" - pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb - pixi run build-cpp - pixi run -- cargo bench -p crowdb-chunk-stream --bench production_bounds --no-run -fi +for label in "${labels[@]}"; do + duration="$DURATION" + if [[ "$label" == append_4k_1t || "$label" == append_4k_32t ]]; then + duration="$APPEND_DURATION" + fi + run_case "$label" "$duration" +done -mkdir -p "$LOG_ROOT" "$(dirname "$RESULTS_FILE")" -regression_init -printf 'case\tworkload\toperations\tbytes\terrors\tseconds\tops_s\tmib_s\tavg_us\tp50_us\tp99_us\trss_start_kib\trss_end_kib\trss_peak_kib\tmax_queue_requests\tmax_queue_bytes\tbatches\tbatch_requests\trollovers\tmetadata_publications\tcache_hits\tcache_misses\tphysical_reads\treclaimed_bytes\twatchdogs\n' >"$RESULTS_FILE" - -regression_cli cluster local-deploy -t combined --metrics-interval 1 --allow-unsafe-ec \ - --kv-backend mem-block --wal-backend mem-block --no-fsync - -run_case append_4k_1t --workload append --object-size 4096 --concurrency 1 -run_case append_4k_32t --workload append --object-size 4096 --concurrency 32 -run_case append_1m_8t --workload append --object-size 1048576 --concurrency 8 -run_case rollover_4m --workload append --object-size 4194304 --concurrency 8 --operations 65 -run_case random_4k_32t --workload random-read --object-size 4096 --concurrency 32 \ - --dataset-bytes 335544320 -run_case replay_320m_1t --workload replay --object-size 4096 --concurrency 1 \ - --dataset-bytes 335544320 -run_case gc_576m --workload gc --object-size 4096 --concurrency 1 --dataset-bytes 603979776 - -destroy_cluster -trap - EXIT -echo "=== DONE ===" -echo "Logs and results retained in $LOG_ROOT" -column -t -s$'\t' "$RESULTS_FILE" -if [ "$FAILURES" -ne 0 ]; then - echo "ERROR: $FAILURES regression case(s) failed" >&2 - exit 1 -fi +report summary --root "$LOG_ROOT" --results "$RESULTS_FILE" +echo "Logs, binary identities, phase distributions and cleanup receipts: $LOG_ROOT" diff --git a/tools/benchmark/bench-chunkdb-regression.sh b/tools/benchmark/bench-chunkdb-regression.sh index 5b6616f74..e6c88d788 100755 --- a/tools/benchmark/bench-chunkdb-regression.sh +++ b/tools/benchmark/bench-chunkdb-regression.sh @@ -100,14 +100,14 @@ any_case_selected() { verify_logs() { local label="$1" kv_metrics diskdb_metrics chunkdb_metrics cli_metrics local kv_rpc diskdb_rpc chunkdb_rpc cli_rpc expected_servers expected_clients - kv_metrics=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/kv-server-*/log/crowdb-kv-server-metrics-*.log' -type f | wc -l) - diskdb_metrics=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/diskdb-*/log/crowdb-diskdb-metrics-*.log' -type f | wc -l) - chunkdb_metrics=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/chunkdb-*/log/crowdb-chunkdb-metrics-*.log' -type f | wc -l) - cli_metrics=$(find "$LOG_ROOT" -path '*/cli-bench-chunkdb-*/crowdb-cli-metrics-*.log' -type f | wc -l) - kv_rpc=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/kv-server-*/log/crowdb-kv-server-rpc-*.log' -type f | wc -l) - diskdb_rpc=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/diskdb-*/log/crowdb-diskdb-rpc-*.log' -type f | wc -l) - chunkdb_rpc=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/chunkdb-*/log/crowdb-chunkdb-rpc-*.log' -type f | wc -l) - cli_rpc=$(find "$LOG_ROOT" -path '*/cli-bench-chunkdb-*/crowdb-cli-rpc-*.log' -type f | wc -l) + kv_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-kv-server-metrics-*.log' -type f | wc -l) + diskdb_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/diskdb-*/log/crowdb-diskdb-metrics-*.log' -type f | wc -l) + chunkdb_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/chunkdb-*/log/crowdb-chunkdb-metrics-*.log' -type f | wc -l) + cli_metrics=$(find "$LOG_ROOT" -path '*/measurements/bench-chunkdb-*/crowdb-cli-metrics-*.log' -type f | wc -l) + kv_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-kv-server-rpc-*.log' -type f | wc -l) + diskdb_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/diskdb-*/log/crowdb-diskdb-rpc-*.log' -type f | wc -l) + chunkdb_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/chunkdb-*/log/crowdb-chunkdb-rpc-*.log' -type f | wc -l) + cli_rpc=$(find "$LOG_ROOT" -path '*/measurements/bench-chunkdb-*/crowdb-cli-rpc-*.log' -type f | wc -l) expected_servers=$((CASE_NUMBER * 3)) expected_clients="$CASE_NUMBER" if [ "$kv_metrics" -ne $((DEPLOY_NUMBER * 3)) ] || [ "$diskdb_metrics" -lt "$expected_servers" ] \ @@ -128,7 +128,7 @@ deploy_group() { CURRENT_CONFIG="$REGRESSION_CONFIG" CASE_IN_GROUP=0 DEPLOY_NUMBER=$((DEPLOY_NUMBER + 1)) - cli cluster local-deploy -t combined \ + regression_fixture setup -- --kind combined \ --kv-backend mem-block --wal-backend mem-block --metrics-interval 1 \ --event-write --peer-pool-size "$connections" --rpc-workers "$workers" \ --max-inflight "$KV_INFLIGHT" --coalesce-max-keys "$KV_COALESCE" \ @@ -157,7 +157,7 @@ run_case() { local output status line space busy expected set +e output=$(timeout --signal=INT --kill-after=10 "$((DURATION + 40))" \ - pixi run -- ./target/release/crowdb-cli --log-root "$LOG_ROOT" --config "$CURRENT_CONFIG" \ + pixi run python tools/benchmark/fixture.py --state "$CURRENT_CONFIG" client -- \ bench chunkdb allocate --duration-secs "$DURATION" --concurrency "$concurrency" \ --chunkdb-connections "$connections" --chunkdb-client-rpc-workers "$workers" \ --strip-count 1 --strip-type ec --data-num 8 --code-num 4 \ diff --git a/tools/benchmark/bench-chunkio-read-regression.sh b/tools/benchmark/bench-chunkio-read-regression.sh index 8c99f1598..48909b6e1 100755 --- a/tools/benchmark/bench-chunkio-read-regression.sh +++ b/tools/benchmark/bench-chunkio-read-regression.sh @@ -4,15 +4,16 @@ # # The preparation phase writes real ChunkDB/DiskDB metadata through the client # library into mem-block KV/WAL backends. The timed phase reads through the -# complete client -> ChunkDB -> KV and client -> DiskIO paths; DiskIO uses -# NullDisk so device capacity and media latency do not dominate the result. +# complete client -> ChunkDB -> KV and client -> DiskIO paths. DiskIO uses +# MemDisk so reads retain and verify the prepared bytes without device latency. # Each case runs for 20 seconds and counts only successful responses as TPS. -# cluster clean restarts the auxiliary stack between cases. +# fixture reset restarts the auxiliary stack between cases. # # Reference platform: Intel Core i9-7960X (16c/32t, x86_64, Linux). # Configuration: 8 DiskIO connections/endpoint, 1 client DiskIO RPC worker, # 8 KiB small objects, 16 MiB large objects, 16 prepared objects/class, -# EC 8+4, 1 MiB blocks, mem-block metadata, NullDisk data (2026-09-09). +# Historical baseline: EC 8+4, 1 MiB blocks, mem-block metadata, +# NullDisk data (2026-09-09). It is not a retained-byte read baseline. # # Reference results (all errors=0, incomplete=0, stop=complete): # workload threads success TPS MiB/s p50 us p99 us @@ -108,8 +109,7 @@ run_case() { local output status line requested reads errors incomplete stop small_reads large_reads set +e output=$(timeout --signal=INT --kill-after=10 "$TIMEOUT_SECS" \ - pixi run -- ./target/release/crowdb-cli --log-root "$LOG_ROOT" \ - --config "$CURRENT_CONFIG" bench chunkio "$verb" \ + pixi run python tools/benchmark/fixture.py --state "$CURRENT_CONFIG" client -- bench chunkio "$verb" \ --requests 18446744073709551615 --duration-secs "$DURATION" \ --dataset-objects "$DATASET_OBJECTS" --concurrency "$concurrency" \ --diskio-connections "$DISKIO_CONNECTIONS" --diskio-rpc-workers "$DISKIO_RPC_WORKERS" \ @@ -156,12 +156,12 @@ mkdir -p "$LOG_ROOT" regression_init printf 'case\tverb\tconcurrency\trequested\treads\tsmall_reads\tlarge_reads\terrors\tincomplete\tstop\treads_s\tlogical_mib_s\tavg_us\tp50_us\tp99_us\tprepare_s\n' >"$RESULTS_FILE" -deploy_args=(cluster local-deploy -t combined --metrics-interval 1 --allow-unsafe-ec \ - --kv-backend mem-block --wal-backend mem-block --no-fsync) +deploy_args=( --kind combined --metrics-interval 1 \ + --kv-backend mem-block --wal-backend mem-block --no-fsync --diskio-dummy-disk-type mem) if [ -n "$SERVER_RPC_WORKERS" ]; then deploy_args+=(--diskio-rpc-workers "$SERVER_RPC_WORKERS") fi -regression_cli "${deploy_args[@]}" +regression_fixture setup -- "${deploy_args[@]}" run_case read_small_1t read-small 1 run_case read_small_4t read-small 4 run_case read_small_8t read-small 8 diff --git a/tools/benchmark/bench-chunkio-small-write-regression.sh b/tools/benchmark/bench-chunkio-small-write-regression.sh index 17ba793f1..3b998d173 100755 --- a/tools/benchmark/bench-chunkio-small-write-regression.sh +++ b/tools/benchmark/bench-chunkio-small-write-regression.sh @@ -115,14 +115,13 @@ run_case() { destroy_cluster fi CURRENT_CONFIG="$REGRESSION_CONFIG" - regression_cli "${deploy_args[@]}" + regression_fixture setup -- "${deploy_args[@]}" CASE_NUMBER=$((CASE_NUMBER + 1)) echo ">>> $label (size=$size concurrency=$concurrency)" local output status line requested completed errors incomplete stop scale_out parity_bytes first_reservation_us set +e output=$(timeout --signal=INT --kill-after=10 "$TIMEOUT_SECS" \ - pixi run -- ./target/release/crowdb-cli --log-root "$LOG_ROOT" \ - --config "$CURRENT_CONFIG" bench chunkio write-small \ + pixi run python tools/benchmark/fixture.py --state "$CURRENT_CONFIG" client -- bench chunkio write-small \ --objects 18446744073709551615 --duration-secs "$DURATION" \ --object-size "$size" --concurrency "$concurrency" \ --diskio-connections "$DISKIO_CONNECTIONS" --diskio-rpc-workers "$DISKIO_RPC_WORKERS" \ @@ -184,7 +183,7 @@ mkdir -p "$LOG_ROOT" regression_init printf 'case\tsize_bytes\tconcurrency\trequested\tcompleted\terrors\tincomplete\tstop\tobjects_s\tlogical_mib_s\tavg_us\tp50_us\tp90_us\tp95_us\tp99_us\tmax_us\tbatches\tbatch_watchdog_expirations\tmax_batch_objects\taggregate_write_requests\taggregate_write_objects\taggregate_write_buffers\taggregate_write_payload_bytes\tmax_objects_per_write_request\tmax_buffers_per_write_request\tmax_active_pipelines\tscale_out\tscale_in\tforeground_parity_bytes\treservation_wait_us\tfirst_reservation_us\n' >"$RESULTS_FILE" -deploy_args=(cluster local-deploy -t combined --metrics-interval 1 --allow-unsafe-ec \ +deploy_args=( --kind combined --metrics-interval 1 \ --kv-backend mem-block --wal-backend mem-block --no-fsync) if [ -n "$SERVER_RPC_WORKERS" ]; then deploy_args+=(--diskio-rpc-workers "$SERVER_RPC_WORKERS") diff --git a/tools/benchmark/bench-chunkio-write-regression.sh b/tools/benchmark/bench-chunkio-write-regression.sh index c14cb9179..fd6a0a83b 100755 --- a/tools/benchmark/bench-chunkio-write-regression.sh +++ b/tools/benchmark/bench-chunkio-write-regression.sh @@ -6,6 +6,7 @@ # CHUNKIO_BENCH_LOG_ROOT retained run root # CHUNKIO_BENCH_RESULTS result TSV path # CHUNKIO_BENCH_TIMEOUT seconds allowed per case (default: 120) +# CHUNKIO_BENCH_SKIP_BUILD set to 1 to reuse release artifacts # CHUNKIO_PREFETCH_CHUNKS chunks warmed before timed load (default: 10) # # Reference run (2026-09-10): Intel Core i9-7960X, 4 memory channels, @@ -30,6 +31,7 @@ unset CROWDB_ASAN CASES="${CHUNKIO_BENCH_CASES:-}" TIMEOUT_SECS="${CHUNKIO_BENCH_TIMEOUT:-120}" PREFETCH_CHUNKS="${CHUNKIO_PREFETCH_CHUNKS:-10}" +SKIP_BUILD="${CHUNKIO_BENCH_SKIP_BUILD:-0}" RUN_STAMP=$(date +%Y%m%d-%H%M%S) LOG_ROOT="${CHUNKIO_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/chunkio-write-regression-$RUN_STAMP}" RESULTS_FILE="${CHUNKIO_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" @@ -80,7 +82,7 @@ verify_logs() { diskdb=$(find "$CURRENT_LOG_ROOT" -type f -name 'crowdb-diskdb-metrics-*.log' | wc -l) chunkdb=$(find "$CURRENT_LOG_ROOT" -type f -name 'crowdb-chunkdb-metrics-*.log' | wc -l) diskio=$(find "$CURRENT_LOG_ROOT" -type f -name 'crowdb-diskio-metrics-*.log' | wc -l) - cli_metrics=$(find "$CURRENT_LOG_ROOT" -type f -name 'crowdb-cli-metrics-*.log' | wc -l) + cli_metrics=$(find "$CURRENT_LOG_ROOT/measurements" -type f -name 'crowdb-cli-metrics-*.log' | wc -l) cli_metrics_file=$(find "$BENCH_LOG_DIR" -type f -name 'crowdb-cli-metrics-*.log' | head -n 1) [ "$kv" -eq 3 ] && [ "$diskdb" -ge $((CASE_NUMBER * 3)) ] \ && [ "$chunkdb" -ge $((CASE_NUMBER * 3)) ] \ @@ -92,7 +94,7 @@ verify_logs() { && regression_require_metric_files 'crowdb-diskdb-metrics-*.log' rust cpp-rpc \ && regression_require_metric_files 'crowdb-chunkdb-metrics-*.log' rust cpp-rpc \ && regression_require_metric_files 'crowdb-diskio-metrics-*.log' cpp-rpc \ - && ! grep -ERq -- 'fd .*not registered|DiskNotExist' "$CURRENT_LOG_ROOT"/cli-cluster-local-deploy-*/deploy/*/*/*/log + && regression_require_no_service_errors } run_case() { @@ -115,7 +117,7 @@ run_case() { fi set +e output=$(timeout --signal=INT --kill-after=10 "$TIMEOUT_SECS" \ - pixi run -- ./target/release/crowdb-cli --log-root "$CURRENT_LOG_ROOT" --config "$CURRENT_CONFIG" \ + pixi run python tools/benchmark/fixture.py --state "$CURRENT_CONFIG" client -- \ bench chunkio write --objects "$objects" --duration-secs "$duration_secs" \ --object-size "$object_size" \ --concurrency "$concurrency" --diskio-connections 8 --data-num 8 --code-num 4 \ @@ -182,14 +184,16 @@ run_case() { fi } -echo "=== building release binaries ===" -pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb -pixi run build-cpp +if [ "$SKIP_BUILD" != 1 ]; then + echo "=== building release binaries ===" + pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb + pixi run build-cpp +fi mkdir -p "$LOG_ROOT" "$(dirname "$RESULTS_FILE")" regression_init printf 'case\trequested\tsize_bytes\tsize_mib\tconcurrency\tcompleted\terrors\tincomplete\tstop\tobjects_s\tlogical_mib_s\tphysical_mib_s\tavg_us\tp50_us\tp99_us\tmem_read_avg_mib\tmem_read_max_mib\tmem_write_avg_mib\tmem_write_max_mib\tmem_total_avg_mib\tmem_total_max_mib\n' >"$RESULTS_FILE" -cli cluster local-deploy -t combined --metrics-interval 1 --allow-unsafe-ec \ +regression_fixture setup -- --kind combined --metrics-interval 1 \ --kv-backend mem-block --wal-backend mem-block --no-fsync run_case stream_1t 16777216 1 stream diff --git a/tools/benchmark/bench-diskdb-regression.sh b/tools/benchmark/bench-diskdb-regression.sh index dc9d9328e..0d4db3b2f 100755 --- a/tools/benchmark/bench-diskdb-regression.sh +++ b/tools/benchmark/bench-diskdb-regression.sh @@ -2,7 +2,7 @@ # CrowDB diskdb allocation regression benchmark. # Usage: bash tools/benchmark/bench-diskdb-regression.sh # -# Every case uses the production CLI lifecycle: deploy three KV nodes, +# Every case uses an explicit native measurement fixture: deploy three KV nodes, # create non-system data groups, deploy one diskdb per node, provision # four logical disks per node, run the workload, and destroy processes. # All invocations share one timestamped root and retain their individual @@ -149,12 +149,12 @@ trap destroy_cluster EXIT verify_logs() { local label="$1" kv_metrics diskdb_metrics cli_metrics kv_rpc diskdb_rpc cli_rpc - kv_metrics=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/kv-server-*/log/crowdb-kv-server-metrics-*.log' -type f | wc -l) - diskdb_metrics=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/diskdb-*/log/crowdb-diskdb-metrics-*.log' -type f | wc -l) - cli_metrics=$(find "$LOG_ROOT" -path '*/cli-bench-diskdb-*/crowdb-cli-metrics-*.log' -type f | wc -l) - kv_rpc=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/kv-server-*/log/crowdb-kv-server-rpc-*.log' -type f | wc -l) - diskdb_rpc=$(find "$LOG_ROOT" -path '*/deploy/rack*/node*/diskdb-*/log/crowdb-diskdb-rpc-*.log' -type f | wc -l) - cli_rpc=$(find "$LOG_ROOT" -path '*/cli-bench-diskdb-*/crowdb-cli-rpc-*.log' -type f | wc -l) + kv_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-kv-server-metrics-*.log' -type f | wc -l) + diskdb_metrics=$(find "$LOG_ROOT" -path '*/services/rack*/node*/diskdb-*/log/crowdb-diskdb-metrics-*.log' -type f | wc -l) + cli_metrics=$(find "$LOG_ROOT" -path '*/measurements/bench-diskdb-*/crowdb-cli-metrics-*.log' -type f | wc -l) + kv_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/kv-server-*/log/crowdb-kv-server-rpc-*.log' -type f | wc -l) + diskdb_rpc=$(find "$LOG_ROOT" -path '*/services/rack*/node*/diskdb-*/log/crowdb-diskdb-rpc-*.log' -type f | wc -l) + cli_rpc=$(find "$LOG_ROOT" -path '*/measurements/bench-diskdb-*/crowdb-cli-rpc-*.log' -type f | wc -l) local expected_servers=$((CASE_NUMBER * 3)) expected_clients="$CASE_NUMBER" if [ "$kv_metrics" -ne $((DEPLOY_NUMBER * 3)) ] || [ "$diskdb_metrics" -lt "$expected_servers" ] \ || [ "$cli_metrics" -ne "$expected_clients" ] || [ "$kv_rpc" -ne $((DEPLOY_NUMBER * 3)) ] \ @@ -210,7 +210,7 @@ deploy_group() { if [ "$KV_COALESCE" -ne 0 ]; then kv_args+=(--coalesce-max-keys "$KV_COALESCE") fi - cli cluster local-deploy -n 3 -t kv "${backend_args[@]}" "${kv_args[@]}" + regression_fixture setup -- --nodes 3 --kind kv "${backend_args[@]}" "${kv_args[@]}" local groups=() for group in $(seq 1 "$DATA_GROUP_COUNT"); do cli kv group add -s 0 -g "$group" -n 1,2,3 @@ -222,7 +222,7 @@ deploy_group() { if [ "$free_batch" = "on" ]; then free_batch_args=(--free-batch --free-flush-max-batch 256) fi - cli cluster local-deploy -t diskdb --data-groups "$group_csv" \ + regression_fixture setup -- --kind diskdb --data-groups "$group_csv" \ --rpc-workers "$DDB_RPC_WORKERS" --kv-connections "$KV_CONNECTIONS" \ --kv-client-rpc-workers "$KV_CLIENT_WORKERS" \ --disk-groups-per-node 1 --disks-per-group 4 \ @@ -259,12 +259,12 @@ run_case() { epoll_workers="$DDB_RPC_WORKERS" set +e output=$(timeout --signal=INT --kill-after=10 "$((DURATION + 40))" \ - pixi run -- ./target/release/crowdb-cli --log-root "$LOG_ROOT" --config "$CURRENT_CONFIG" \ + pixi run python tools/benchmark/fixture.py --state "$CURRENT_CONFIG" client -- \ bench diskdb "$workload" --duration-secs "$DURATION" \ --concurrency "$concurrency" --unit-count 1 --blocks-per-request "$blocks" \ --diskdb-connections "$DDB_CONNECTIONS" \ --diskdb-client-rpc-workers "$DDB_CLIENT_WORKERS" \ - --mode "$mode" --seed 1 --metrics-interval 1 2>&1) + --seed 1 --metrics-interval 1 2>&1) status=$? set -e printf '%s\n' "$output" @@ -317,6 +317,7 @@ run_case() { echo "=== building release binaries ===" pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb +regression_init mkdir -p "$LOG_ROOT" "$(dirname "$RESULTS_FILE")" printf 'Wl\tGrp\tThr\tBlk\tCli\tDdb\tKv\tWkr\tWin\tCoal\tops/s\tavg\tp50\tp99\tDur\tErr\tSpc\tFB\tFReq\tFKV\tFCo\n' >"$RESULTS_FILE" diff --git a/tools/benchmark/bench-kv-read-regression.sh b/tools/benchmark/bench-kv-read-regression.sh index 7abc8674e..ab8933859 100755 --- a/tools/benchmark/bench-kv-read-regression.sh +++ b/tools/benchmark/bench-kv-read-regression.sh @@ -64,16 +64,19 @@ run_subtest() { fi echo ">>> $display ..." local output - output=$(pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ + if ! output=$(regression_client_for "$CONFIG_FILE" \ bench kv read --store 0 --group 1 --duration-secs "$DURATION" \ --loader-num "$threads" --connections "$connections" \ --read-mode "$read_mode" --min-slot "$min_slot" \ --read-endpoint-policy "$read_endpoint" \ --key-space "$KEYSPACE" \ - --value-size 64 --verify-bytes "$verify_bytes" --json 2>&1) - local json; json=$(echo "$output" | sed -n '/^{/,/^}/p') + --value-size 64 --verify-bytes "$verify_bytes" 2>&1); then + echo "$output" >&2 + return 1 + fi + local json; json=$(regression_report_for "$CONFIG_FILE") if [ -z "$json" ]; then - echo " ERROR: no JSON output"; echo "$output" | tail -5 + echo " ERROR: no JSON output"; echo "$output" echo -e "$label\t$read_mode\t${threads}T${connections}C\t$verify_bytes\t0\t0\t0\t0\t0\t0\t1\t1" >> "$RESULTS_FILE" return fi @@ -82,7 +85,7 @@ run_subtest() { avg_us=$(echo "$json" | jq -r '.by_op.read.latency_us.avg_us') p50_us=$(echo "$json" | jq -r '.by_op.read.latency_us.p50_us') p99_us=$(echo "$json" | jq -r '.by_op.read.latency_us.p99_us') - p999_us=$(echo "$json" | jq -r '.by_op.read.latency_us.p999_us') + p999_us=$(echo "$json" | jq -r '.by_op.read.latency_us.p999_us // "unavailable"') errors=$(echo "$json" | jq -r '.total_errors') corr_err=$(echo "$json" | jq -r '.correctness_errors') echo " ops/s=$ops_s avg=${avg_us}us p50=${p50_us}us p99=${p99_us}us p999=${p999_us}us err=$errors corr=$corr_err" @@ -155,23 +158,21 @@ run_subtest() { echo -e "label\tread_mode\tT:C\tverify\tops_s\tavg_us\tp50_us\tp99_us\tp999_us\terrors\tcorrectness_errors" > "$RESULTS_FILE" -# Phase 1: deploy the cluster once via `cluster local-deploy`. -echo "=== Deploying 3-node KV cluster via local-deploy ===" +# Phase 1: deploy the cluster once via the explicit test fixture. +echo "=== Deploying 3-node KV cluster via the explicit test fixture ===" CONFIG_FILE="$REGRESSION_CONFIG" -pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - cluster local-deploy -n 3 -t kv \ +regression_fixture_for "$CONFIG_FILE" setup -- --nodes 3 --kind kv \ --kv-backend mem-block --wal-backend mem-block # Create bench group 1 (group 0 sysdata preserved). echo "=== Creating bench group 0/1 (group 0 sysdata preserved) ===" -pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - kv group add -s 0 -g 1 -n 1,2,3 2>&1 | tail -3 +regression_client_for "$CONFIG_FILE" \ + kv group add -s 0 -g 1 -n 1,2,3 2>&1 -# Phase 2: pre-populate keys once (Phase 3: bench prepare not yet wired). -echo "=== Pre-populating $KEYSPACE keys (Phase 3: bench prepare stub) ===" -pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - bench kv prepare --store 0 --group 1 --keys "$KEYSPACE" --value-size 64 2>&1 || \ - echo "WARNING: bench prepare is a Phase 3 stub — skipping pre-pop" +# Phase 2: pre-populate keys once (prepare data before measuring). +echo "=== Pre-populating $KEYSPACE keys ===" +regression_client_for "$CONFIG_FILE" \ + bench kv prepare --store 0 --group 1 --keys "$KEYSPACE" --value-size 64 2>&1 # Phase 3: run all sub-tests against the same cluster. echo "=== Single-thread (1T:1C) — per-read engine cost ===" @@ -190,10 +191,9 @@ echo "=== Correctness verification ===" run_subtest "lin_16t_verify" linearizable auto 16 16 8 run_subtest "minslot_16t_verify" minslot zero 16 16 8 -# Phase 4: teardown via `cluster destroy`. +# Phase 4: teardown via owned fixture shutdown. echo "=== Tearing down cluster ===" -pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - cluster destroy +regression_fixture_for "$CONFIG_FILE" stop echo "=== DONE ===" echo "Results in $RESULTS_FILE" diff --git a/tools/benchmark/bench-kv-scan-regression.sh b/tools/benchmark/bench-kv-scan-regression.sh index e2d26f797..49d83c167 100755 --- a/tools/benchmark/bench-kv-scan-regression.sh +++ b/tools/benchmark/bench-kv-scan-regression.sh @@ -71,7 +71,7 @@ run_subtest() { fi echo ">>> $label (${threads}T:${connections}C) ..." local output - output=$(pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ + if ! output=$(regression_client_for "$CONFIG_FILE" \ bench kv scan --store 0 --group 1 --duration-secs "$DURATION" \ --loader-num "$threads" --connections "$connections" \ --read-mode "$read_mode" --min-slot "$min_slot" \ @@ -79,10 +79,13 @@ run_subtest() { --scan-limit "$limit" --scan-prefix "$prefix" --scan-start-after "$start_after" \ --scan-direction "$direction" \ --value-size "$value_size" \ - --key-space "$KEYSPACE" --verify-bytes 0 --json $mix_arg 2>&1) - local json; json=$(echo "$output" | sed -n '/^{/,/^}/p') + --key-space "$KEYSPACE" --verify-bytes 0 $mix_arg 2>&1); then + echo "$output" >&2 + return 1 + fi + local json; json=$(regression_report_for "$CONFIG_FILE") if [ -z "$json" ]; then - echo " ERROR: no JSON output"; echo "$output" | tail -5 + echo " ERROR: no JSON output"; echo "$output" echo -e "$label\t$limit\t$prefix\t$start_after\t$value_size\t$read_mode\t$direction\t${threads}T${connections}C\t0\t0\t0\t0\t0\t1" >> "$RESULTS_FILE" return fi @@ -91,7 +94,7 @@ run_subtest() { avg_us=$(echo "$json" | jq -r '.by_op.list.latency_us.avg_us') p50_us=$(echo "$json" | jq -r '.by_op.list.latency_us.p50_us') p99_us=$(echo "$json" | jq -r '.by_op.list.latency_us.p99_us') - p999_us=$(echo "$json" | jq -r '.by_op.list.latency_us.p999_us') + p999_us=$(echo "$json" | jq -r '.by_op.list.latency_us.p999_us // "unavailable"') errors=$(echo "$json" | jq -r '.total_errors') echo " scans/s=$ops_s avg=${avg_us}us p50=${p50_us}us p99=${p99_us}us p999=${p999_us}us err=$errors" echo -e "$label\t$limit\t$prefix\t$start_after\t$value_size\t$read_mode\t$direction\t${threads}T${connections}C\t$ops_s\t$avg_us\t$p50_us\t$p99_us\t$p999_us\t$errors" >> "$RESULTS_FILE" @@ -172,23 +175,21 @@ run_subtest() { echo -e "label\tlimit\tprefix\tstart_after\tvalue_size\tread_mode\tdirection\tT:C\tscans_s\tavg_us\tp50_us\tp99_us\tp999_us\terrors" > "$RESULTS_FILE" -# Phase 1: deploy the cluster once via `cluster local-deploy`. -echo "=== Deploying 3-node KV cluster via local-deploy ===" +# Phase 1: deploy the cluster once via the explicit test fixture. +echo "=== Deploying 3-node KV cluster via the explicit test fixture ===" CONFIG_FILE="$REGRESSION_CONFIG" -pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - cluster local-deploy -n 3 -t kv \ +regression_fixture_for "$CONFIG_FILE" setup -- --nodes 3 --kind kv \ --kv-backend mem-block --wal-backend mem-block # Create bench group 1 (group 0 sysdata preserved). echo "=== Creating bench group 0/1 (group 0 sysdata preserved) ===" -pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - kv group add -s 0 -g 1 -n 1,2,3 2>&1 | tail -3 +regression_client_for "$CONFIG_FILE" \ + kv group add -s 0 -g 1 -n 1,2,3 2>&1 -# Phase 2: pre-populate keys once (Phase 3: bench prepare not yet wired). -echo "=== Pre-populating $KEYSPACE keys (Phase 3: bench prepare stub) ===" -pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - bench kv prepare --store 0 --group 1 --keys "$KEYSPACE" --value-size 64 2>&1 || \ - echo "WARNING: bench prepare is a Phase 3 stub — skipping pre-pop" +# Phase 2: pre-populate keys once (prepare data before measuring). +echo "=== Pre-populating $KEYSPACE keys ===" +regression_client_for "$CONFIG_FILE" \ + bench kv prepare --store 0 --group 1 --keys "$KEYSPACE" --value-size 64 2>&1 # Phase 3: run all sub-tests against the same cluster. echo "=== Single-thread (1T:1C) — per-scan engine cost ===" @@ -222,10 +223,9 @@ run_subtest "minslot_16t" 1000 "" "" 64 minslot run_subtest "lin_32t" 1000 "" "" 64 linearizable auto 32 32 forward run_subtest "minslot_32t" 1000 "" "" 64 minslot zero 32 32 forward -# Phase 4: teardown via `cluster destroy`. +# Phase 4: teardown via owned fixture shutdown. echo "=== Tearing down cluster ===" -pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - cluster destroy +regression_fixture_for "$CONFIG_FILE" stop echo "=== DONE ===" echo "Results in $RESULTS_FILE" diff --git a/tools/benchmark/bench-kv-write-regression.sh b/tools/benchmark/bench-kv-write-regression.sh index 612c3c043..7b407a1f9 100644 --- a/tools/benchmark/bench-kv-write-regression.sh +++ b/tools/benchmark/bench-kv-write-regression.sh @@ -112,11 +112,10 @@ run_bench() { local rss_pre_clean; rss_pre_clean=$(sample_rss "$config_file" "pre-clean") # Clean: wipe user data on bench group (group 0 sysdata preserved). local clean_out - clean_out=$(pixi run -- cargo run --release -p crowdb-cli -- --config "$config_file" \ - cluster clean --store 0 --group 1 --json 2>&1) + clean_out=$(regression_fixture_for "$config_file" reset --store 0 --group 1 2>&1) local clean_json; clean_json=$(echo "$clean_out" | sed -n '/^{/,/^}/p') if [ -z "$clean_json" ] || ! echo "$clean_json" | jq -e '.new_leader' >/dev/null 2>&1; then - echo " ERROR: clean failed"; echo "$clean_out" | tail -5 + echo " ERROR: clean failed"; echo "$clean_out" append_failed_row "$label" "$duration" "$keyspace" "$value_size" return fi @@ -125,20 +124,21 @@ run_bench() { local clean_delta=$((rss_pre_clean - rss_post_clean)) echo " rss clean delta: ${clean_delta}MB freed" local output bench_status=0 - output=$(pixi run -- cargo run --release -p crowdb-cli -- --config "$config_file" \ + output=$(regression_client_for "$config_file" \ bench kv write --store 0 --group 1 --duration-secs "$duration" \ --loader-num "$threads" --connections "$conn" \ --key-space "$keyspace" --value-size "$value_size" \ --event-write --rpc-workers "${RPC_WORKERS:-2}" \ - --verify-bytes 0 --json 2>&1 | tee "$LOG_ROOT/$label-cli.log") || bench_status=$? + --verify-bytes 0 2>&1 | tee "$LOG_ROOT/$label-cli.log") || bench_status=$? if [ "$bench_status" -ne 0 ]; then echo " ERROR: benchmark command exited with status $bench_status" + echo "$output" >&2 append_failed_row "$label" "$duration" "$keyspace" "$value_size" return fi - local json; json=$(echo "$output" | sed -n '/^{/,/^}/p') + local json; json=$(regression_report_for "$config_file") if [ -z "$json" ]; then - echo " ERROR: no JSON output"; echo "$output" | tail -5 + echo " ERROR: no JSON output"; echo "$output" append_failed_row "$label" "$duration" "$keyspace" "$value_size" return fi @@ -196,7 +196,7 @@ run_bench() { } # deploy_group -# Deploy a 3-node cluster with the given server tunables via local-deploy, +# Deploy a 3-node cluster with the given server tunables via the explicit test fixture, # then create a bench group (store 0, group 1) so benchmarks don't touch # group 0 sysdata. deploy_group() { @@ -204,15 +204,14 @@ deploy_group() { local config_file="$LOG_ROOT/${name}-console.toml" echo "=== deploying cluster '$name' (win=$win, coalesce=$coalesce, workers=$workers) ===" rm -f "$config_file" - pixi run -- cargo run --release -p crowdb-cli -- --config "$config_file" \ - cluster local-deploy -n 3 -t kv \ + regression_fixture_for "$config_file" setup -- --nodes 3 --kind kv \ --event-write --peer-pool-size 4 \ --max-inflight "$win" --coalesce-max-keys "$coalesce" \ --rpc-workers "$workers" \ - --kv-backend mem-block --wal-backend mem-block 2>&1 | tail -3 + --kv-backend mem-block --wal-backend mem-block 2>&1 echo "=== creating bench group 0/1 (group 0 sysdata preserved) ===" - pixi run -- cargo run --release -p crowdb-cli -- --config "$config_file" \ - kv group add -s 0 -g 1 -n 1,2,3 2>&1 | tail -3 + regression_client_for "$config_file" \ + kv group add -s 0 -g 1 -n 1,2,3 2>&1 # Store config path for run_bench/teardown_group. echo "$config_file" > "$LOG_ROOT/${name}.cfgpath" # Baseline RSS right after deploy (before any sub-test). @@ -225,8 +224,7 @@ teardown_group() { local config_file config_file=$(cat "$LOG_ROOT/${name}.cfgpath" 2>/dev/null || echo "") if [ -n "$config_file" ] && [ -f "$config_file" ]; then - pixi run -- cargo run --release -p crowdb-cli -- --config "$config_file" \ - cluster destroy 2>&1 | tail -2 + regression_fixture_for "$config_file" stop 2>&1 rm -f "$LOG_ROOT/${name}.cfgpath" fi } @@ -242,7 +240,7 @@ teardown_group() { # sagg/ragg columns are 0 (moved to crowdb-common histograms). p50/p99 # use coarser histogram buckets (500us increments) — not directly # comparable to the 2026-08-27 exact values. 128T and 256T show -# r2=0us/r3=0us (replicas not responding) after `cluster clean` — +# r2=0us/r3=0us (replicas not responding) after the fixture data reset — # consensus instability from the clean→run transition, not storage- # related. A standalone compare-128t run (fresh deploy, no prior # sub-tests) gets 198K ops/s with co=15.75/16, confirming the storage @@ -264,7 +262,7 @@ teardown_group() { # and calls remove_group BEFORE create_group_with_wal. Bench runs on # group 1 (group 0 sysdata preserved). The 256T consensus instability # is fixed: 22K→181K ops/s, 256→0 errors. r2/r3 now respond at all -# thread counts. cluster clean frees RSS between sub-tests (128MB–9.5GB +# thread counts. fixture data reset frees RSS between sub-tests (128MB–9.5GB # freed). 512T slightly lower (264K→240K) — within run-to-run noise. # # T C W win co ops/s WAL/node p50 p99 err sagg ragg r2 r2tps r3 r3tps enq wait @@ -324,7 +322,7 @@ teardown_group() { # (cargo skips unchanged crates); this only recompiles if a prior # sanitize run left an ASan-instrumented artifact in target/release. echo "=== building release (CROWDB_ASAN unset) ===" -pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server 2>&1 | tail -3 +pixi run -- cargo build --release -p crowdb-cli -p crowdb-kv-server 2>&1 echo -e "label\twin\tcoalesce\tworkers\tops_s\twal_per_node\tavg_us\tp50_us\tp99_us\terrors\tsrv_sagg\tsrv_ragg\tcli_sagg\tcli_ragg\tr2_avg\tr2_tps\tr3_avg\tr3_tps\tinflight_enq\tinflight_wait_us\tduration_s\tkeyspace\tvalue_size\tcorrectness_errors\tsnapshot_completed\tsnapshot_failed\tsnapshot_max_us\telection_delta" > "$RESULTS_FILE" diff --git a/tools/benchmark/bench-leak-inspect.sh b/tools/benchmark/bench-leak-inspect.sh index 3557f0d7e..78c9f4117 100644 --- a/tools/benchmark/bench-leak-inspect.sh +++ b/tools/benchmark/bench-leak-inspect.sh @@ -10,22 +10,25 @@ KEYSPACE=1000000 VALUE_SIZE=512 DEPLOY="leak-inspect-$$-$(date +%s)" RUNTIME_ROOT="${CROWDB_RUNTIME_ROOT:-${PIXI_PROJECT_ROOT:-$(pwd)}/.crowdb-runtime}" -mkdir -p "$RUNTIME_ROOT/ephemeral/bench-leak-inspect" -CFG="$RUNTIME_ROOT/ephemeral/bench-leak-inspect/${DEPLOY}.toml" +REGRESSION_LOG_ROOT="$RUNTIME_ROOT/artifacts/bench/leak-inspect-$DEPLOY" +CFG="$REGRESSION_LOG_ROOT/console.toml" BENCH_STORE=0 BENCH_GROUP=1 +REGRESSION_CONFIG="$CFG" +source tools/benchmark/bench-regression-common.sh +regression_init +trap regression_destroy EXIT echo "=== deploying cluster (win=32, coalesce=16, workers=2, MALLOC_ARENA_MAX=1 + TRIM_THRESHOLD=0) ===" -MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=0 pixi run -- cargo run --release -p crowdb-cli -- --config "$CFG" \ - cluster local-deploy -n 3 -t kv \ +MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=0 regression_fixture_for "$CFG" setup -- --nodes 3 --kind kv \ --event-write --peer-pool-size 4 \ --max-inflight 32 --coalesce-max-keys 16 \ --rpc-workers 2 \ - --kv-backend mem-block --wal-backend mem-block 2>&1 | tail -3 + --kv-backend mem-block --wal-backend mem-block 2>&1 echo "=== creating bench group $BENCH_STORE/$BENCH_GROUP (group 0 sysdata preserved) ===" -pixi run -- cargo run --release -p crowdb-cli -- --config "$CFG" \ - kv group add -s "$BENCH_STORE" -g "$BENCH_GROUP" -n 1,2,3 2>&1 | tail -3 +regression_client_for "$CFG" \ + kv group add -s "$BENCH_STORE" -g "$BENCH_GROUP" -n 1,2,3 2>&1 sample_rss() { local label="$1" @@ -54,16 +57,15 @@ run_sub() { local threads="$1" conn="$2" label="$3" workers="${4:-2}" echo ">>> $label ..." sample_rss "pre-clean" - MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=0 pixi run -- cargo run --release -p crowdb-cli -- --config "$CFG" \ - cluster clean --store "$BENCH_STORE" --group "$BENCH_GROUP" --json 2>&1 | tail -1 + MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=0 regression_fixture_for "$CFG" reset --store "$BENCH_STORE" --group "$BENCH_GROUP" 2>&1 sample_rss "post-clean" - MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=0 pixi run -- cargo run --release -p crowdb-cli -- --config "$CFG" \ + MALLOC_ARENA_MAX=1 MALLOC_TRIM_THRESHOLD_=0 regression_client_for "$CFG" \ bench kv write --store "$BENCH_STORE" --group "$BENCH_GROUP" \ --duration-secs "$DURATION" \ --loader-num "$threads" --connections "$conn" \ --key-space "$KEYSPACE" --value-size "$VALUE_SIZE" \ --event-write --rpc-workers "$workers" \ - --verify-bytes 0 --json 2>&1 | tail -1 + --verify-bytes 0 2>&1 sample_rss "post-bench" } @@ -74,7 +76,7 @@ run_sub 64 4 "write_64t" 2 run_sub 128 4 "write_128t" 4 echo "" -echo "=== DONE — servers kept alive for inspection ===" +echo "=== Measurements complete — servers remain owned while inspecting ===" echo "Config: $CFG" echo "Server PIDs:" grep '^pid' "$CFG" | awk '{print " pid=" $3}' @@ -88,3 +90,5 @@ for pid in $(grep '^pid' "$CFG" | awk '{print $3}'); do echo " pid=$pid RSS=${local_mb}MB" fi done +echo "Inspect the services now; Ctrl-C stops this fixture and its owned services." +while sleep 60; do :; done diff --git a/tools/benchmark/bench-regression-common.sh b/tools/benchmark/bench-regression-common.sh index 0e8d3796c..1169f1cc7 100644 --- a/tools/benchmark/bench-regression-common.sh +++ b/tools/benchmark/bench-regression-common.sh @@ -12,20 +12,47 @@ fi REGRESSION_CONFIG="${REGRESSION_CONFIG:-$REGRESSION_LOG_ROOT/console.toml}" REGRESSION_CLI="${REGRESSION_CLI:-./target/release/crowdb-cli}" +export REGRESSION_CLI regression_init() { mkdir -p "$REGRESSION_LOG_ROOT" + if [ -z "${CROWDB_REGRESSION_FIXTURE_BIN:-}" ]; then + pixi run cargo build --release -p crowdb-console-shared --example regression-fixture --features test-util + fi } regression_cli() { - pixi run -- "$REGRESSION_CLI" --log-root "$REGRESSION_LOG_ROOT" \ - --config "$REGRESSION_CONFIG" "$@" + regression_client_for "$REGRESSION_CONFIG" "$@" +} + +regression_client_for() { + local state="$1" + shift + REGRESSION_CLI="$REGRESSION_CLI" pixi run python tools/benchmark/fixture.py \ + --state "$state" client -- "$@" +} + +regression_fixture_for() { + local state="$1" + shift + pixi run python tools/benchmark/fixture.py --state "$state" --owner-pid "$$" "$@" +} + +regression_fixture() { + regression_fixture_for "$REGRESSION_CONFIG" "$@" +} + +regression_fixture_log_root() { + pixi run python tools/benchmark/fixture.py --state "$REGRESSION_CONFIG" root +} + +regression_report_for() { + pixi run python tools/benchmark/fixture.py --state "$1" report } regression_destroy() { if [ -f "$REGRESSION_CONFIG" ]; then - timeout 30 pixi run -- "$REGRESSION_CLI" --log-root "$REGRESSION_LOG_ROOT" \ - --config "$REGRESSION_CONFIG" cluster destroy + regression_fixture stop fi } @@ -39,9 +66,9 @@ regression_reset_stack() { last=$((${#groups[@]} - 1)) for index in "${!groups[@]}"; do if [ "$index" -eq "$last" ]; then - regression_cli cluster clean --store 0 --group "${groups[$index]}" --restart-services + regression_fixture reset --store 0 --group "${groups[$index]}" --restart-storage else - regression_cli cluster clean --store 0 --group "${groups[$index]}" + regression_fixture reset --store 0 --group "${groups[$index]}" fi done } @@ -74,3 +101,14 @@ regression_require_metric_files() { return 1 fi } + +regression_require_no_service_errors() { + local root status=0 + root=$(regression_fixture_log_root) || return 1 + grep -ERn --include='*.log' -- 'fd .*not registered|DiskNotExist' "$root/services" || status=$? + # One means no log matched. Matching errors and scan failures both fail. + if [ "$status" -ne 1 ]; then + echo "ERROR: service log validation failed (grep status $status)" >&2 + return 1 + fi +} diff --git a/tools/benchmark/bench-rpc-regression.sh b/tools/benchmark/bench-rpc-regression.sh index 770536c88..c5864addb 100755 --- a/tools/benchmark/bench-rpc-regression.sh +++ b/tools/benchmark/bench-rpc-regression.sh @@ -1,222 +1,44 @@ #!/usr/bin/env bash -# CrowdbRPC echo regression benchmark. -# Usage: bash tools/benchmark/bench-rpc-regression.sh -# -# Starts a standalone crowdb-rpc-fb-server (built via `pixi run build-cpp`), -# then runs crowdb-cli bench rpc against it for each config. The server is -# restarted per config so its io_engines/io_workers match the client. -# -# Server lifecycle is managed by `cluster local-deploy -t rpc` / `cluster -# reset` — the CLI handles spawn, readiness, PID tracking, and teardown. -# -# After a run, update doc/design/rpc/rpc-flow-analysis.md: add a -# dated subsection under "Current Data" with the results table and -# scaling analysis, plus a "History" entry. Always record the CPU model. -# -# Regression policy: only update the reference tables below when a new -# run is strictly better (higher ops/s, lower latency, fewer errors). -# If a run is worse, do NOT update — investigate and fix the regression -# first, otherwise silent performance regressions slip in. -# -# macOS (2026-08-21): Apple M5 Pro, 18c, arm64, macOS 26, 128B, 20s, -# standalone server over kqueue loopback. After send() unification + -# global static counters (no per-instance atomics on hot path). -# Eng Wkr T C ops/s avg p50 p99 p999 raggr saggr err -# 1 1 1 1 49,445 19 19 29 49 1.0 1.0 0 -# 1 4 64 4 558,326 112 106 231 299 2.2 6.8 0 -# 1 8 512 8 900,017 564 503 1,446 3,938 2.9 12.5 0 -# 2 8 512 8 927,537 547 521 951 3,630 2.9 11.1 0 -# 1 16 1,000 32 722,644 1,372 1,009 5,484 14,384 7.2 9.3 0 -# 2 16 1,000 16 900,252 1,099 851 4,012 9,056 6.3 13.0 0 -# -# AMD (2026-08-27): Ryzen 9 5950X, 16c/32t, Linux 6.8, 128B, 20s, -# standalone server over epoll loopback. Same build as 2026-08-25 plus -# metrics cv shutdown fix (condition_variable wake in -# MetricsRegistry::stop — 5s→60ms shutdown). Single-engine only (2-engine -# configs removed). Nagle on = TCP coalescing (multiple small frames per -# writev/read syscall). Nagle gives +62-113% throughput and 4x lower p99 -# at high concurrency — bursty coroutine workloads are syscall-bound. -# raggr/saggr columns dropped (frames_sent/frames_parsed moved to -# crowdb-common metrics histograms); nagle column added. -# Rows updated 2026-08-31 where strictly better (per-call control -# flatbuffer fix — request_id now embedded in ConnectionPingRequest.id -# per call, enabling correct slab slot correlation). -# Coroutine (nagle off): -# Eng Wkr T C ops/s avg p50 p99 p999 nagle err -# 1 1 1 1 52,759 17 17 25 31 0 0 -# 1 4 64 4 517,687 122 94 96 260 0 0 -# 1 8 512 8 830,174 614 486 505 1,074 0 0 -# 1 16 1,000 32 1,307,488 762 302 324 9,645 0 0 -# Coroutine (nagle on): -# 1 4 64 4 983,245 63 59 162 520 1 0 -# 1 8 512 8 1,869,083 271 209 219 3,993 1 0 -# 1 16 1,000 32 2,213,182 448 157 171 1,999 1 0 -# Tokio (nagle off): -# 1 1 1 1 29,367 32 24 25 71 0 0 -# 1 4 64 4 557,362 113 105 265 425 0 6 -# 1 16 1,000 32 849,575 1,156 1,063 2,652 3,838 0 591 -# -# AMD (2026-08-28): same hw/build as 2026-08-27. TCP_QUICKACK decoupled -# from Nagle into a separate --quickack flag. QUICKACK adds a setsockopt -# per read + more ACK packets — hurts the RPC echo workload (continuous -# request stream, Nagle never stalls). Only the 1000T/32C config tested. -# Eng Wkr T C ops/s avg p50 p99 p999 nagle qa err -# 1 16 1,000 32 1,209,549 823 413 7,280 12,720 0 0 0 -# 1 16 1,000 32 1,949,603 508 363 2,142 3,250 1 0 0 -# 1 16 1,000 32 1,731,709 573 471 1,893 2,764 1 1 0 -# 1 16 1,000 32 1,206,111 825 420 5,844 9,600 0 1 0 -# Conclusion: nagle on, quickack off is optimal for RPC echo (-11% vs -# nagle+quickack). QUICKACK is for Paxos (KV bench), not raw RPC echo. -# -# 2026-09-02 (same hw, +page-count metrics +flush re-check loop): -# RPC echo does not touch the storage engine — this run confirms the -# changes have no transport-level impact. Results are within noise -# (-1% to -5% vs 2026-08-31). p50/p99 use coarser histogram buckets -# (100/500/1000/5000us) — not directly comparable to 2026-08-31 exact -# values. Not strictly better — reference NOT updated per regression -# policy. -# -# Wkr Load Mode Nagle ops/s avg p50 p99 p999 err -# 1 1T:1C coroutine off 50,663 18 100 100 null 0 -# 4 64T:4C coroutine off 514,318 123 500 500 null 0 -# 8 512T:8C coroutine off 814,807 626 1000 1000 null 0 -# 16 1,000T:32C coroutine off 1,271,548 783 500 10000 null 0 -# 4 64T:4C coroutine on 973,317 64 100 500 null 0 -# 8 512T:8C coroutine on 1,775,602 286 500 500 null 0 -# 16 1,000T:32C coroutine on 2,130,032 466 500 5000 null 0 -# 1 1T:1C tokio off 28,062 34 100 100 null 0 -# 4 64T:4C tokio off 492,924 126 500 500 null 0 -# 16 1,000T:32C tokio off 988,956 999 1000 5000 null 0 -# -# Intel i9-7960X (2026-09-10, 16c/32t, Linux 6.11, x86_64): -# Same build/config as AMD 2026-09-02. 128B, 20s, standalone server -# over epoll loopback. Zero errors across all configs. 1T:1C 42-63% -# slower (per-op overhead higher on Intel). 64T+ within 3-25%, with -# 512T/1000T coroutine nagle-off and 1000T tokio actually FASTER on -# Intel (higher memory bandwidth at saturation). p50/p99 are exact -# (not coarser 100/500us buckets like AMD). Gaps > 30% documented in -# doc/working/regression-perf-review.md. -# -# Wkr Load Mode Nagle ops/s avg p50 p99 p999 err -# 1 1T:1C coroutine off 29,231 32 65 66 null 0 -# 1 1T:1C tokio off 10,411 90 88 146 null 0 -# 4 64T:4C coroutine off 455,572 139 112 372 null 0 -# 4 64T:4C tokio off 369,145 169 139 610 null 0 -# 8 512T:8C coroutine off 891,668 572 475 1310 null 0 -# 16 1,000T:32C coroutine off 1,422,928 699 342 5734 null 0 -# 16 1,000T:32C tokio off 1,045,067 950 749 3719 null 0 -# 4 64T:4C coroutine on 762,575 82 71 225 null 0 -# 8 512T:8C coroutine on 1,501,914 338 268 1171 null 0 -# 16 1,000T:32C coroutine on 2,056,895 482 364 1720 null 0 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +# Real echo transport regression; the test fixture owns each server lifetime. set -euo pipefail cd "$(dirname "$0")/../.." - +unset CROWDB_ASAN RUN_STAMP=$(date +%Y%m%d-%H%M%S) LOG_ROOT="${RPC_BENCH_LOG_ROOT:-${CROWDB_RUNTIME_ROOT:-$(pwd)/.crowdb-runtime}/artifacts/bench/rpc-regression-$RUN_STAMP}" RESULTS_FILE="${RPC_BENCH_RESULTS:-$LOG_ROOT/results.tsv}" -REGRESSION_LOG_ROOT="$LOG_ROOT" -source tools/benchmark/bench-regression-common.sh -export CROWDB_LOG_ROOT="$LOG_ROOT" -regression_init DURATION="${RPC_BENCH_DURATION:-20}" VALUE_SIZE="${RPC_BENCH_VALUE_SIZE:-128}" CASES="${RPC_BENCH_CASES:-}" -CONFIG_FILE="$REGRESSION_CONFIG" - -# Deploy a fresh fb-server via `cluster local-deploy -t rpc`. -# Echoes the allocated port on stdout. -start_server() { - local io_engines="$1" io_workers="$2" nagle="$3" - local nagle_arg="" - if [ "$nagle" = "1" ]; then - nagle_arg="--enable-nagle" - fi - echo " [server] cluster local-deploy -t rpc --io-engines=$io_engines --io-workers=$io_workers $nagle_arg" >&2 - local output - output=$(pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - cluster local-deploy -t rpc \ - --io-engines "$io_engines" --io-workers "$io_workers" $nagle_arg 2>&1) - echo "$output" >&2 - # Parse "port=NNNNN" from "local-deploy rpc: port=NNNNN, pid=...". - local port - port=$(echo "$output" | grep -oP 'port=\K[0-9]+' | head -1) - if [ -z "$port" ]; then - echo " [server] ERROR: could not parse port from local-deploy output" - return 1 - fi - echo "$port" -} - -# Stop the fb-server via `cluster destroy`. -stop_server() { - if [ -f "$CONFIG_FILE" ]; then - pixi run -- cargo run --release -p crowdb-cli -- --config "$CONFIG_FILE" \ - cluster destroy >&2 2>&1 || true - fi -} +FAILURES=0 +mkdir -p "$LOG_ROOT" +pixi run cargo build --release -p crowdb-rpc-test +pixi run cmake --build lib/crowdb-rpc/build -j 4 --target crowdb-rpc-fb-server +printf 'label\twkr\tops_s\tavg_us\tp50_us\tp99_us\tnagle\terrors\n' > "$RESULTS_FILE" run_bench() { - local loaders="$1" conn="$2" label="$3" io_engines="${4:-1}" wkr="${5:-1}" mode="${6:-coroutine}" nagle="${7:-0}" - if [ -n "$CASES" ] && [[ " $CASES " != *" $label "* ]]; then - return - fi - echo ">>> $label (io_engines=$io_engines, io_workers=$wkr, mode=$mode, nagle=$nagle) ..." - - # Start fb server with matching config. - local server_port - server_port=$(start_server "$io_engines" "$wkr" "$nagle") || { - echo -e "$label\t$wkr\t0\t0\t0\t0\t0\t$nagle\t1" >> "$RESULTS_FILE" - return - } - echo " [server] listening on port=$server_port" - - # Build and print the full client command. - local nagle_flag="" - if [ "$nagle" = "1" ]; then - nagle_flag="--enable-nagle" - fi - local client_cmd="pixi run -- ./target/release/crowdb-cli bench rpc \ - --duration-secs $DURATION \ - --loader-num $loaders --connections $conn \ - --value-size $VALUE_SIZE \ - --io-engines $io_engines --io-workers $wkr \ - --mode $mode \ - --server-port $server_port \ - $nagle_flag \ - --json" - echo " [client] $client_cmd" - - local output - output=$(eval "$client_cmd" 2>&1) - local json; json=$(echo "$output" | sed -n '/^{/,/^}/p') - if [ -z "$json" ]; then - echo " ERROR: no JSON output"; echo "$output" | tail -5 - echo -e "$label\t$wkr\t0\t0\t0\t0\t0\t$nagle\t1" >> "$RESULTS_FILE" - stop_server + local loaders="$1" connections="$2" label="$3" engines="${4:-1}" workers="${5:-1}" mode="${6:-coroutine}" nagle="${7:-0}" + if [ -n "$CASES" ] && [[ " $CASES " != *" $label "* ]]; then return; fi + local options=() code=0 output + if [ "$nagle" = 1 ]; then options+=(--enable-nagle); fi + echo ">>> $label" + output=$(pixi run python tools/rpc-test/run.py --log-dir "$LOG_ROOT/$label" \ + --duration-secs "$DURATION" --loader-num "$loaders" --connections "$connections" \ + --value-size "$VALUE_SIZE" --io-engines "$engines" --io-workers "$workers" \ + --mode "$mode" "${options[@]}") || code=$? + printf '%s\n' "$output" > "$LOG_ROOT/$label.json" + if [ "$code" -ne 0 ] || ! jq -e '.total_ops > 0 and .total_errors == 0' <<< "$output" >/dev/null; then + printf '%s\n' "$output" >&2 + echo "ERROR: $label failed (exit=$code)" >&2 + FAILURES=$((FAILURES + 1)) return fi - local ops_s avg_us p50_us p99_us p999_us errors - ops_s=$(echo "$json" | jq -r '.total_ops * 1000 / .duration_ms' | awk '{printf "%.0f", $1}') - avg_us=$(echo "$json" | jq -r '.by_op.write.latency_us.avg_us') - p50_us=$(echo "$json" | jq -r '.by_op.write.latency_us.p50_us') - p99_us=$(echo "$json" | jq -r '.by_op.write.latency_us.p99_us') - p999_us=$(echo "$json" | jq -r '.by_op.write.latency_us.p999_us') - errors=$(echo "$json" | jq -r '.total_errors') - - # Stop server (triggers final stats flush). - stop_server - - echo " ops/s=$ops_s avg=${avg_us}us p50=${p50_us}us p99=${p99_us}us p999=${p999_us}us err=$errors" - echo -e "$label\t$wkr\t$ops_s\t$avg_us\t$p50_us\t$p99_us\t$p999_us\t$nagle\t$errors" >> "$RESULTS_FILE" + jq -r --arg label "$label" --arg workers "$workers" --arg nagle "$nagle" \ + '[$label, $workers, (.total_ops * 1000 / .duration_ms), .by_op.write.latency_us.avg_us, .by_op.write.latency_us.p50_us, .by_op.write.latency_us.p99_us, $nagle, .total_errors] | @tsv' \ + <<< "$output" >> "$RESULTS_FILE" } -# Cleanup on exit — stop server if still running. -trap 'stop_server' EXIT - -echo -e "label\twkr\tops_s\tavg_us\tp50_us\tp99_us\tp999_us\tnagle\terrors" > "$RESULTS_FILE" - -echo "=== rpc echo regression (128B, 20s, 11 configs) ===" - # Standard regression sweep — single-engine, scale workers with load. # Coroutine: all 4 configs. Tokio: only 1/64/1000 loaders (skip 512). run_bench 1 1 "rpc_1e1w_1l_1c" 1 1 @@ -236,6 +58,6 @@ run_bench 64 4 "rpc_1e4w_64l_4c_nagle" 1 4 coroutine 1 run_bench 512 8 "rpc_1e8w_512l_8c_nagle" 1 8 coroutine 1 run_bench 1000 32 "rpc_1e16w_1000l_32c_nagle" 1 16 coroutine 1 -echo "=== DONE ===" -echo "Results in $RESULTS_FILE" -column -t -s$'\t' "$RESULTS_FILE" +printf 'Results: %s\n' "$RESULTS_FILE" +cat "$RESULTS_FILE" +[ "$FAILURES" -eq 0 ] diff --git a/tools/benchmark/fixture-smoke.py b/tools/benchmark/fixture-smoke.py new file mode 100644 index 000000000..5e9a95f7a --- /dev/null +++ b/tools/benchmark/fixture-smoke.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Exercise native measurement ownership and real attached KV workloads.""" + +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import tomllib +import urllib.error +import urllib.request +import uuid + +import fixture + + +def wait_stopped(processes): + deadline = time.monotonic() + 15 + while any(fixture.process_start(pid) == start for pid, start in processes): + if time.monotonic() >= deadline: + raise RuntimeError(f"owned measurement processes survived shutdown: {processes}") + time.sleep(0.05) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--worker", type=Path, required=True) + parser.add_argument("--client", type=Path, required=True) + parser.add_argument("--artifacts", type=Path, + default=fixture.ROOT / ".crowdb-runtime/artifacts/measurement-fixture" / uuid.uuid4().hex) + args = parser.parse_args() + args.artifacts.mkdir(parents=True, exist_ok=False) + environment = dict(os.environ, CROWDB_REGRESSION_FIXTURE_BIN=str(args.worker.resolve()), + REGRESSION_CLI=str(args.client.resolve())) + + def command(state, *arguments, owner=None, succeeds=True): + argv = [sys.executable, str(fixture.ROOT / "tools/benchmark/fixture.py"), "--state", str(state)] + if owner is not None: + argv += ["--owner-pid", str(owner)] + result = subprocess.run([*argv, *arguments], env=environment, capture_output=True, text=True, timeout=60) + print(result.stdout, end="", flush=True) + print(result.stderr, end="", file=sys.stderr, flush=True) + with (args.artifacts / "commands.log").open("a") as log: + log.write(f"{arguments}: exit={result.returncode}\n{result.stdout}{result.stderr}") + if (result.returncode == 0) != succeeds: + raise RuntimeError(f"measurement command returned {result.returncode}: {arguments}") + return result.stdout + + def setup(state, owner): + command(state, "setup", "--", "--kind", "kv", "--nodes", "3", "--kv-backend", "mem-block", + "--wal-backend", "mem-block", "--metrics-interval", "1", owner=owner) + value = fixture.descriptor(state) + servers = tomllib.loads(state.read_text())["server"] + processes = [(server["pid"], fixture.process_start(server["pid"])) for server in servers] + assert len(processes) == 3 and all(start for _, start in processes) + return value, [(value["pid"], value["start"]), *processes] + + # This process is deliberately unrelated to either fixture's receipt. + unrelated = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"]) + state = args.artifacts / "explicit.toml" + owner = None + try: + value, owned = setup(state, os.getpid()) + request = urllib.request.Request(value["endpoint"] + "/_fixture/control", + data=b'{"action":"stop"}', headers={"Content-Type": "application/json", + "X-Crowdb-Fixture-Owner": "foreign"}) + try: + fixture.HTTP.open(request, timeout=5) + except urllib.error.HTTPError as error: + assert error.code == 403, error.read().decode() + else: + raise AssertionError("foreign control token was accepted") + command(state, "reset", "--group", "0", succeeds=False) + command(state, "client", "--", "kv", "group", "add", "-s", "0", "-g", "1", "-n", "1,2,3") + command(state, "client", "--", "bench", "kv", "prepare", "--store", "0", "--group", "1", + "--keys", "16", "--value-size", "64") + for workload in ("read", "scan", "write"): + command(state, "client", "--", "bench", "kv", workload, "--store", "0", "--group", "1", + "--duration-secs", "1", "--loader-num", "1", "--connections", "1", + "--key-space", "16", "--value-size", "64") + report = json.loads(command(state, "report")) + assert report["total_ops"] > 0 and report["total_errors"] == 0, report + command(state, "client", "--", "bench", "kv", "invalid", succeeds=False) + command(state, "report", succeeds=False) + result = json.loads(command(state, "reset", "--group", "1")) + assert result["new_leader"] and result["wiped_nodes"] == 3, result + command(state, "stop") + wait_stopped(owned) + assert unrelated.poll() is None, "fixture stopped an unrelated process" + + owner = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"]) + abandoned = args.artifacts / "owner-exit.toml" + _, owned = setup(abandoned, owner.pid) + owner.terminate() + owner.wait(timeout=5) + wait_stopped(owned) + stopped = json.loads(abandoned.with_suffix(".fixture-stopped.json").read_text()) + assert stopped["ok"], stopped + assert unrelated.poll() is None, "owner-exit cleanup stopped an unrelated process" + print(f"PASS: native measurement ownership and KV workloads; artifacts: {args.artifacts}") + finally: + command(state, "stop") + if owner is not None and owner.poll() is None: + owner.terminate() + owner.wait(timeout=5) + unrelated.terminate() + unrelated.wait(timeout=5) + + +if __name__ == "__main__": + main() diff --git a/tools/benchmark/fixture.py b/tools/benchmark/fixture.py new file mode 100644 index 000000000..15a643852 --- /dev/null +++ b/tools/benchmark/fixture.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# Copyright 2026-present Gian +# Licensed under the Apache License, Version 2.0. +"""Explicit native measurement fixture; never an installed product command.""" + +import argparse +import json +import os +from pathlib import Path +import shutil +import stat +import subprocess +import sys +import tempfile +import time +import uuid +import urllib.request +import urllib.error + +ROOT = Path(__file__).resolve().parents[2] +HTTP = urllib.request.build_opener(urllib.request.ProxyHandler({})) +BENCHMARKS = {"kv", "diskdb", "chunkdb", "chunkio", "s3"} + + +class FixtureStopped(ValueError): + pass + + +def process_start(pid): + try: + if sys.platform.startswith("linux"): + fields = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + return None if fields[0] == "Z" else fields[19] + value = subprocess.check_output(["ps", "-p", str(pid), "-o", "lstart="], text=True).strip() + return value or None + except (OSError, subprocess.SubprocessError): + return None + + +def descriptor(state): + path = state.with_suffix(".fixture.json") + info = path.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077 or info.st_size > 65536: + raise ValueError("fixture descriptor must be a bounded private regular file") + value = json.loads(path.read_text()) + if process_start(value["pid"]) != value["start"]: + raise FixtureStopped("measurement fixture is stopped; start an explicit fixture first") + return value + + +def control(state, action): + value = descriptor(state) + request = urllib.request.Request(value["endpoint"] + "/_fixture/control", + data=json.dumps(action).encode(), headers={"Content-Type": "application/json", + "X-Crowdb-Fixture-Owner": value["token"]}) + try: + with HTTP.open(request, timeout=180) as reply: + result = json.load(reply) + except urllib.error.HTTPError as error: + raise RuntimeError(f"fixture control failed {error.code}: " + error.read().decode()) from error + if result.get("ok") is not True: + raise ValueError("fixture control did not confirm the requested action") + return value, result + + +def setup(args): + if args.arguments[:1] == ["--"]: + args.arguments = args.arguments[1:] + if args.state.with_suffix(".fixture.json").exists(): + try: + descriptor(args.state) + except FixtureStopped: + pass + else: + control(args.state, {"action": "setup", "arguments": args.arguments}) + return + if args.owner_pid is None or process_start(args.owner_pid) is None: + raise ValueError("fixture setup requires --owner-pid for the live measurement owner") + binary = Path(os.environ.get("CROWDB_REGRESSION_FIXTURE_BIN", + ROOT / "target/release/examples/regression-fixture")) + if not binary.is_file(): + raise ValueError("build the test fixture with: pixi run cargo build --release -p " + "crowdb-console-shared --example regression-fixture --features test-util") + args.state.parent.mkdir(parents=True, exist_ok=True) + if args.state.is_symlink(): + raise ValueError("fixture state cannot be a symlink") + root = Path(tempfile.mkdtemp(prefix=args.state.stem + "-fixture-", dir=args.state.parent)) + with (root / "controller.log").open("w") as log: + # The worker requires a fresh root; its diagnostic output stays alongside it. + empty = root / "owned" + empty.mkdir(mode=0o700) + environment = dict(os.environ) + for setting in ("LD_PRELOAD", "ASAN_OPTIONS"): + override = environment.pop("CROWDB_FIXTURE_" + setting, None) + if override is not None: + environment[setting] = override + worker = subprocess.Popen([str(binary), "--root", str(empty), "--state", str(args.state), + "--owner-pid", str(args.owner_pid), "--", *args.arguments], stdout=log, stderr=log, + start_new_session=True, env=environment) + deadline = time.monotonic() + 180 + while time.monotonic() < deadline: + if worker.poll() is not None: + raise RuntimeError(f"fixture setup exited {worker.returncode}:\n" + (root / "controller.log").read_text()) + try: + value = descriptor(args.state) + if value["pid"] == worker.pid: + print(f"Measurement fixture ready; artifacts: {root}", file=sys.stderr) + return + except (FileNotFoundError, ValueError): + pass + time.sleep(0.05) + worker.terminate() + worker.wait(timeout=60) + raise TimeoutError("explicit measurement fixture did not become ready within 180 seconds") + + +def run(args): + args.state = args.state.resolve(strict=False) if not args.state.is_symlink() else args.state.absolute() + if args.action == "setup": + setup(args) + elif args.action == "reset": + _, result = control(args.state, {"action": "reset", "store": args.store, "group": args.group, + "restart_storage": args.restart_storage}) + print(json.dumps(result["result"], indent=2)) + elif args.action == "inspect": + _, result = control(args.state, {"action": "inspect"}) + print(json.dumps(result["result"], indent=2)) + elif args.action == "stop": + try: + value, _ = control(args.state, {"action": "stop"}) + except (FileNotFoundError, FixtureStopped): + return 0 + deadline = time.monotonic() + 60 + while process_start(value["pid"]) == value["start"]: + if time.monotonic() >= deadline: + raise TimeoutError("owned measurement fixture did not stop within 60 seconds") + time.sleep(0.05) + stopped = json.loads(args.state.with_suffix(".fixture-stopped.json").read_text()) + if (stopped.get("pid"), stopped.get("start"), stopped.get("ok")) != (value["pid"], value["start"], True): + raise RuntimeError(f"owned fixture cleanup did not complete: {stopped.get('error')}") + elif args.action == "client": + value = descriptor(args.state) + environment = dict(os.environ, CROWDB_RUNTIME_ROOT=str(args.state.parent / "client-runtime")) + environment.pop("CROWDB_CLUSTER", None) + arguments = args.arguments[1:] if args.arguments[:1] == ["--"] else args.arguments + logs = None + if arguments[:1] == ["bench"] and len(arguments) > 1 and arguments[1] in BENCHMARKS: + logs = args.state.parent / "client-runtime/artifacts/cli" / ("bench-" + arguments[1]) + if logs.exists(): + shutil.rmtree(logs) + cli = os.environ.get("REGRESSION_CLI", str(ROOT / "target/release/crowdb-cli")) + code = subprocess.call([cli, "--management-endpoint", value["endpoint"], *arguments], env=environment) + if logs is not None: + if logs.is_dir(): + # Product CLI owns its latest-run directory. Measurements retain + # each completed invocation before the next one replaces it. + retained = args.state.parent / "measurements" / (logs.name + "-" + uuid.uuid4().hex) + shutil.copytree(logs, retained) + return code + elif args.action in ("target", "root"): + value = descriptor(args.state) + print(value["endpoint" if args.action == "target" else "root"]) + elif args.action == "seed": + value = descriptor(args.state) + with HTTP.open(value["endpoint"] + "/api/cluster/target", timeout=5) as response: + target = json.load(response) + seeds = target.get("group0_management_endpoints") + if not isinstance(seeds, list) or not seeds or not all(isinstance(seed, str) and seed for seed in seeds): + raise ValueError("live measurement target has no Group-0 endpoints") + print(seeds[0]) + elif args.action == "report": + path = args.state.parent / "client-runtime/artifacts/cli/bench-kv/report.json" + report = json.loads(path.read_text()) + if not all(key in report for key in ("total_ops", "total_errors", "duration_ms", "by_op")): + raise ValueError("latest benchmark report is incomplete") + print(json.dumps(report, indent=2)) + return 0 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--state", type=Path, required=True) + parser.add_argument("--owner-pid", type=int) + subcommands = parser.add_subparsers(dest="action", required=True) + subcommands.add_parser("setup").add_argument("arguments", nargs=argparse.REMAINDER) + subcommands.add_parser("stop") + subcommands.add_parser("target") + subcommands.add_parser("root") + subcommands.add_parser("seed") + subcommands.add_parser("report") + subcommands.add_parser("inspect") + reset = subcommands.add_parser("reset") + reset.add_argument("--store", type=int, default=0) + reset.add_argument("--group", type=int, required=True) + reset.add_argument("--restart-storage", action="store_true") + subcommands.add_parser("client").add_argument("arguments", nargs=argparse.REMAINDER) + args = parser.parse_args() + try: + return run(args) + except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error: + print(f"measurement fixture failed: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci-checks/check-ci-test-tasks.py b/tools/ci-checks/check-ci-test-tasks.py index bbb3a39bc..d883547ea 100644 --- a/tools/ci-checks/check-ci-test-tasks.py +++ b/tools/ci-checks/check-ci-test-tasks.py @@ -8,7 +8,7 @@ COMPONENT_PACKAGES = { - "test-cpp": {"crowdb-tree-ffi", "crowdb-rpc-ffi"}, + "test-cpp": {"crowdb-tree-ffi", "crowdb-rpc-ffi", "crowdb-rpc-test"}, "test-core": { "crowdb-common", "crowdb-test-harness", "crowdb-protocol", "crowdb-kv", "crowdb-kv-client", diff --git a/tools/cpp-checks/sanitize-regression.sh b/tools/cpp-checks/sanitize-regression.sh index 27bdf804f..327f90250 100755 --- a/tools/cpp-checks/sanitize-regression.sh +++ b/tools/cpp-checks/sanitize-regression.sh @@ -57,13 +57,10 @@ # - Client bench process: MUST exit 0 (no leaks). The client is a # short-lived process that connects, runs the workload, and exits # cleanly. -# - Server processes: killed via SIGTERM during `cluster destroy`. +# - Server processes: killed via SIGTERM during owned fixture shutdown. # Graceful shutdown runs (SIGTERM handler → PxKvStore::shutdown → # stop_rpc_server → RpcServer::stop → clear_handlers). If the # server exits cleanly (no ASan log), all leaks are fixed. -# - The local-deploy CLI process may show ~97KB tokio runtime noise -# (113 allocations) — this is a known tokio cleanup issue, not -# our code. We filter it out in the leak check. # # Prerequisites: # - pixi installed, project dependencies resolved @@ -73,10 +70,10 @@ # restore the default debug binary. set -euo pipefail cd "$(dirname "$0")/../.." +unset CROWDB_ASAN RUNTIME_ROOT="${CROWDB_RUNTIME_ROOT:-${PIXI_PROJECT_ROOT:-$(pwd)}/.crowdb-runtime}" -RUNTIME_DIR="$RUNTIME_ROOT/artifacts/sanitize-regression" -rm -rf "$RUNTIME_DIR" +RUNTIME_DIR="$RUNTIME_ROOT/artifacts/sanitize-regression-$(date +%Y%m%d-%H%M%S)-$$" mkdir -p "$RUNTIME_DIR" RESULTS_FILE="$RUNTIME_DIR/sanitize-regression.tsv" DURATION=5 @@ -94,14 +91,33 @@ CROWDB_CLI="$(cd "$(dirname "$0")/../.." && pwd)/target/debug/crowdb-cli" # internals. LIBASAN="$(cd "$(dirname "$0")/../.." && pixi run -- pwd)/.pixi/envs/default/lib/libasan.so" -# Clean up stale ASan logs from previous runs. -rm -f ${RUNTIME_DIR}/asan-sanitize-*.* 2>/dev/null || true -rm -f ${RUNTIME_DIR}/sanitize-unexpected-asan-sanitize-* 2>/dev/null || true +restore_binaries() { + pixi run cargo build -p crowdb-cli -p crowdb-rpc-test -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb + pixi run cargo build -p crowdb-console-shared --example regression-fixture --features test-util +} + +cleanup_sanitize() { + local status=$? state descriptor + trap - EXIT + for descriptor in "$RUNTIME_DIR"/*.fixture.json; do + [ -f "$descriptor" ] || continue + state="${descriptor%.fixture.json}.toml" + pixi run python tools/benchmark/fixture.py --state "$state" stop || status=1 + done + restore_binaries || status=1 + exit "$status" +} +trap cleanup_sanitize EXIT + # --- Phase 1: Build with ASan + LSan enabled --- echo "=== building with CROWDB_ASAN=1 (debug) ===" -CROWDB_ASAN=1 pixi run -- cargo build -p crowdb-cli -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb +CROWDB_ASAN=1 pixi run -- cargo build -p crowdb-cli -p crowdb-rpc-test -p crowdb-kv-server -p crowdb-diskdb -p crowdb-chunkdb +CROWDB_ASAN=1 pixi run cargo build -p crowdb-console-shared --example regression-fixture --features test-util pixi run -- cmake --build app/crowdb-diskio/build -j +RPC_ASAN_BUILD="$RUNTIME_DIR/rpc-server-build" +pixi run cmake -S lib/crowdb-rpc -B "$RPC_ASAN_BUILD" -DCMAKE_BUILD_TYPE=Debug -DCROWDB_RPC_SANITIZER=address +pixi run cmake --build "$RPC_ASAN_BUILD" -j 4 --target crowdb-rpc-fb-server # Verify the binary exists. if [ ! -x "$CROWDB_CLI" ]; then @@ -116,28 +132,33 @@ if [ ! -f "$LIBASAN" ]; then exit 1 fi -# asan_cli -# Runs crowdb-cli directly (not via cargo run) under ASan with the -# given log_path prefix. Using the binary directly avoids ASan -# instrumenting cargo/pixi themselves (which produces false leaks). -asan_cli() { - local log_prefix="$1"; shift - env ASAN_OPTIONS="detect_leaks=1:abort_on_error=0:log_path=${log_prefix}" \ - LD_PRELOAD="$LIBASAN" \ - "$CROWDB_CLI" "$@" +# ASan is applied to the measured native processes, never to Python or Pixi. +asan_client() { + local log_prefix="$1" state="$2"; shift 2 + local endpoint + endpoint=$(pixi run python tools/benchmark/fixture.py --state "$state" target) + rm -f "$(dirname "$state")/client-runtime/artifacts/cli/bench-kv/report.json" + CROWDB_RUNTIME_ROOT="$(dirname "$state")/client-runtime" pixi run env -u CROWDB_CLUSTER ASAN_OPTIONS="detect_leaks=1:abort_on_error=0:log_path=${log_prefix}" \ + LD_PRELOAD="$LIBASAN" "$CROWDB_CLI" --management-endpoint "$endpoint" "$@" +} + +asan_fixture() { + local log_prefix="$1" state="$2"; shift 2 + CROWDB_REGRESSION_FIXTURE_BIN="$(pwd)/target/debug/examples/regression-fixture" \ + CROWDB_FIXTURE_LD_PRELOAD="$LIBASAN" \ + CROWDB_FIXTURE_ASAN_OPTIONS="detect_leaks=1:abort_on_error=0:log_path=${log_prefix}" \ + pixi run python tools/benchmark/fixture.py --state "$state" --owner-pid "$$" setup -- "$@" } # check_asan_logs