diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e240f9aa..7cee304e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,7 +77,9 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Install Protobuf compiler - run: sudo apt-get install -y protobuf-compiler + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler - name: Install cargo tools run: | diff --git a/Cargo.lock b/Cargo.lock index 5c74fa52..bbd536d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -693,7 +693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1095,7 +1095,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1825,14 +1825,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "log", "once_cell", @@ -1854,9 +1854,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -2099,7 +2099,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2182,9 +2182,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" dependencies = [ "bytes", "libc", @@ -2662,7 +2662,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 86914470..44894f86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ exclude = [ "examples/quick-start-embedded", "examples/service-discovery-standalone", "examples/service-discovery-embedded", + "examples/raft-engine-log", "benches/standalone-bench", "benches/embedded-bench" ] @@ -28,7 +29,7 @@ resolver = "2" version = "0.2.4" edition = "2024" authors = ["Joshua Chi "] -homepage = "https://github.com/deventlab/d-engine" +homepage = "https://d-engine.deventlab.com" repository = "https://github.com/deventlab/d-engine" license = "MIT OR Apache-2.0" rust-version = "1.89" diff --git a/README.md b/README.md index 963b476f..4043feeb 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,8 @@ async fn main() { **→ New to d-engine?** [5-minute guide](https://docs.rs/d-engine/latest/d_engine/#-new-to-d-engine-start-here) +**→ Website:** [d-engine.deventlab.com](https://d-engine.deventlab.com) + --- ## Integration Modes @@ -89,7 +91,7 @@ d-engine = "0.2" **Use when**: Building Rust applications that need distributed coordination **Why**: Zero-overhead (<0.1ms), single binary, zero network cost -> **Performance**: AWS EC2 3-node cluster — **110K writes/sec**, **327K linearizable reads/sec**, sub-millisecond latency. See [benches/reports/v0.2.4/](https://github.com/deventlab/d-engine/tree/main/benches/reports/v0.2.4) for details. +> **Performance**: see [Performance](#performance) below. **→ Examples:** @@ -107,7 +109,8 @@ d-engine = { version = "0.2", features = ["client"], default-features = false } **Use when**: Application and d-engine run as separate processes **Why**: Language-agnostic (Go/Python/Java/Rust), independent scaling, easier operations -> **Performance**: 59K writes/sec, 77K linearizable reads/sec via gRPC. For maximum throughput, embedded mode is 1.9× faster on writes and 4.2× on reads. See [benches/reports/v0.2.4/](https://github.com/deventlab/d-engine/tree/main/benches/reports/v0.2.4). +> **Performance**: gRPC adds a network hop, so throughput is lower than embedded mode. +> See the [full report](https://github.com/deventlab/d-engine/tree/main/benches/reports) for standalone results. **→ Example:** [Quick Start Standalone (Go client)](https://github.com/deventlab/d-engine/tree/main/examples/quick-start-standalone) @@ -129,17 +132,25 @@ Implement the `StorageEngine` and `StateMachine` traits for custom backends: ## Performance -### d-engine v0.2.5 vs etcd - -![d-engine vs etcd comparison](https://raw.githubusercontent.com/deventlab/d-engine/main/benches/reports/v0.2.5/d-engine_comparison_v0.2.5.png) - -### d-engine v0.2.5 vs v0.2.4 - -![d-engine v0.2.5 vs v0.2.4 comparison](https://raw.githubusercontent.com/deventlab/d-engine/main/benches/reports/v0.2.5/d-engine_v0.2.5_vs_v0.2.4_embedded_mode.png) - -```bash -open benches/reports/ -``` +Embedded mode against etcd v3.7.2, same AWS instance shape (3 × c5.2xlarge, gp3 disk), +256 B values, 100 requests in flight (single client: 1). d-engine runs with the optional +raft-engine log store; storage engines are swappable and perform differently. + +| Case | etcd (ops/s) | d-engine (ops/s) | d-engine ÷ etcd | +| ---------------------------------- | -----------: | ---------------: | --------------: | +| Write, 1 in flight | 605 | 836 | 1.38× | +| Write, 100 in flight | 10,064 | 19,860 | 1.97× | +| Linearizable read | 64,071 | 368,068 | 5.74× | +| Eventual read (etcd: serializable) | 148,709 | 1,253,650 | 8.43× | + +- Every commit is synced to disk before it is acknowledged (RPO=0), so writes are slower than + in releases before this guarantee, by design. +- Embedded calls skip serialization and the network, so the read ratios measure that, not only the engine. +- With the default RocksDB log store, single-client write is slower than etcd (0.56×); + 100 in flight is 1.81×. +- Three rounds per case on different instances of the same shape; ratios are indicative (about ±20%). + +→ [Full report, standalone results and setup](https://github.com/deventlab/d-engine/tree/main/benches/reports) --- diff --git a/benches/embedded-bench/.gitignore b/benches/embedded-bench/.gitignore index 70242f77..4550ff7a 100644 --- a/benches/embedded-bench/.gitignore +++ b/benches/embedded-bench/.gitignore @@ -32,6 +32,9 @@ snapshots/* data data/* +data-raft-engine/ cobertura.xml .DS_Store + +target-raft-engine/ diff --git a/benches/embedded-bench/Cargo.toml b/benches/embedded-bench/Cargo.toml index 45c2b9e1..0c6ea845 100644 --- a/benches/embedded-bench/Cargo.toml +++ b/benches/embedded-bench/Cargo.toml @@ -19,3 +19,9 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing = { version = "0.1" } metrics-exporter-prometheus = "0.17.2" console-subscriber = "0.5" + +# Optional: benchmark the raft-engine log store instead of the RocksDB WAL. +raft-engine-log = { path = "../../examples/raft-engine-log", optional = true } + +[features] +raft-engine = ["dep:raft-engine-log"] diff --git a/benches/embedded-bench/Makefile b/benches/embedded-bench/Makefile index 281c685e..11237e73 100644 --- a/benches/embedded-bench/Makefile +++ b/benches/embedded-bench/Makefile @@ -7,7 +7,16 @@ # =============================== # Global Variables # =============================== -BENCH_BIN := ./target/release/embedded-bench +# Storage engine under test. Default: Raft log in raft-engine (experimental). +# ENGINE=rocksdb builds and runs the stock RocksDB-WAL binary, which is the +# configuration of the v0.2.4 / v0.2.5 reports. Each engine has its own target dir +# and its own data dir, so switching never mixes their files. +ENGINE ?= raft-engine +$(if $(filter raft-engine rocksdb,$(ENGINE)),,$(error ENGINE must be raft-engine or rocksdb, got '$(ENGINE)')) +TARGET_DIR := $(if $(filter raft-engine,$(ENGINE)),target-raft-engine,target) +CARGO_FEATURES := $(if $(filter raft-engine,$(ENGINE)),--features raft-engine,) +BENCH_BIN := ./$(TARGET_DIR)/release/embedded-bench +DATA_ROOT := $(if $(filter raft-engine,$(ENGINE)),./data-raft-engine,./data) CONFIG_DIR := ./config LOG_LEVEL ?= warn @@ -46,11 +55,11 @@ RAMDISK_VOLUMES := RAMDisk1 RAMDisk2 RAMDisk3 RAMDISK_INDEX := $(if $(filter n2,$(NODE)),2,$(if $(filter n3,$(NODE)),3,1)) ifeq ($(RAMDISK),true) -DATA_DIR := /Volumes/RAMDisk$(RAMDISK_INDEX)/$(NODE) -SINGLE_NODE_DATA_DIR := /Volumes/RAMDisk1/single-node +DATA_DIR := /Volumes/RAMDisk$(RAMDISK_INDEX)/$(ENGINE)/$(NODE) +SINGLE_NODE_DATA_DIR := /Volumes/RAMDisk1/$(ENGINE)/single-node else -DATA_DIR := ./data/$(NODE) -SINGLE_NODE_DATA_DIR := ./data/single-node +DATA_DIR := $(DATA_ROOT)/$(NODE) +SINGLE_NODE_DATA_DIR := $(DATA_ROOT)/single-node endif # On macOS with Homebrew: auto-detect compression lib paths to skip bundled C++ @@ -79,8 +88,12 @@ help: @echo "Embedded-bench Makefile - Performance Testing" @echo "" @echo "Build:" - @echo " make build Build release binary" - @echo " make clean Clean build artifacts" + @echo " make build Build release binary (engine: $(ENGINE))" + @echo " make clean Clean everything (both engines, local data, RAM disks)" + @echo "" + @echo "Storage engine (applies to every target below):" + @echo " default raft-engine log store (experimental)" + @echo " ENGINE=rocksdb stock RocksDB-WAL build, e.g. make all-tests ENGINE=rocksdb" @echo "" @echo "Write Tests (only run on Leader - default: Node 1):" @echo " make test-single-write Single client write (10K requests)" @@ -116,14 +129,18 @@ help: @echo " cd ./embedded-bench && make start-cluster" build: - @echo "Building embedded-bench..." - $(BREW_ROCKSDB_ENV) cargo build --release --jobs 8 + @echo "Building embedded-bench (engine: $(ENGINE))..." + $(BREW_ROCKSDB_ENV) cargo build --release --jobs 8 $(CARGO_FEATURES) --target-dir $(TARGET_DIR) +# Removes both engines' build output and local data. RAM disk data is wiped separately with +# `make clean-ram-log-db`. clean: @echo "Cleaning build artifacts and data directories..." - cargo clean + cargo clean --target-dir target + cargo clean --target-dir target-raft-engine @echo "Removing data directories..." rm -rf ./data/* + rm -rf ./data-raft-engine/* rm -rf ./logs/* rm -rf ./snapshots/* @echo "Clean complete!" @@ -132,6 +149,7 @@ clean-log-db: @echo "Cleaning db and logs..." rm -rf ./logs/* rm -rf ./data/* + rm -rf ./data-raft-engine/* rm -rf ./snapshots/* # ============================================ @@ -271,7 +289,7 @@ test-hot-key: build all-tests: build @echo "========================================" @echo "Running All Tests in Batch Mode" - @echo "Node: $(NODE)" + @echo "Node: $(NODE) Engine: $(ENGINE)" @echo "========================================" CONFIG_PATH=$(CONFIG_PATH) \ DATA_DIR=$(DATA_DIR) \ @@ -311,9 +329,12 @@ ramdisk-create: [ -d "/Volumes/$$v" ] || diskutil erasevolume HFS+ $$v `hdiutil attach -nomount ram://$(RAMDISK_SECTORS)` >/dev/null; \ done -# Wipes node data off the RAM disks; keeps the volumes mounted. +# Wipes this benchmark's per-engine directories off the RAM disks and keeps the volumes +# mounted. Nothing else on the volumes is touched. clean-ram-log-db: - @for v in $(RAMDISK_VOLUMES); do rm -rf /Volumes/$$v/*; done + @for e in raft-engine rocksdb; do \ + rm -rf /Volumes/RAMDisk1/$$e /Volumes/RAMDisk2/$$e /Volumes/RAMDisk3/$$e; \ + done # Unmounts the RAM disks entirely, releasing the memory back to the OS. ramdisk-release: diff --git a/benches/embedded-bench/src/main.rs b/benches/embedded-bench/src/main.rs index d96ef876..3fc81923 100644 --- a/benches/embedded-bench/src/main.rs +++ b/benches/embedded-bench/src/main.rs @@ -1,4 +1,38 @@ -use d_engine::DefaultEmbeddedEngine; +// Engine under test. The default build is d-engine's stock RocksDB engine. The +// `raft-engine` feature replaces only the Raft log store (the state machine, +// config file and data layout are unchanged), so the two builds differ by the +// log store alone and their results stay comparable. +#[cfg(not(feature = "raft-engine"))] +type BenchEngine = d_engine::DefaultEmbeddedEngine; +#[cfg(feature = "raft-engine")] +type BenchEngine = + d_engine::EmbeddedEngine; + +#[cfg(not(feature = "raft-engine"))] +async fn start_engine( + data_dir: &str, + config_path: &str, +) -> Result { + BenchEngine::start_with(data_dir, config_path).await +} + +#[cfg(feature = "raft-engine")] +async fn start_engine( + data_dir: &str, + config_path: &str, +) -> Result { + std::fs::create_dir_all(data_dir) + .map_err(|e| d_engine::Error::Fatal(format!("create {data_dir}: {e}")))?; + let options = raft_engine_log::RaftEngineOptions::from_toml_file(config_path)?; + let storage = Arc::new(raft_engine_log::RaftEngineStorage::open_with( + format!("{data_dir}/storage"), + &options, + )?); + let sm = Arc::new(d_engine::RocksDBStateMachine::new(format!( + "{data_dir}/state_machine" + ))?); + BenchEngine::start_custom(data_dir, storage, sm, Some(config_path)).await +} use d_engine::protocol::ReadConsistencyPolicy; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -263,7 +297,7 @@ async fn start_metrics_server(port: u16) { /// Run a single benchmark test with specified parameters #[allow(clippy::too_many_arguments)] async fn run_benchmark_task( - engine: &Arc, + engine: &Arc, test_name: &str, command: Commands, total: u64, @@ -369,7 +403,7 @@ async fn run_benchmark_task( /// Run all benchmark tests in batch mode async fn run_batch_tests( - engine: &Arc, + engine: &Arc, cli: &Cli, ) { println!("\n╔════════════════════════════════════════╗"); @@ -495,7 +529,7 @@ async fn run_batch_tests( /// Helper function to run a single test in batch mode async fn run_single_batch_test( - engine: &Arc, + engine: &Arc, test_name: &str, command: Commands, total: u64, @@ -522,7 +556,7 @@ async fn run_local_benchmark(cli: Cli) { println!("Starting local benchmark mode..."); let engine = Arc::new( - DefaultEmbeddedEngine::start_with(&cli.data_dir, &cli.config_path) + start_engine(&cli.data_dir, &cli.config_path) .await .expect("Failed to start engine"), ); @@ -762,7 +796,7 @@ async fn run_http_server(cli: Cli) { println!("Health check port: {}", cli.health_port); let engine = Arc::new( - DefaultEmbeddedEngine::start_with(&cli.data_dir, &cli.config_path) + start_engine(&cli.data_dir, &cli.config_path) .await .expect("Failed to start engine"), ); @@ -790,7 +824,7 @@ async fn run_http_server(cli: Cli) { } async fn start_health_check_server( - engine: Arc, + engine: Arc, port: u16, ) { let app = Router::new() @@ -807,7 +841,7 @@ async fn start_health_check_server( axum::serve(listener, app).await.expect("Health check server failed"); } -async fn health_primary(State(engine): State>) -> StatusCode { +async fn health_primary(State(engine): State>) -> StatusCode { if engine.is_leader() { StatusCode::OK } else { @@ -815,7 +849,7 @@ async fn health_primary(State(engine): State>) -> Sta } } -async fn health_replica(State(engine): State>) -> StatusCode { +async fn health_replica(State(engine): State>) -> StatusCode { if !engine.is_leader() { StatusCode::OK } else { @@ -824,7 +858,7 @@ async fn health_replica(State(engine): State>) -> Sta } async fn start_business_server( - engine: Arc, + engine: Arc, port: u16, ) { let app = Router::new() @@ -842,7 +876,7 @@ async fn start_business_server( } async fn handle_put( - State(engine): State>, + State(engine): State>, Json(req): Json, ) -> StatusCode { match engine.client().put(req.key.into_bytes(), req.value.into_bytes()).await { @@ -852,7 +886,7 @@ async fn handle_put( } async fn handle_get( - State(engine): State>, + State(engine): State>, Path(key): Path, ) -> Result, StatusCode> { match engine.client().get_eventual(key.into_bytes()).await { diff --git a/benches/reports/v0.2.5/bench_report_v0.2.5.md b/benches/reports/v0.2.5/bench_report_v0.2.5.md index 3a7f0f7d..acc1aa49 100644 --- a/benches/reports/v0.2.5/bench_report_v0.2.5.md +++ b/benches/reports/v0.2.5/bench_report_v0.2.5.md @@ -1,109 +1,102 @@ # d-engine v0.2.5 Benchmark Report -**Test Environments**: - -- **AWS**: EC2 c5.2xlarge (8 vCPUs, 16GB RAM, 50GB SSD) × 3 nodes -- **Key/Value**: 8 bytes / 256 bytes - ---- - -![d-engine v0.2.5 vs v0.2.4 Embedded Mode](d-engine_v0.2.5_vs_v0.2.4_embedded_mode.png) - -![d-engine v0.2.5 vs v0.2.4 Standalone Mode](d-engine_v0.2.5_vs_v0.2.4_standalone_mode.png) - -![d-engine v0.2.5 vs etcd 3.2.0](d-engine_comparison_v0.2.5.png) - -## AWS 3-Node Cluster: d-engine v0.2.5 vs v0.2.4 / etcd 3.2.0 - -**Hardware**: AWS EC2 c5.2xlarge (8 vCPUs, 16GB RAM, 50GB SSD) × 3 nodes -**Date**: September 26, 2026 | 3-round average | Key/Value: 8 bytes / 256 bytes -**Embedded mode concurrency**: `--clients 1000` (previously omitted from the AWS harness — see Benchmark Configuration note below) -**etcd reference**: Official etcd benchmark (GCE, 8 vCPUs + 16GB + SSD × 3 nodes, etcd 3.2.0)² - -> Updated 2026-09-26: this section now reflects v0.2.5 **with #446 merged** -> (quorum commit gated on physical fsync — durable_index only advances after -> fdatasync, per ADR-038 Level 3 / RPO=0). The June 2, 2026 numbers previously -> here predate #446 and measured #392 (ReadActor) alone; #446 changes write-path -> latency substantially and is reflected below. - -### Embedded Mode: v0.2.5 vs v0.2.4 / etcd 3.2.0 - -| **Scenario** | **Metric** | **v0.2.4 (AWS)** | **v0.2.5 (AWS)** | **Δ vs v0.2.4** | **etcd 3.2.0²** | **Δ vs etcd** | -| ------------------- | ----------- | ---------------- | ---------------- | --------------- | --------------- | -------------- | -| Single Client Write | Throughput | 4,398 ops/s | 334 ops/s | **-92.4%** ⚠️ | 583 ops/s | -42.8% ⚠️ | -| | Avg Latency | 0.227 ms | 2.994 ms | **+1219%** ⚠️ | 1.6 ms | +87.1% ⚠️ | -| | p99 Latency | 0.316 ms | 3.154 ms | **+898%** ⚠️ | — | — | -| High Conc. Write | Throughput | 110,798 ops/s | 115,648 ops/s | +4.4% → | 44,341 ops/s | **+2.6x** ✅ | -| | Avg Latency | 0.902 ms | 8.586 ms | **+852%** ⚠️ | 22.0 ms | -61.0% ✅ | -| | p99 Latency | 1.063 ms | 12.012 ms | **+1030%** ⚠️ | — | — | -| Linearizable Read | Throughput | 327,355 ops/s | 404,684 ops/s | +23.6% ✅ | 141,578 ops/s | **+2.9x** ✅ | -| | Avg Latency | 0.305 ms | 2.454 ms | **+705%** ⚠️ | 5.5 ms | -55.4% ✅ | -| | p99 Latency | 0.374 ms | 4.378 ms | **+1071%** ⚠️ | — | — | -| Lease Read | Throughput | 341,254 ops/s | 1,186,318 ops/s | **+247.7%** ✅ | —³ | — | -| | Avg Latency | 0.293 ms | 0.001 ms | **-99.7%** ✅ | — | — | -| | p99 Latency | 0.367 ms | 0.002 ms | **-99.5%** ✅ | — | — | -| Eventual Read | Throughput | 363,407 ops/s | 1,220,515 ops/s | **+235.9%** ✅ | 185,758 ops/s | **+6.6x** ✅ | -| | Avg Latency | 0.274 ms | 0.001 ms | **-99.6%** ✅ | 2.2 ms | **-99.95%** ✅ | -| | p99 Latency | 0.343 ms | 0.002 ms | **-99.4%** ✅ | — | — | -| Hot-Key (10 keys) | Throughput | 319,957 ops/s | 410,173 ops/s | +28.2% ✅ | —³ | — | -| | Avg Latency | 0.313 ms | 2.421 ms | **+674%** ⚠️ | — | — | -| | p99 Latency | 0.360 ms | 4.858 ms | **+1249%** ⚠️ | — | — | - -**Key Findings**: - -- **Single Client Write is the clearest #446 signal, and it's a regression**: throughput -92.4%, avg latency 0.227ms → 2.994ms (13x). No concurrency to amortize the cost — every request pays the full quorum-fsync round trip directly. d-engine's SC Write is now _slower_ than etcd's reference number (-42.8% throughput), reversing the previous +7.6x lead. -- **Concurrent write/read throughput holds up (HC Write +4.4%, Lin Read +23.6%, Hot-Key +28.2%)** — with enough in-flight requests, individual fsync/replication latency overlaps instead of stacking, so aggregate throughput survives. But **every scenario's avg/p99 latency is 7-13x worse than v0.2.4**, including reads that don't touch fsync (Hot-Key avg +674%) — consistent with the write path now serializing behind real fsync+quorum-ack round trips that the old `MemFirst` config never paid. -- **Lease/Eventual Read remain #392's win, untouched by #446** (+248%/+236% vs v0.2.4, both are pure in-memory reads with no Raft RPO=0 gating). -- **This is a much larger cost than #446's own loopback A/B measurements** (`tickets/milestones/v0.2.5/RESULTS-main-vs-446.md`: +9.9% `propose_to_apply`, -2.1% throughput). AWS's EBS gp3 is network-attached storage — every fsync is a real network round trip, unlike loopback disk I/O. Confirms the loopback numbers understate #446's real-world cost by roughly an order of magnitude. - ---- - -### Standalone Mode: v0.2.5 vs v0.2.4 / etcd 3.2.0 - -| **Scenario** | **Metric** | **v0.2.4 (AWS)** | **v0.2.5 (AWS)** | **Δ vs v0.2.4** | **etcd 3.2.0²** | **Δ vs etcd** | -| ------------------- | ----------- | ---------------- | ---------------- | --------------- | --------------- | ------------- | -| Single Client Write | Throughput | 2,305 ops/s | 302 ops/s | **-86.9%** ⚠️ | 583 ops/s | -48.2% ⚠️ | -| | Avg Latency | 0.433 ms | 3.316 ms | **+666%** ⚠️ | 1.6 ms | +107.3% ⚠️ | -| | p99 Latency | 0.567 ms | 3.513 ms | **+520%** ⚠️ | — | — | -| High Conc. Write | Throughput | 59,687 ops/s | 70,395 ops/s | +17.9% ✅ | 44,341 ops/s | +58.8% ✅ | -| | Avg Latency | 3.346 ms | 14.137 ms | **+323%** ⚠️ | 22.0 ms | -35.7% ✅ | -| | p99 Latency | 7.106 ms | 20.943 ms | **+195%** ⚠️ | — | — | -| Linearizable Read | Throughput | 77,907 ops/s | 89,218 ops/s | +14.5% ✅ | 141,578 ops/s | -37.0% ⚠️ | -| | Avg Latency | 2.563 ms | 11.145 ms | **+335%** ⚠️ | 5.5 ms | +102.6% ⚠️ | -| | p99 Latency | 6.308 ms | 22.922 ms | **+263%** ⚠️ | — | — | -| Lease Read | Throughput | 76,144 ops/s | 99,699 ops/s | **+30.9%** ✅ | —³ | — | -| | Avg Latency | 2.624 ms | 9.972 ms | **+280%** ⚠️ | — | — | -| | p99 Latency | 6.130 ms | 22.058 ms | **+260%** ⚠️ | — | — | -| Eventual Read | Throughput | 170,000 ops/s | 243,775 ops/s | **+43.4%** ✅ | 185,758 ops/s | +31.2% ✅ | -| | Avg Latency | 1.171 ms | 4.044 ms | **+245%** ⚠️ | 2.2 ms | +83.8% ⚠️ | -| | p99 Latency | 2.245 ms | 9.087 ms | **+305%** ⚠️ | — | — | -| Hot-Key (10 keys) | Throughput | 90,609 ops/s | 104,701 ops/s | +15.6% ✅ | —³ | — | -| | Avg Latency | 2.203 ms | 9.493 ms | **+331%** ⚠️ | — | — | -| | p99 Latency | 6.127 ms | 19.060 ms | **+211%** ⚠️ | — | — | - -**Key Findings**: - -- **Same pattern as Embedded, more pronounced**: SC Write throughput -86.9%, avg latency 0.433ms → 3.316ms. Standalone adds real gRPC network RTT on top of the same fsync cost, so it's the worst-case number. -- **Concurrent scenarios (HC Write, Lin Read, Lease/Eventual Read, Hot-Key) all gain throughput (+15-43%)** — likely #392 ReadActor + other carried-over improvements outweighing #446's per-op cost once enough concurrency hides it — **but every one of them has 2-4x worse avg/p99 latency**. Throughput alone is not the full picture for this release; latency is where #446's cost is visible everywhere. -- **vs etcd**: d-engine has flipped from beating etcd on Single Client Write (+3.6x in v0.2.4) to losing to it (-48.2%). Linearizable Read is now worse than etcd on both throughput _and_ latency (previously only throughput was behind). This scenario deserves explicit sign-off before shipping v0.2.5 — RPO=0 is a deliberate trade-off (see ADR-038), but the "vs etcd" story materially changes. - -² etcd data sourced from [etcd official benchmark documentation](https://etcd.io/docs/v3.6/op-guide/performance/), tested on GCE infrastructure. Different cloud platform; results are for reference only. -³ etcd does not have an equivalent mode. - ---- - - - -```toml -[raft] -read_actor_channel_capacity = 10240 -read_actor_max_drain = 2000 - -[raft.persistence] -flush_policy = { Batch = { idle_flush_interval_ms = 1000 } } - -[raft.batching] -max_batch_size = 200 -``` - -Embedded-bench invocation: `--clients 1000` (AWS harness previously omitted this flag; Single Client Write always used `clients=1` regardless of this setting). +**Date**: October 18, 2026 | **Platform**: AWS us-east-1, 3 x c5.2xlarge nodes + 1 x c5.4xlarge client, 50 GB gp3 root volume (3,000 IOPS, 125 MB/s) | 3-round means | 8 B keys, 256 B values + +**Which numbers:** the d-engine v0.2.5 columns in Summary and Sections 1 and 2 are the best configuration measured (October 20, 2026): the Raft log in the optional raft-engine log store, an experimental setting that is not part of the v0.2.5 release. The default log store (RocksDB WAL, October 18) is in Sections 1b and 1c; full tables are in [`bench_report_v0.2.5_raft_engine.md`](bench_report_v0.2.5_raft_engine.md). + +Details: [`bench_report_v0.2.5_environment.md`](bench_report_v0.2.5_environment.md) (exact setup, commands, configuration) and [`bench_report_v0.2.5_details.md`](bench_report_v0.2.5_details.md) (every number, spreads, p99). + +## Summary + +1. **v0.2.5 makes every commit wait for a physical `fdatasync` (RPO=0, #446), and on AWS EBS that costs one disk sync per commit.** At the same concurrency, single-client write falls from 4,393 to 836 ops/s (avg latency 0.228 to 1.196 ms) and 100-in-flight embedded write from 115,641 to 19,860 ops/s (0.864 to 5.036 ms). The added latency is close to one `fdatasync` that overwrites preallocated files, 0.89 ms measured: 0.228 + 0.89 = 1.12 ms against 1.196 ms measured. With the default log store the sync costs an append `fdatasync` (2.71 ms with fio on the same volume type) and the single-client write is 337 ops/s (2.962 ms), Section 1b. +2. **Reads are mostly unaffected or faster.** In embedded mode lease and eventual reads are 2.6x faster (ReadActor, #392); in standalone mode they gain 12.3% and 4.1%. Linearizable read is 17% lower in embedded mode and 4.0% lower in standalone mode (within noise; the v0.2.4 value is one round). +3. **Against etcd v3.7.2 on the same AWS nodes, d-engine is ahead on writes and on eventual reads, in both modes.** Standalone (200 in-flight): writes 35,080 vs 17,143 ops/s (**2.05x**), single-client write 712 vs 605 ops/s (**1.18x**), eventual read vs etcd's serializable read 212,625 vs 147,426 ops/s (**1.44x**); linearizable read is 0.90x. Embedded (in-process calls, no network hop, 100 in-flight): writes 1.97x, single-client write 1.38x, linearizable read 5.74x, eventual read 8.43x. Every acknowledged write is synced to disk first, as in etcd. These are the numbers with the optional raft-engine log store, an experimental setting that is not part of the v0.2.5 release; with the default log store the single-client write is behind etcd (0.52x standalone, 0.56x embedded) and the 200-in-flight write is 1.87x. +4. **With the RocksDB as storage, etcd's single-client write is faster because its WAL sync is cheaper: 1.05 ms per sync in etcd's own metrics, against about 2.7 ms for an appending write on this volume.** fio shows a preallocated overwrite costs 1.29 ms on average against 2.72 ms for an append, with about half the device writes per sync. Measured since: the RocksDB WAL sync averages 2.7 ms per commit on this volume (the append figure), against 0.9 ms for a log that overwrites preallocated files (Sections 1b and 1c). +5. **The cost depends on the disk.** d-engine provides developer the flexibility to switch to any storage engine. + +![d-engine v0.2.4 vs v0.2.5, Embedded Mode, 100 in-flight](d-engine_v0.2.5_vs_v0.2.4_embedded_mode.png) + +![d-engine v0.2.4 vs v0.2.5, Standalone Mode, 200 in-flight](d-engine_v0.2.5_vs_v0.2.4_standalone_mode.png) + +![d-engine v0.2.5 (Embedded) vs etcd v3.7.2, 100 in-flight](d-engine_comparison_v0.2.5.png) + +Charts (log scale, same data as Sections 1 and 2): v0.2.4 vs v0.2.5 in embedded mode at 100 in-flight; the same in standalone mode at 200 in-flight; d-engine v0.2.5 embedded against etcd v3.7.2 at 100 in-flight. There is no chart for standalone against etcd; that pair is in the Section 2 table. + +## 1. v0.2.5 vs v0.2.4, same concurrency + +Embedded mode, 100 in-flight requests (single client: 1). Cells are ops/s (average latency). v0.2.5 = the best configuration (see the note at the top). + +| Scenario | v0.2.4 | v0.2.5 | Throughput | +| ---------------------- | ------------------ | -------------------- | ---------- | +| Single client write | 4,393 (0.228 ms) | 836 (1.196 ms) | -81.0% | +| High concurrency write | 115,641 (0.864 ms) | 19,860 (5.036 ms) | -82.8% | +| Linearizable read | 441,087 (0.226 ms) | 368,068 (0.271 ms) | -16.6% | +| Lease read | 470,786 (0.212 ms) | 1,204,641 (0.001 ms) | +155.9% | +| Eventual read | 478,101 (0.209 ms) | 1,253,650 (0.001 ms) | +162.2% | +| Hot-key read (10 keys) | 467,864 (0.213 ms) | 356,768 (0.280 ms) | -23.7% | + +Standalone mode, 200 in-flight (single client: 1). Starred cells use the one clean round of v0.2.4 (the cluster was not ready in rounds 1 and 2, so they logged timeouts). + +| Scenario | v0.2.4 | v0.2.5 | Throughput | +| ---------------------- | ------------------- | ------------------ | ---------- | +| Single client write | 2,042 (0.494 ms) | 712 (1.404 ms) | -65.1% | +| High concurrency write | 63,308* (3.155 ms) | 35,080 (5.696 ms) | -44.6% | +| Linearizable read | 85,018* (2.349 ms) | 81,610 (2.448 ms) | -4.0% | +| Lease read | 83,181 (2.403 ms) | 93,372 (2.139 ms) | +12.3% | +| Eventual read | 204,337 (0.975 ms) | 212,625 (0.937 ms) | +4.1% | +| Hot-key read (10 keys) | 104,149* (1.917 ms) | 95,466 (2.093 ms) | -8.3% | + +### 1b. Log store effect, embedded: default RocksDB WAL vs raft-engine (experimental) + +Both columns are v0.2.5, ops/s (average latency). The default column is October 18 on another instance set; raft-engine is October 20. + +| Scenario | RocksDB WAL, default | raft-engine | Throughput | +| ---------------------- | -------------------- | -------------------- | ---------- | +| Single client write | 337 (2.962 ms) | 836 (1.196 ms) | +147.6% | +| High concurrency write | 18,198 (5.494 ms) | 19,860 (5.036 ms) | +9.1% | +| Linearizable read | 373,820 (0.267 ms) | 368,068 (0.271 ms) | -1.5% | +| Eventual read | 1,350,684 (0.001 ms) | 1,253,650 (0.001 ms) | -7.2% | + +### 1c. Log store effect, standalone (200 in-flight) + +| Scenario | RocksDB WAL, default | raft-engine | Throughput | +| ---------------------- | -------------------- | ------------------ | ---------- | +| Single client write | 312 (3.204 ms) | 712 (1.404 ms) | +128.2% | +| High concurrency write | 32,068 (6.228 ms) | 35,080 (5.696 ms) | +9.4% | +| Linearizable read | 82,215 (2.432 ms) | 81,610 (2.448 ms) | -0.7% | +| Eventual read | 217,464 (0.916 ms) | 212,625 (0.937 ms) | -2.2% | + +## 2. d-engine v0.2.5 vs etcd v3.7.2 on the same AWS nodes + +etcd ran in a separate session on new instances of the same shape, three times from a freshly wiped cluster, with etcd's documented `benchmark` commands (leader address instead of the removed `--target-leader`). Both d-engine modes are compared, because d-engine is used both ways. Standalone is the like-for-like pair: etcd is also reached through a network client. d-engine = the best configuration (see the note at the top). + +**Standalone** (200 in-flight; single client: 1) + +| Case | etcd v3.7.2 | d-engine v0.2.5 standalone | d-engine / etcd | +| --------------------------------------------------- | ----------------------------------- | ------------------------------------ | --------------- | +| Write, 1 in-flight | 605 ops/s, 1.65 ms, p99 5.07 ms | 712 ops/s, 1.40 ms, p99 3.15 ms | 1.18x | +| Write, 200 in-flight | 17,143 ops/s, 11.7 ms, p99 19.7 ms | 35,080 ops/s, 5.70 ms, p99 12.28 ms | 2.05x | +| Linearizable read, 200 | 90,569 ops/s, 2.20 ms, p99 6.03 ms | 81,610 ops/s, 2.45 ms, p99 4.58 ms | 0.90x | +| Serializable (etcd) / eventual (d-engine) read, 200 | 147,426 ops/s, 1.23 ms, p99 6.40 ms | 212,625 ops/s, 0.937 ms, p99 1.85 ms | 1.44x | + +**Embedded** (100 in-flight; single client: 1). The embedded bench ran at 100 in-flight, so etcd's 100 in-flight rows are the matched ones. + +| Case | etcd v3.7.2 | d-engine v0.2.5 embedded | d-engine / etcd | +| --------------------------------------------------- | ------------------------------------ | --------------------------------------- | --------------- | +| Write, 1 in-flight | 605 ops/s, 1.65 ms, p99 5.07 ms | 836 ops/s, 1.20 ms, p99 3.00 ms | 1.38x | +| Write, 100 in-flight | 10,064 ops/s, 9.93 ms, p99 17.0 ms | 19,860 ops/s, 5.04 ms, p99 12.04 ms | 1.97x | +| Linearizable read, 100 | 64,071 ops/s, 1.57 ms, p99 3.60 ms | 368,068 ops/s, 0.271 ms, p99 0.332 ms | 5.74x | +| Serializable (etcd) / eventual (d-engine) read, 100 | 148,709 ops/s, 0.633 ms, p99 3.60 ms | 1,253,650 ops/s, 0.001 ms, p99 0.002 ms | 8.43x | + +An embedded call pays no serialization and no network, so the read ratios measure that, not only the engine. The single-client write is the case where the network matters least (both sides wait for the disk sync). With the default log store d-engine is behind etcd there (0.52x standalone, 0.56x embedded); its other default-store ratios are 1.87x (200 in-flight write), 0.91x (linearizable read) and 1.48x (eventual read) standalone, and 1.81x, 5.83x and 9.08x embedded. + +etcd's WAL `fdatasync` averaged 1.05 ms at 1 in-flight, 2.16 ms at 100, 2.37 ms at 200 and 3.24 ms at 1,000. Its other cases (1,000 in-flight, writes to all members, single-client reads) are in the details file. d-engine's lease read has no etcd equivalent (1,204,641 ops/s embedded, 93,372 standalone). + +## Limitations + +- Three rounds per case; d-engine v0.2.4 and the default log store (set A), d-engine's best numbers (sets E and F) and etcd (set B) ran on different instances of the same shape. Treat ratios as indicative, roughly plus or minus 20%. +- v0.2.4 standalone write, linearizable read and hot-key use a single clean round. v0.2.4 embedded single-client write varies 17.7% between rounds. +- d-engine was not run on AWS at 1,000 in-flight, nor for single-client reads, so etcd's 1,000 in-flight results (in the details file) have no d-engine pair. etcd's reads fetch one key and last only a second or two; d-engine's read benchmark reads many keys. +- The cost of the RocksDB WAL sync was measured (2.68 to 2.78 ms per commit, from the servers' own metrics); how RocksDB writes the WAL was not traced. Its configuration does not set `recycle_log_file_num` and leaves WAL preallocation at the defaults. The fio overwrite result is bimodal (about 0.9 ms or 2.7 ms); on the raft-engine runs 1.2% of the syncs took 1.6 to 3.2 ms, which matches the share of commits that coincide with a journal transaction, but this was not shown to be the same events. +- The best numbers (Summary points 1 to 3, Sections 1, 1b, 1c, 2) come from the optional raft-engine log store on new instance sets, not set A or B, and from a forked build of raft-engine (the published crate's `Engine::sync()` does nothing). Only the standalone default-store control was repeated on the new set. The p99 of the 100 to 200 in-flight writes is higher than with the RocksDB WAL (about 12 vs 6 to 7 ms); the cause was not investigated. +- Single AZ, no placement group, gp3 only. Other volume types (io2, local NVMe) were not measured. diff --git a/benches/reports/v0.2.5/bench_report_v0.2.5_details.md b/benches/reports/v0.2.5/bench_report_v0.2.5_details.md new file mode 100644 index 00000000..c89feb6d --- /dev/null +++ b/benches/reports/v0.2.5/bench_report_v0.2.5_details.md @@ -0,0 +1,76 @@ +# d-engine v0.2.5 Benchmark: Full Results + +Companion to `bench_report_v0.2.5.md`. Environment: `bench_report_v0.2.5_environment.md`. +All values are 3-round means unless marked. Spread = (max - min) / mean between rounds. + +## 1. Embedded, 100 in-flight (instance set A) + +| Scenario | v0.2.4 ops/s | avg / p99 ms | v0.2.5 ops/s | avg / p99 ms | Throughput change | +|---|---|---|---|---|---| +| Single client write (1 in-flight) | 4,393 (spread 17.7%) | 0.228 / 0.305 | 337 (0.5%) | 2.962 / 3.153 | -92.3% | +| High concurrency write | 115,641 (5.4%) | 0.864 / 1.064 | 18,198 (1.1%) | 5.494 / 6.203 | -84.3% | +| Linearizable read | 441,087 (5.0%) | 0.226 / 0.269 | 373,820 (0.3%) | 0.267 / 0.329 | -15.3% | +| Lease read | 470,786 (10.3%) | 0.212 / 0.259 | 1,319,390 (1.8%) | 0.001 / 0.002 | +180.3% | +| Eventual read | 478,101 (12.8%) | 0.209 / 0.247 | 1,350,684 (1.7%) | 0.001 / 0.002 | +182.5% | +| Hot-key read (10 keys) | 467,864 (2.7%) | 0.213 / 0.255 | 369,349 (5.6%) | 0.270 / 0.337 | -21.1% | + +Lease and eventual reads return in about a microsecond, so their in-flight count (throughput x latency) is +about 1, not 100: the client side is the limit there. Every other row measured 100 (single client: 1). + +## 2. Standalone, 200 in-flight (instance set A) + +| Scenario | v0.2.4 ops/s | avg / p99 ms | v0.2.5 ops/s | avg / p99 ms | Throughput change | +|---|---|---|---|---|---| +| Single client write | 2,042 (23.7%) | 0.494 / 0.650 | 312 (1.2%) | 3.204 / 3.400 | -84.7% | +| High concurrency write | 63,308 (round 3 only) | 3.155 / 4.727 | 32,068 (0.8%) | 6.228 / 7.428 | -49.3% | +| Linearizable read | 85,018 (round 3 only) | 2.349 / 3.729 | 82,215 (8.3%) | 2.432 / 4.692 | -3.3% | +| Lease read | 83,181 (7.4%) | 2.403 / 3.684 | 95,219 (7.1%) | 2.099 / 4.258 | +14.5% | +| Eventual read | 204,337 (1.7%) | 0.975 / 1.723 | 217,464 (0.8%) | 0.916 / 1.714 | +6.4% | +| Hot-key read (10 keys) | 104,149 (round 3 only) | 1.917 / 3.037 | 96,553 (2.3%) | 2.068 / 3.846 | -7.3% | + +"Round 3 only": in rounds 1 and 2 of v0.2.4 the cluster was not ready when the bench started and the +client logged `ConnectionTimeout` / `DeadlineExceeded` (write: 5,488 and 16,076 ops/s; linearizable read and +hot-key also affected). Their means are meaningless, so only the clean round is used and the spread is unknown. +v0.2.5 had no errors in any round. + +## 3. etcd v3.7.2 (instance set B, three fresh-cluster runs) + +| Case | ops/s (spread) | avg ms | p99 ms | WAL fsync avg ms (members' range) | +|---|---|---|---|---| +| Write 1 in-flight | 605 (0.6%) | 1.654 | 5.07 | 1.05 (1.05 to 1.05) | +| Write 100, leader | 10,064 (0.3%) | 9.933 | 16.97 | 2.16 (2.11 to 2.24) | +| Write 200, leader | 17,143 (0.3%) | 11.667 | 19.67 | 2.37 (2.28 to 2.49) | +| Write 1,000, leader | 40,011 (0.5%) | 24.867 | 43.23 | 3.24 (3.10 to 3.44) | +| Write 1,000, all members | 42,405 (0.7%) | 23.433 | 44.47 | 3.45 (3.31 to 3.54) | +| Linearizable read 1 / 100 / 200 / 1,000 | 1,624 / 64,071 / 90,569 / 150,109 | 0.616 / 1.567 / 2.200 / 6.267 | 0.80 / 3.60 / 6.03 / 25.03 | - | +| Serializable read 1 / 100 / 200 / 1,000 | 4,217 / 148,709 / 147,426 / 202,464 | 0.237 / 0.633 / 1.233 / 4.300 | 0.40 / 3.60 / 6.40 / 21.27 | - | + +The 1 in-flight average is 1 / throughput (the tool prints 0.1 ms steps). Reads run only a second or two, +so their measured in-flight count is below nominal: serializable 94 / 182 / 870 and linearizable 941 at +nominal 100 / 200 / 1,000; this flatters etcd's read latency slightly. + +## 4. Matched comparison on AWS (d-engine / etcd) + +Latency columns are etcd / d-engine, so values above 1 favor d-engine. The d-engine runs are from set A and +the etcd runs from set B (different instances of the same shape). + +| Pair | etcd ops/s | d-engine ops/s | Throughput x | Avg latency x | p99 x | +|---|---|---|---|---|---| +| Write, 1 in-flight vs standalone | 605 | 312 | 0.52 | 0.52 | 1.49 | +| Write, 200 in-flight vs standalone | 17,143 | 32,068 | 1.87 | 1.87 | 2.65 | +| Linearizable read, 200 vs standalone | 90,569 | 82,215 | 0.91 | 0.90 | 1.29 | +| Serializable vs eventual read, 200 vs standalone | 147,426 | 217,464 | 1.48 | 1.35 | 3.73 | +| Write, 1 in-flight vs embedded | 605 | 337 | 0.56 | 0.56 | 1.61 | +| Write, 100 vs embedded | 10,064 | 18,198 | 1.81 | 1.81 | 2.74 | +| Linearizable read, 100 vs embedded | 64,071 | 373,820 | 5.83 | 5.88 | 10.93 | +| Serializable vs eventual read, 100 vs embedded | 148,709 | 1,350,684 | 9.08 | about 600 (0.633 ms vs 0.001 ms) | about 1,800 (3.60 ms vs 0.002 ms) | + +Embedded has no network hop (in-process calls), so its read rows show the cost of the network client as +much as the engine. The latency ratio of the last row is not meaningful: embedded eventual reads return +in about a microsecond, at the resolution of the measurement. d-engine was not run on AWS at 1 or 1,000 +in-flight for reads, nor at 200 in-flight in embedded mode, nor at 1,000 for writes, in this session. + +## 5. Not measured in this session + +d-engine on AWS at 1,000 in-flight and for single-client reads; etcd with v0.2.4 (it +does not fsync every commit); a d-engine run with a different RocksDB WAL configuration. diff --git a/benches/reports/v0.2.5/bench_report_v0.2.5_environment.md b/benches/reports/v0.2.5/bench_report_v0.2.5_environment.md new file mode 100644 index 00000000..ff02486b --- /dev/null +++ b/benches/reports/v0.2.5/bench_report_v0.2.5_environment.md @@ -0,0 +1,76 @@ +# d-engine v0.2.5 Benchmark: Test Environment + +Companion to `bench_report_v0.2.5.md`. Everything here was measured on **October 18, 2026**, except the raft-engine run of +**October 20, 2026** (new instance sets E and F, same shapes), described in `bench_report_v0.2.5_raft_engine.md`. + +## 1. Instance sets + +Three sets of AWS instances were used, all created from the same Terraform configuration (same shapes), +hours apart. A number is only comparable with another number from the same set, or with the caveat that +the instances differ. + +| Set | Used for | +|---|---| +| A | d-engine v0.2.4 and v0.2.5 runs (v0.2.4 first, v0.2.5 about 12 minutes later) | +| B | etcd v3.7.2 runs (three fresh-cluster runs) and the second fio calibration | +| C | the first fio calibration and node-to-node ping | + +## 2. AWS shape + +| Item | Value | +|---|---| +| Region / zone | us-east-1, one availability zone, one subnet (10.0.1.0/24), no placement group | +| Server nodes | 3 x c5.2xlarge (8 vCPU, 16 GiB) | +| Client machine | 1 x c5.4xlarge (16 vCPU, 32 GiB), used by the standalone and etcd benches | +| OS | Ubuntu 20.04 LTS (Canonical AMI), ext4 root filesystem | +| Disk | 50 GB gp3 root volume, 3,000 IOPS, 125 MB/s. The data of every system under test is on this volume | +| Network | security group allows all traffic between members; client to node RTT 0.11 to 0.15 ms (set B, ping averages); node to node 0.09 and 0.18 ms (set C, ping averages to two peers) | + +## 3. d-engine runs (set A) + +- v0.2.5 = `main` at `b5fce385`. v0.2.4 = tag `v0.2.4`. Both built on the instances with the release profile. +- **Embedded**: `embedded-bench --mode local --batch --key-size 8 --value-size 256 --sequential-keys put`, + one process per node, the leader's output is used. 100 in-flight (v0.2.4's `--batch` hard-codes 100; + v0.2.5 gets `--clients 100`). The bench runtime has 2 worker threads on both versions (v0.2.4's bench was + patched to match v0.2.5's). +- **Standalone**: `three-nodes-standalone` on each node, `standalone-bench` on the client with three + endpoints. `--conns 200 --clients 200` (v0.2.4 limits in-flight requests by `--conns`, v0.2.5 by `--clients`; + both are 200). Single-client write: `--conns 1 --clients 1`. +- Payload: 8 B keys, 256 B values, sequential keys; hot-key uses 10 keys. 10,000 requests for the + single-client write, 100,000 for the others. 3 rounds; cluster restarted and data wiped before each round. +- Configuration (same file serves both versions; each ignores keys it does not know): + `max_batch_size = 200`, snapshot off, `unified_db = false`, backpressure 10,000 writes / 50,000 reads, + election timeout 1,500 to 2,500 ms and lease 500 ms (standalone), ReadActor 2,048/1,024 (embedded) and + 10,240/2,000 (standalone). Replication settings are left at their defaults. +- **Persistence**: v0.2.4 `MemFirst` (asynchronous flush, 1,000 ms idle interval). v0.2.5 advances the durable + index only after `fdatasync` (RPO=0). The raft log is stored in RocksDB. + +## 4. etcd runs (set B) + +- etcd v3.7.2 official linux-amd64 binary, downloaded on the instances. Three members, one per node, data + directory on the root volume, default settings apart from the cluster flags. +- The `benchmark` tool is not in the release package. It was built from the v3.7.2 source and + cross-compiled to a static linux/amd64 binary (Go 1.26.8), then run on the client machine. +- Commands are etcd's documented ones, except that `--target-leader` no longer exists in v3.7.2, so the + leader's address is passed instead. 8 B keys, 256 B values, sequential keys. Reads fetch one key through + all three members. +- Cases per run: writes at 1, 100, 200 and 1,000 in-flight to the leader and 1,000 in-flight to all + members; linearizable and serializable reads at 1, 100, 200 and 1,000 in-flight. 10,000 requests at 1 + in-flight, 100,000 otherwise. `--conns` equals `--clients`, except at 1,000 in-flight (100 connections, as + in etcd's documentation). +- Three runs, each on a freshly wiped cluster. WAL fsync averages come from the change of + `etcd_disk_wal_fsync_duration_seconds` before and after each case. + +## 5. Published etcd numbers (not used for any comparison here) + +etcd's documentation table comes from Google Cloud: 3 servers with 8 vCPU, 16 GB and a 50 GB SSD, a client with +16 vCPU and 30 GB, Ubuntu 17.04, etcd 3.2.0, Go 1.8.3. The disk type is not stated. The cloud, disk, software +version and age all differ from this report, so the earlier draft's comparison with those numbers was removed. + +## 6. Calibration and other runs + +- **fio** (`--ioengine=sync --fdatasync=1`, 1 KiB, iodepth 1, 20 s) on the same volume type: append write + 2.710 ms (set C) and 2.718 ms (set B) average `fdatasync`; preallocated overwrite 1.288 ms (set B). + 4 KiB overwrite 0.873 ms and 16 KiB overwrite 2.855 ms (set C). +- **Local runs** (not in the tables here): Mac mini M2, three nodes on loopback, each on its own RAM disk, where + `fdatasync` costs about 0.05 ms. They show the RPO=0 cost when the disk is almost free. diff --git a/benches/reports/v0.2.5/bench_report_v0.2.5_raft_engine.md b/benches/reports/v0.2.5/bench_report_v0.2.5_raft_engine.md new file mode 100644 index 00000000..5fe476ce --- /dev/null +++ b/benches/reports/v0.2.5/bench_report_v0.2.5_raft_engine.md @@ -0,0 +1,82 @@ +# d-engine v0.2.5 Benchmark: raft-engine log store (embedded and standalone) + +Companion to `bench_report_v0.2.5.md`. Measured on **October 20, 2026** on two new instance sets (E and F). +Experimental: the log store is a spike (#460), not part of a release. Values are 3-round means; spread = (max - min) / mean. + +## 1. What was measured + +| Item | Value | +|---|---| +| Engine | The Raft log and the hard state are stored in raft-engine (a fork pinned at `e5ef995`: upstream master `097c499` with the `lz4-sys` pin relaxed); the state machine is RocksDB. Embedded: `EmbeddedEngine`; standalone: `three-nodes-standalone` built with `--features raft-engine` (`StandaloneEngine::run_custom`). Branch `spike/460-fdatasync` | +| Control | Stock v0.2.5: Raft log in a RocksDB WAL, `unified_db = false`; the October 18 report (set A). For standalone there is also a RocksDB run on set F itself (Section 3) | +| Instances | Same shapes as the other sets: 3 x c5.2xlarge, 50 GB gp3 root volume (3,000 IOPS, 125 MB/s), us-east-1. Set E: embedded, 07:24 to 07:27 UTC. Set F: embedded 14:20 to 14:23 UTC, standalone from 14:41 UTC. Instance IDs were not recorded | +| Bench | Embedded: `embedded-bench --mode local --batch`, 100 in-flight, leader output (set F: node2, node3, node3). Standalone: `standalone-bench` on the client, 200 in-flight, single client 1. 8 B keys, 256 B values, data wiped between rounds | +| Config | Snapshots disabled, as in the earlier reports. `[raft_engine]`: `target_file_size_mib = 1`, `purge_threshold_mib = 512`, `prefill_limit_mib = 512` (every start created 768 files, which is 512 x 1.5) | + +## 2. Embedded, 100 in-flight (set F; set E in the third column) + +Cells are ops/s (average latency). Control = October 18, RocksDB WAL. + +| Scenario | raft-engine, set F | Spread | raft-engine, set E | Control | Set F vs control | +|---|---|---|---|---|---| +| Single client write | 836 (1.196 ms, p99 3.00 ms) | 2.8% | 693 (1.485 ms) | 337 (2.962 ms, p99 3.15) | +147.6% | +| High concurrency write | 19,860 (5.036 ms, p99 12.04 ms) | 4.8% | 20,271 (4.932 ms) | 18,198 (5.494 ms, p99 6.20) | +9.1% | +| Linearizable read | 368,068 (0.271 ms) | 4.0% | 363,803 | 373,820 | -1.5% | +| Lease read | 1,204,641 | 5.9% | 1,213,826 | 1,319,390 | -8.7% | +| Eventual read | 1,253,650 | 4.4% | 1,236,197 | 1,350,684 | -7.2% | +| Hot-key read | 356,768 (0.280 ms) | 6.6% | 360,246 | 369,349 | -3.4% | + +Per round on set F, single client write: 850 / 830 / 827 ops/s; high concurrency write: 20,075 / 20,232 / 19,273 ops/s. +Throughput x latency is 1.0 and 100.0 for the writes, 99.7 for the reads that use 100 in-flight. + +## 3. Standalone, 200 in-flight (set F) + +"Same-set RocksDB" is a standalone run on set F whose build did not get the raft-engine feature, so it ran stock RocksDB. + +| Scenario | raft-engine | Spread | Oct 18 RocksDB | Same-set RocksDB | vs Oct 18 | +|---|---|---|---|---|---| +| Single client write | 712 (1.404 ms, p99 3.15 ms) | 1.5% | 312 (3.204 ms, p99 3.40) | 299 (3.343 ms) | +128.2% | +| High concurrency write | 35,080 (5.696 ms, p99 12.28 ms) | 0.4% | 32,068 (6.228 ms, p99 7.43) | 31,818 (6.277 ms) | +9.4% | +| Linearizable read | 81,610 (2.448 ms) | 5.5% | 82,215 | 81,094 | -0.7% | +| Lease read | 93,372 | 5.8% | 95,219 | 93,077 | -1.9% | +| Eventual read | 212,625 (0.937 ms) | 1.1% | 217,464 | 212,492 | -2.2% | +| Hot-key read | 95,466 | 5.6% | 96,553 | 93,985 | -1.1% | + +Throughput x latency is 1.0 for the single client and 199.2 to 199.8 for the rest. No timeouts or failed requests in any round. +The two RocksDB runs differ by 0.3% to 4.1%, so the October 18 control is a fair reference for the standalone writes. + +## 4. Against etcd v3.7.2 (October 18, instance set B) + +| Case | etcd | Ratio with raft-engine | Ratio with the RocksDB WAL | +|---|---|---|---| +| Standalone write, 1 in-flight | 605 ops/s | **1.18x** | 0.52x | +| Standalone write, 200 in-flight | 17,143 ops/s | **2.05x** | 1.87x | +| Standalone linearizable read | 90,569 ops/s | 0.90x | 0.91x | +| Standalone eventual vs etcd serializable read | 147,426 ops/s | 1.44x | 1.48x | +| Embedded write, 1 in-flight | 605 ops/s | 1.38x | 0.56x | +| Embedded write, 100 in-flight | 10,064 ops/s | 1.97x | 1.81x | + +## 5. Evidence that a commit waits for a physical sync + +- Standalone, set F: each server's `core_raft_fsync_duration_ms` averaged 0.867 to 0.948 ms in all nine node-rounds (about 12,600 + flushes each), against 2.68 to 2.78 ms for the RocksDB run on the same set. The fio overwrite floor is 0.78 to 0.92 ms. +- Embedded, set E (`verify-fsync.sh`, 07:11 UTC, 20,000 single-client writes): average `flush()` 0.894 ms, 20,022 `fdatasync` calls for + 20,001 commits, 288 journal transactions (1.4% of the commits). On the leader 95.3% of the flushes took 0.8 to 1.6 ms and 1.2% took 1.6 + to 3.2 ms, which matches the journal share; whether these are the same events was not checked. One follower had 0.4% of its flushes + above 25 ms (up to 102 ms), not explained. For contrast, the run with the no-op sync made 19 `fdatasync` calls per node. +- Embedded, set F: no separate run. The single client write averages 1.196 ms, above the fastest sync seen on this disk (0.775 ms). +- An earlier run on October 20 showed 0.21 ms commits because the published raft-engine 0.4.2 has a no-op `Engine::sync()`; it is discarded. + +## 6. Reading + +- A sync that overwrites preallocated files makes the single client write 2.3 to 2.5 times faster in both modes. At 100 to 200 in-flight + it is 9% faster and the p99 is higher (12 vs 6 to 7 ms). The sync is not the only limit at that concurrency; not investigated. +- Against etcd, the single client write goes from behind (0.52x, 0.56x) to ahead (1.18x, 1.38x). Reads do not depend on the log store. + +## 7. Limits + +- The control is from another instance set (A), except the standalone same-set run. Embedded reads differ by 3% to 9% between sets. +- Snapshots are off, so the log is never purged and the recycle pool is consumed during a run (about 33 files per run against 768). +- The standalone raft-engine path starts the state machine through `run_custom`, which does not attach the TTL lease manager; no + benchmark uses TTL. The fork is a stop-gap for a no-op `Engine::sync()` in the published crate and a pinned `lz4-sys`. +- Not measured: 1,000 in-flight, a RocksDB embedded run on set F, the cause of the higher p99, power-loss behaviour, lz4 compression (off). diff --git a/benches/reports/v0.2.5/d-engine_comparison_v0.2.5.png b/benches/reports/v0.2.5/d-engine_comparison_v0.2.5.png index f6524404..dccb0b6d 100644 Binary files a/benches/reports/v0.2.5/d-engine_comparison_v0.2.5.png and b/benches/reports/v0.2.5/d-engine_comparison_v0.2.5.png differ diff --git a/benches/reports/v0.2.5/d-engine_v0.2.5_vs_v0.2.4_embedded_mode.png b/benches/reports/v0.2.5/d-engine_v0.2.5_vs_v0.2.4_embedded_mode.png index bee61812..960c2387 100644 Binary files a/benches/reports/v0.2.5/d-engine_v0.2.5_vs_v0.2.4_embedded_mode.png and b/benches/reports/v0.2.5/d-engine_v0.2.5_vs_v0.2.4_embedded_mode.png differ diff --git a/benches/reports/v0.2.5/d-engine_v0.2.5_vs_v0.2.4_standalone_mode.png b/benches/reports/v0.2.5/d-engine_v0.2.5_vs_v0.2.4_standalone_mode.png index 032e71b9..348d50ac 100644 Binary files a/benches/reports/v0.2.5/d-engine_v0.2.5_vs_v0.2.4_standalone_mode.png and b/benches/reports/v0.2.5/d-engine_v0.2.5_vs_v0.2.4_standalone_mode.png differ diff --git a/d-engine-core/src/errors.rs b/d-engine-core/src/errors.rs index a5e0a04f..a0039480 100644 --- a/d-engine-core/src/errors.rs +++ b/d-engine-core/src/errors.rs @@ -476,6 +476,33 @@ pub enum ReplicationError { #[error("Replication requires leader role (known leader: {leader_id:?})")] NotLeader { leader_id: Option }, + /// The entries chosen for a peer are not one contiguous run of indices. This is a bug + /// in how the batch was built, never a normal Raft condition. + #[error( + "non-contiguous batch for peer {peer_id}: indices {first_index}..={last_index} \ + but {len} entries" + )] + NonContiguousBatch { + peer_id: u32, + first_index: u64, + last_index: u64, + len: usize, + }, + + /// An AppendEntries request whose entries are not consecutive indices starting at + /// `prev_log_index + 1`. It is rejected before the log is touched, so the log never gets a + /// hole. + #[error( + "non-contiguous AppendEntries: prev_log_index {prev_log_index}, \ + entries {first_index}..={last_index}, {len} entries" + )] + NonContiguousAppend { + prev_log_index: u64, + first_index: u64, + last_index: u64, + len: usize, + }, + /// `entry_term` lookup for `prev_log_index` failed even though the index is /// within the leader's own retained range [first_index, last_index] — an /// unexplained storage-layer gap, not a normal Raft condition. diff --git a/d-engine-core/src/raft_role/leader_state.rs b/d-engine-core/src/raft_role/leader_state.rs index 28601bd6..03af3ce1 100644 --- a/d-engine-core/src/raft_role/leader_state.rs +++ b/d-engine-core/src/raft_role/leader_state.rs @@ -259,6 +259,7 @@ impl ReplicationWorkerConfig { /// Per-peer metric handles, built once so the hot path skips the label /// allocation and registry lookup that `metrics::gauge!(..)` does per call. struct PeerMetrics { + batch_over_cap_total: metrics::Counter, in_flight: metrics::Gauge, match_index: metrics::Gauge, last_ack_timestamp_seconds: metrics::Gauge, @@ -270,6 +271,10 @@ impl PeerMetrics { fn new(peer_id: u32) -> Self { let peer = peer_id.to_string(); Self { + batch_over_cap_total: metrics::counter!( + "core.raft.peer.batch_over_cap_total", + "peer_id" => peer.clone() + ), in_flight: metrics::gauge!("core.raft.peer.in_flight", "peer_id" => peer.clone()), match_index: metrics::gauge!("core.raft.peer.match_index", "peer_id" => peer.clone()), last_ack_timestamp_seconds: metrics::gauge!( @@ -3619,6 +3624,12 @@ impl LeaderState { if !is_heartbeat { let occupancy = self.in_flight_count(peer_id) as f64; + if request.entries.len() as u64 + > self.node_config.raft.replication.append_entries_max_entries_per_replication + { + self.peer_metrics(peer_id).batch_over_cap_total.increment(1); + } + self.peer_metrics(peer_id).in_flight_at_dispatch.record(occupancy); metrics::histogram!("core.raft.replication.entries_per_request") .record(request.entries.len() as f64); diff --git a/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs b/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs index e30085d6..fd4d31b9 100644 --- a/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs +++ b/d-engine-core/src/raft_role/leader_state_test/probe_backpressure_test.rs @@ -916,6 +916,70 @@ async fn test_dispatch_metrics_record_occupancy_entries_and_gated_count() { ); } +/// `batch_over_cap_total` counts requests that carry more entries than the per-request cap. +/// +/// # Scenario +/// - Cap is 2 entries per request +/// - Three dispatches to peer 2 carry 2, 3 and 4 entries; a fourth is a heartbeat +/// +/// # Test Purpose +/// A developer reads this counter to learn that the cap is too small for the real batch +/// size. A request exactly at the cap is not over it, and a heartbeat carries no batch, so +/// only the 3- and 4-entry requests count. Must fail if the comparison becomes `>=` or +/// heartbeats are counted. +#[tokio::test] +#[traced_test] +async fn test_batch_over_cap_counter_counts_only_requests_above_the_cap() { + let capture = MetricsCapture::new(); + let _guard = metrics::set_default_local_recorder(&capture); + + let (mut ctx, mut state, _task_rx, internal_event_tx) = + setup_gate_harness("/tmp/test_batch_over_cap_counter").await; + + let mut cfg = (*ctx.node_config).clone(); + cfg.raft.replication.append_entries_max_entries_per_replication = 2; + state.node_config = Arc::new(cfg); + + fn request_with(entry_count: u64) -> AppendEntriesRequest { + AppendEntriesRequest { + entries: (1..=entry_count) + .map(|index| Entry { + index, + term: 1, + payload: None, + }) + .collect(), + ..AppendEntriesRequest::default() + } + } + + let mut sizes = vec![2_u64, 3, 4, 0].into_iter(); + ctx.handlers + .replication_handler + .expect_prepare_batch_requests() + .times(4) + .returning(move |_, _, _, _, _, _| { + let n = sizes.next().expect("one size per call"); + Ok(crate::PrepareResult { + append_requests: vec![(2, request_with(n), 1)], + snapshot_targets: vec![], + }) + }); + + state.set_peer_replication_state(2, PeerReplicationState::Replicate); + state.next_index.insert(2, 1); + + for _ in 0..4 { + state.process_batch(one_entry_batch(), &internal_event_tx, &ctx).await.unwrap(); + } + + assert_eq!( + capture.counter("core.raft.peer.batch_over_cap_total", &[("peer_id", "2")]), + 2, + "only the 3- and 4-entry requests exceed a cap of 2" + ); +} + /// A heartbeat (empty entries) occupies no window slot, so it must neither be counted as /// gated when the window is full nor contribute to the dispatch histograms. #[tokio::test] diff --git a/d-engine-core/src/replication/mod.rs b/d-engine-core/src/replication/mod.rs index f2ec3c2b..b75f0095 100644 --- a/d-engine-core/src/replication/mod.rs +++ b/d-engine-core/src/replication/mod.rs @@ -5,11 +5,6 @@ //! - Follower log consistency checks //! - Conflict resolution algorithms mod replication_handler; -pub use replication_handler::*; - -#[cfg(test)] -pub mod replication_handler_test; - // Client Request Extension Definition // ----------------------------------------------------------------------------- use std::collections::HashMap; @@ -24,6 +19,7 @@ use d_engine_proto::server::replication::ConflictResult; use d_engine_proto::server::replication::SuccessResult; #[cfg(any(test, feature = "__test_support"))] use mockall::automock; +pub use replication_handler::*; use tonic::Status; use super::LeaderStateSnapshot; @@ -178,24 +174,30 @@ where leader_commit_index: u64, ) -> Option; - /// Gathers legacy logs for lagging peers, classifying each peer's outcome. + /// Decides what each follower gets in this round's AppendEntries. + /// + /// The leader's log is split at `leader_last_index_before_inserting_new_entries`: + /// entries at or below it are "legacy" (already in the log), and `new_entries` are + /// the ones appended this round. A follower that is behind needs legacy first. + /// + /// Per follower, the result is one of: + /// - `NeedSnapshot`: its `next_index` is below `first_index`, the entries are already purged, + /// so only a snapshot can catch it up. + /// - `CorruptGap`: the legacy range read back short or with a hole although it was not purged. + /// That is a storage fault, so the caller must not hide it behind a snapshot. + /// - `Ready(entries)`: legacy entries from `next_index`. `new_entries` follow only when the + /// legacy read reaches the leader's old tail; if the cap cut it short, the batch stops there + /// so it stays contiguous, and the follower catches up in later rounds. /// - /// For each peer behind the leader's latest index: - /// - `next_index < first_index` (purged prefix) → `NeedSnapshot` - /// - Fetched range is short or has a gap despite being in-range → `CorruptGap` (storage-layer - /// inconsistency, not a purge — caller must not silently fall back to snapshot) - /// - Otherwise → `Ready(entries)`, contiguous from `next_index` onward + /// Legacy is capped at `max_legacy_entries_per_peer` per follower; `0` means none. /// - /// `first_index` is the leader's retained-log boundary (`raft_log.first_entry_id()`), - /// passed in so the purge check happens before the range fetch, not after. + /// A follower whose in-flight window is full (`peer_gating_decisions[id] == true`) is + /// skipped and gets no entry in the result. The caller then sends it an empty + /// heartbeat, so it stays alive without receiving new data. The gating decision is + /// computed once by the caller, so the same answer also governs dispatch. /// - /// `peer_gating_decisions[peer_id] == true` means that peer's in-flight - /// window is full: skip it here, insert nothing into the result map. The - /// caller's dispatch phase sees no entries for this peer and sends an - /// empty-entries heartbeat instead — this is how the window-full peer - /// still gets a liveness heartbeat without new data. This map is computed - /// once by the top-level caller, not here, so the same gating decision - /// also governs dispatch — see `prepare_batch_requests` doc above. + /// `first_index` is the oldest index the leader still keeps. It is passed in so the + /// purge check runs before the log is read. fn retrieve_to_be_synced_logs_for_peers( &self, new_entries: &[Entry], diff --git a/d-engine-core/src/replication/replication_handler.rs b/d-engine-core/src/replication/replication_handler.rs index ff7a01ad..2d87ec59 100644 --- a/d-engine-core/src/replication/replication_handler.rs +++ b/d-engine-core/src/replication/replication_handler.rs @@ -139,6 +139,13 @@ where count, "CorruptGap: legacy entries range read inconsistent — skipping this peer this round" ); + metrics::counter!( + "core.raft.peer.skipped_total", + "peer_id" => peer_id.to_string(), + "reason" => "corrupt_gap" + ) + .increment(1); + skip_peers.insert(peer_id); } } @@ -167,6 +174,13 @@ where snapshot_targets.push(peer.id); } Err(e) => { + metrics::counter!( + "core.raft.peer.skipped_total", + "peer_id" => peer.id.to_string(), + "reason" => "build_request_failed" + ) + .increment(1); + // #436: isolate to this one peer, don't abort the whole batch. // Structured event reporting (upgrade from bare log) is a separate, // later change — this matches the existing pattern in @@ -269,6 +283,18 @@ where leader_last_index_before_inserting_new_entries ); + // The leader's log at this moment, in index order: + // + // [ first_index ...... old_last ][ new_first ...... new_last ] + // oldest entry still kept this round's `new_entries` + // .. newest stored entry + // + // Both parts are in the leader's log, but the new part is also at hand in + // `new_entries`, so it is copied from there instead of being read again. + let old_last = leader_last_index_before_inserting_new_entries; + let new_first = old_last + 1; + let new_last = old_last + new_entries.len() as u64; + for (&id, &peer_next_id) in peer_next_indices { if id == self.my_id { continue; @@ -288,59 +314,59 @@ where debug!("peer: {} next: {}", id, peer_next_id); - let mut entries = Vec::new(); - if leader_last_index_before_inserting_new_entries >= peer_next_id { - // Purge boundary check BEFORE fetching. This is the only legitimate - // reason legacy entries can be missing — anything past this point - // that still comes back short/gapped is corruption, not purge. - if peer_next_id < first_index { + let (start, end) = match plan_peer_interval( + peer_next_id, + old_last, + new_last, + first_index, + max_legacy_entries_per_peer, + ) { + PeerInterval::NeedSnapshot => { peer_entries.insert(id, PeerEntriesResult::NeedSnapshot); continue; } - - if max_legacy_entries_per_peer > 0 { - let until_index = if (leader_last_index_before_inserting_new_entries - - peer_next_id) - >= max_legacy_entries_per_peer - { - peer_next_id + max_legacy_entries_per_peer - 1 - } else { - leader_last_index_before_inserting_new_entries - }; - - let legacy_entries = - match raft_log.get_entries_range(peer_next_id..=until_index) { - Ok(entries) => entries, - Err(e) => { - error!("Failed to get legacy entries for peer {}: {:?}", id, e); - Vec::new() - } - }; - - if !entries_are_contiguous_in_range(&legacy_entries, peer_next_id, until_index) - { - peer_entries.insert(id, PeerEntriesResult::CorruptGap { - start: peer_next_id, - end: until_index, - first_seen: legacy_entries.first().map(|e| e.index), - last_seen: legacy_entries.last().map(|e| e.index), - count: legacy_entries.len(), - }); - continue; + PeerInterval::Nothing => continue, + PeerInterval::Send { start, end } => (start, end), + }; + + // Fill the interval. It has up to two parts, split at `old_last`: + // head = start ..= min(end, old_last) already stored -> read from the log + // tail = max(start, new_first) ..= end this round's -> copy from memory + let head_end = end.min(old_last); + let mut entries = Vec::with_capacity((end - start + 1) as usize); + + if start <= head_end { + let head = match raft_log.get_entries_range(start..=head_end) { + Ok(entries) => entries, + Err(e) => { + error!("Failed to get legacy entries for peer {}: {:?}", id, e); + Vec::new() } + }; - trace!("legacy_entries: {:?}", &legacy_entries); - - entries.extend(legacy_entries); + if !entries_are_contiguous_in_range(&head, start, head_end) { + peer_entries.insert(id, PeerEntriesResult::CorruptGap { + start, + end: head_end, + first_seen: head.first().map(|e| e.index), + last_seen: head.last().map(|e| e.index), + count: head.len(), + }); + continue; } - } - if !new_entries.is_empty() { - entries.extend_from_slice(new_entries); + trace!("legacy_entries: {:?}", &head); + + entries.extend(head); } - if !entries.is_empty() { - peer_entries.insert(id, PeerEntriesResult::Ready(entries)); + + if end >= new_first { + let from = (start.max(new_first) - new_first) as usize; + let to = (end - new_first) as usize; + entries.extend_from_slice(&new_entries[from..=to]); } + + peer_entries.insert(id, PeerEntriesResult::Ready(entries)); } peer_entries @@ -349,7 +375,7 @@ where /// As Follower only async fn handle_append_entries( &self, - request: AppendEntriesRequest, + mut request: AppendEntriesRequest, state_snapshot: &StateSnapshot, raft_log: &Arc>, ) -> Result { @@ -392,7 +418,7 @@ where .filter_out_conflicts_and_append( request.prev_log_index, request.prev_log_term, - request.entries.clone(), + std::mem::take(&mut request.entries), ) .await?; } @@ -580,11 +606,37 @@ where let first_index = raft_log.first_entry_id(); let last_index = raft_log.last_entry_id(); - // Ledger overshot the leader's own log (stale speculative advance from before - // a disconnect): clamp to the real tip. Nothing was purged, no snapshot needed — - // effective_next_index carries the correction straight back to the ledger, - // no need to wait on a follower reject round-trip. - let next_index = recorded_next_index.min(last_index + 1); + // The entries were chosen by `retrieve_to_be_synced_logs_for_peers`, which also + // decided where this peer's batch starts. Take them first: the batch is the single + // source of truth for the starting position. + let entries = entries_per_peer.remove(&peer_id).unwrap_or_default(); + + // `prev_log_index` is derived from the first entry below, so a hole inside the batch + // would reach the follower looking like a valid request. Comparing the span with the + // entry count keeps this check O(1). It is a cheap sanity check; contiguity itself is + // guaranteed by how the batch is built. + if let (Some(first), Some(last)) = (entries.first(), entries.last()) { + let span = last.index.checked_sub(first.index).map(|d| d + 1); + if span != Some(entries.len() as u64) { + return Err(ReplicationError::NonContiguousBatch { + peer_id, + first_index: first.index, + last_index: last.index, + len: entries.len(), + } + .into()); + } + } + + // Where this request starts: + // - with entries: at the first entry, so `prev_log_index` is the index right before it; + // - heartbeat (no entries): at the recorded next index, but never past the end of the + // leader's log. A stale optimistic advance may have pushed the record beyond it; the + // corrected value goes back to the ledger through `effective_next_index`. + let next_index = match entries.first() { + Some(first) => first.index, + None => recorded_next_index.min(last_index + 1), + }; let prev_index = next_index.saturating_sub(1); let prev_log_term = match raft_log.entry_term(prev_index) { @@ -604,9 +656,6 @@ where } }; - // Move entries out of the map — avoids Vec clone - let entries = entries_per_peer.remove(&peer_id).unwrap_or_default(); - debug!( "[Leader {} -> Follower {}] Replicating {} entries", self.my_id, @@ -680,3 +729,87 @@ pub fn client_command_to_entry_payloads(commands: Vec) -> Vec PeerInterval { + let start = peer_next_id.min(new_last + 1); + if start <= old_last && start < first_index { + return PeerInterval::NeedSnapshot; + } + let catchup_end = old_last.min(start.saturating_add(max_legacy_entries).saturating_sub(1)); + let end = if catchup_end == old_last { + new_last + } else { + catchup_end + }; + if start > end { + PeerInterval::Nothing + } else { + PeerInterval::Send { start, end } + } +} + +#[cfg(test)] +#[path = "replication_handler_test/basic_scenarios_test.rs"] +mod basic_scenarios_test; + +#[cfg(test)] +#[path = "replication_handler_test/command_handling_test.rs"] +mod command_handling_test; + +#[cfg(test)] +#[path = "replication_handler_test/edge_cases_test.rs"] +mod edge_cases_test; + +#[cfg(test)] +#[path = "replication_handler_test/heartbeat_match_index_test.rs"] +mod heartbeat_match_index_test; + +#[cfg(test)] +#[path = "replication_handler_test/helper_functions_test.rs"] +mod helper_functions_test; + +#[cfg(test)] +#[path = "replication_handler_test/index_tracking_test.rs"] +mod index_tracking_test; + +#[cfg(test)] +#[path = "replication_handler_test/learner_management_test.rs"] +mod learner_management_test; + +#[cfg(test)] +#[path = "replication_handler_test/log_retrieval_test.rs"] +mod log_retrieval_test; + +#[cfg(test)] +#[path = "replication_handler_test/quorum_calculation_test.rs"] +mod quorum_calculation_test; + +#[cfg(test)] +#[path = "replication_handler_test/snapshot_trigger_test.rs"] +mod snapshot_trigger_test; diff --git a/d-engine-core/src/replication/replication_handler_test/log_retrieval_test.rs b/d-engine-core/src/replication/replication_handler_test/log_retrieval_test.rs index d974a7a4..d6ae4ffc 100644 --- a/d-engine-core/src/replication/replication_handler_test/log_retrieval_test.rs +++ b/d-engine-core/src/replication/replication_handler_test/log_retrieval_test.rs @@ -13,6 +13,8 @@ use std::sync::Arc; use d_engine_proto::common::Entry; use d_engine_proto::common::EntryPayload; +use super::PeerInterval; +use super::plan_peer_interval; use crate::MockTypeConfig; use crate::ReplicationCore; use crate::ReplicationHandler; @@ -192,18 +194,19 @@ async fn test_retrieve_only_old_entries_when_no_new_entries() { ); } -/// Peer behind with max entries limit should receive limited old logs plus new entries. +/// A peer that lags beyond the limit receives only the capped old logs, without the new entry. /// /// # Scenario /// - Leader has 3 old log entries (index 1-3) /// - New entry to replicate at index 4 /// - Peer's next_index = 1 (needs all old logs) /// - Max entries limit = 2 (can only send 2 old entries) -/// - Expected: First 2 old logs (index 1-2) + new entry +/// - Expected: old logs 1-2 only; the peer catches up on 3 and 4 in later rounds /// /// # Test Purpose -/// Validates that max_entries limit is enforced for old logs, -/// preventing overwhelming peers with too much historical data. +/// Validates two things at once: the limit is enforced for old logs, and the cut-short batch +/// is not extended with the new entry. Sending 1, 2 and 4 would leave a hole at 3 that the +/// follower accepts and acknowledges. #[tokio::test] async fn test_retrieve_limited_old_entries_with_max_limit() { // Arrange: Setup test context with 3 old entries @@ -238,30 +241,30 @@ async fn test_retrieve_limited_old_entries_with_max_limit() { &HashMap::new(), ); - // Assert: Only first 2 old entries + new entry (limited by max) - let mut expected = vec![old_entries[0].clone(), old_entries[1].clone()]; - expected.extend(new_entries.clone()); + // Assert: only the first 2 old entries; the new entry at 4 must not follow them + let expected = vec![old_entries[0].clone(), old_entries[1].clone()]; assert_eq!( result.get(&peer3_id), Some(&crate::PeerEntriesResult::Ready(expected)), - "peer should receive limited old logs (max=2) plus new entries" + "peer should receive only the capped old logs (max=2), no new entries after a cut" ); } -/// Max entries limit of zero should return only new entries. +/// With a limit of zero, a peer that needs old logs gets nothing (the caller sends a heartbeat). /// /// # Scenario /// - Leader has 3 old log entries (index 1-3) /// - New entry to replicate at index 4 /// - Peer's next_index = 1 (needs old logs) /// - Max entries limit = 0 (no old logs allowed) -/// - Expected: Only new entry (no old logs) +/// - Expected: no entry for this peer /// /// # Test Purpose -/// Validates that max_entries=0 skips all historical logs, -/// useful for scenarios prioritizing new data over catch-up. +/// A batch of only the new entry 4 would start after the peer's next_index 1 and leave a +/// hole at 1-3. Configuration validation rejects 0, so this is a defensive check that a +/// disabled catch-up never produces a gapped batch. #[tokio::test] -async fn test_retrieve_only_new_entries_when_max_limit_zero() { +async fn test_retrieve_nothing_when_max_limit_zero_and_peer_behind() { // Arrange: Setup test context with 3 old entries let mut context = setup_mock_replication_test_context(1); let my_id = 1; @@ -294,11 +297,422 @@ async fn test_retrieve_only_new_entries_when_max_limit_zero() { &HashMap::new(), ); - // Assert: Only new entry (no old logs due to max=0) + // Assert: nothing for this peer, so dispatch falls back to a heartbeat assert_eq!( result.get(&peer3_id), - Some(&crate::PeerEntriesResult::Ready(new_entries.clone())), - "peer should receive only new entries when max_legacy=0" + None, + "peer behind with max_legacy=0 must not get a batch that skips its old logs" + ); +} + +/// A batch sent to a lagging peer must be contiguous from `next_index`. +/// +/// # Scenario +/// - Leader's last index before the new write is 5000; three new entries 5001..=5003 +/// - Peer 2 reconnected after an outage: next_index = 1000, so prev_log_index = 999 +/// - The per-RPC limit is 500 +/// - The follower appends a batch right after prev_log_index without any gap check, so a hole +/// inside the batch (for example 1499 followed by 5001) becomes a hole in its log +/// +/// # Test Purpose +/// The batch must start at `next_index` and every index must be the previous one plus one, +/// whatever the limit truncates. This asserts the property, not a particular payload shape. +#[tokio::test] +async fn test_retrieve_batch_stays_contiguous_when_peer_lags_beyond_limit() { + let mut context = setup_mock_replication_test_context(1); + let handler = ReplicationHandler::::new(1); + let peer_id = 2; + + let old_entries = mock_insert_log_entries((1000..=5000).collect(), 1, 1000); + let raft_log_mut = Arc::get_mut(&mut context.raft_log).unwrap(); + mock_log_entries_exist(raft_log_mut, old_entries); + + let new_entries = mock_insert_log_entries(vec![1, 2, 3], 1, 5001); + let peer_next_indices = HashMap::from([(peer_id, 1000_u64)]); + + let result = handler.retrieve_to_be_synced_logs_for_peers( + &new_entries, + 5000, // leader_last_index_before_inserting_new_entries + 500, // max_legacy_entries_per_peer + &peer_next_indices, + &context.raft_log, + 1, + &HashMap::new(), + ); + + let Some(crate::PeerEntriesResult::Ready(entries)) = result.get(&peer_id) else { + panic!( + "expected Ready entries for the lagging peer, got {:?}", + result.get(&peer_id) + ); + }; + assert_eq!( + entries.first().map(|e| e.index), + Some(1000), + "batch must start at the peer's next_index, which prev_log_index is derived from" + ); + for pair in entries.windows(2) { + assert_eq!( + pair[1].index, + pair[0].index + 1, + "hole in batch between {} and {}", + pair[0].index, + pair[1].index + ); + } +} + +/// The request built for a peer must carry `prev_log_index` right before its first entry. +/// +/// # Scenario +/// - Leader's log ended at 5000; this round appended 5001..=5003, so the log now ends at 5003 +/// - Peer 2's recorded next_index is 6000, beyond the leader's log: an earlier optimistic advance +/// ran past what the peer really has (for example after a reconnect) +/// - The peer's entries are chosen from the recorded next_index (6000), while prev_log_index is +/// derived from a next_index clamped to the log end (5004) +/// +/// # Test Purpose +/// Entries and prev_log_index must describe the same position, whatever batch is selected +/// (a heartbeat with prev=5003 is fine). Otherwise the follower is asked to match prev=5003 +/// and is then handed 5001..=5003, which starts before prev. +#[tokio::test] +async fn test_build_request_prev_index_precedes_first_entry_when_next_index_overshoots() { + let mut context = setup_mock_replication_test_context(1); + let handler = ReplicationHandler::::new(1); + let peer_id = 2; + + let new_entries = mock_insert_log_entries(vec![1, 2, 3], 1, 5001); + let raft_log_mut = Arc::get_mut(&mut context.raft_log).unwrap(); + mock_log_entries_exist(raft_log_mut, new_entries.clone()); + raft_log_mut.expect_first_entry_id().returning(|| 1); + + let peer_next_indices = HashMap::from([(peer_id, 6000_u64)]); + let mut result = handler.retrieve_to_be_synced_logs_for_peers( + &new_entries, + 5000, // leader_last_index_before_inserting_new_entries + 100, // max_legacy_entries_per_peer + &peer_next_indices, + &context.raft_log, + 1, + &HashMap::new(), + ); + // Whatever batch was selected (possibly none: the peer is past the log end), the request + // built from it must stay self-consistent. + let entries = match result.remove(&peer_id) { + Some(crate::PeerEntriesResult::Ready(entries)) => entries, + None => Vec::new(), + Some(other) => panic!("unexpected result for the overshooting peer: {other:?}"), + }; + + let data = crate::ReplicationData { + leader_last_index_before: 5000, + current_term: 1, + commit_index: 0, + peer_next_indices, + }; + let mut entries_per_peer = HashMap::from([(peer_id, entries)]); + let (_, outcome) = handler + .build_append_request(&context.raft_log, peer_id, &mut entries_per_peer, &data) + .expect("build_append_request should not fail"); + + let crate::BuildAppendOutcome::Append { request, .. } = outcome else { + panic!("expected an Append outcome"); + }; + if let Some(first) = request.entries.first() { + assert_eq!( + request.prev_log_index + 1, + first.index, + "prev_log_index {} does not precede the first entry {}", + request.prev_log_index, + first.index + ); + } +} + +/// A peer whose next index was purged needs a snapshot, whatever the catch-up limit is. +/// +/// # Scenario +/// - Leader retained window: first_index = 11, last = 15; New entry at index 16 +/// - Peer 2 next_index = 5, below the retained window (its entries were purged) +/// - The catch-up limit is 0, so no old entries can be sent in any case +/// +/// # Test Purpose +/// The purge check decides between "send entries" and "send a snapshot". If it only runs +/// when there is something to send, a peer that can never be caught up from the log is +/// silently left on heartbeats forever instead of being routed to a snapshot. +#[tokio::test] +async fn test_retrieve_need_snapshot_when_next_index_purged_even_if_limit_zero() { + let context = setup_mock_replication_test_context(1); + let handler = ReplicationHandler::::new(1); + let peer_id = 2; + + let new_entries = mock_insert_log_entries(vec![1], 1, 16); + let peer_next_indices = HashMap::from([(peer_id, 5_u64)]); + + let result = handler.retrieve_to_be_synced_logs_for_peers( + &new_entries, + 15, // leader_last_index_before_inserting_new_entries + 0, // max_legacy_entries_per_peer + &peer_next_indices, + &context.raft_log, + 11, // first_index + &HashMap::new(), + ); + + assert_eq!( + result.get(&peer_id), + Some(&crate::PeerEntriesResult::NeedSnapshot), + "a purged next_index must route the peer to a snapshot even when no catch-up is allowed" + ); +} + +/// Runs `retrieve_to_be_synced_logs_for_peers` for one peer against a leader log of 1..=20 +/// plus `new_entries`, and returns the indices of the selected batch (`None` = no batch). +/// +/// The mock log only answers range reads when `stored` is true. When it is false any read +/// panics, so a test using it also proves that no read from the log happened. +fn select_batch( + stored: bool, + new_entries: &[Entry], + cap: u64, + peer_next: u64, + first_index: u64, +) -> Option { + let mut context = setup_mock_replication_test_context(1); + if stored { + let log = mock_insert_log_entries((1..=20).collect(), 1, 1); + mock_log_entries_exist(Arc::get_mut(&mut context.raft_log).unwrap(), log); + } + let handler = ReplicationHandler::::new(1); + let mut result = handler.retrieve_to_be_synced_logs_for_peers( + new_entries, + 20, // leader_last_index_before_inserting_new_entries + cap, + &HashMap::from([(2, peer_next)]), + &context.raft_log, + first_index, + &HashMap::new(), + ); + result.remove(&2) +} + +fn indices(result: Option) -> Option> { + match result { + Some(crate::PeerEntriesResult::Ready(entries)) => { + Some(entries.iter().map(|e| e.index).collect()) + } + Some(other) => panic!("expected Ready or no batch, got {other:?}"), + None => None, + } +} + +fn new_21_to_23() -> Vec { + mock_insert_log_entries(vec![1, 2, 3], 1, 21) +} + +/// The catch-up reaches the old tail exactly, so this round's entries join the same batch. +/// +/// # Scenario +/// Log 1..=20, new 21..=23, cap 5, peer next 16: the catch-up covers 16..=20. +/// +/// # Test Purpose +/// Boundary of the rule "new entries ride along only when the catch-up reaches the old +/// tail". One index short (next 15) must not join; see the next test. +#[tokio::test] +async fn test_retrieve_catchup_reaching_old_tail_exactly_includes_new_entries() { + let got = indices(select_batch(true, &new_21_to_23(), 5, 16, 1)); + assert_eq!(got, Some((16..=23).collect())); +} + +/// The catch-up stops one index short of the old tail, so this round's entries must wait. +/// +/// # Scenario +/// Log 1..=20, new 21..=23, cap 5, peer next 15: the catch-up covers 15..=19. +/// +/// # Test Purpose +/// Joining 21..=23 here would skip index 20 and hand the follower a hole. +#[tokio::test] +async fn test_retrieve_catchup_one_short_of_old_tail_excludes_new_entries() { + let got = indices(select_batch(true, &new_21_to_23(), 5, 15, 1)); + assert_eq!(got, Some((15..=19).collect())); +} + +/// The cap bounds the catch-up only; a caught-up peer receives the whole new batch. +/// +/// # Scenario +/// Ten new entries 21..=30, cap 5, peer already at 21. The log must not be read. +/// +/// # Test Purpose +/// New entries are bounded by the batching limit, not by the catch-up cap. Cutting them +/// here would make every round leave a remainder behind under sustained load. +#[tokio::test] +async fn test_retrieve_caught_up_peer_gets_whole_new_batch_above_cap() { + let ten_new = mock_insert_log_entries((1..=10).collect(), 1, 21); + let got = indices(select_batch(false, &ten_new, 5, 21, 1)); + assert_eq!(got, Some((21..=30).collect())); +} + +/// A zero catch-up limit does not stop a caught-up peer from receiving new entries. +/// +/// # Test Purpose +/// The limit switches off catching up, not replication. Peer at 21, cap 0, new 21..=23. +#[tokio::test] +async fn test_retrieve_cap_zero_still_sends_new_entries_to_caught_up_peer() { + let got = indices(select_batch(false, &new_21_to_23(), 0, 21, 1)); + assert_eq!(got, Some(vec![21, 22, 23])); +} + +/// A peer whose next index falls inside this round's entries receives only the missing suffix. +/// +/// # Test Purpose +/// The slice must start at the peer's next index, not at the first new entry, otherwise the +/// batch would begin before `prev_log_index`. Peer next 22, new 21..=23 -> 22..=23. +#[tokio::test] +async fn test_retrieve_peer_inside_new_entries_gets_only_missing_suffix() { + let got = indices(select_batch(false, &new_21_to_23(), 100, 22, 1)); + assert_eq!(got, Some(vec![22, 23])); +} + +/// A peer at or past the end of the leader's log has nothing to receive. +/// +/// # Test Purpose +/// next 24 is exactly one past the log end (new 21..=23); next 50 is far past it (a stale +/// optimistic advance). Both must yield no batch so the caller sends a heartbeat. +#[tokio::test] +async fn test_retrieve_peer_at_or_beyond_log_end_gets_no_batch() { + assert_eq!( + indices(select_batch(false, &new_21_to_23(), 100, 24, 1)), + None + ); + assert_eq!( + indices(select_batch(false, &new_21_to_23(), 100, 50, 1)), + None + ); +} + +/// A caught-up peer with no new entries in this round has nothing to receive. +/// +/// # Test Purpose +/// An idle round (no client writes) must not produce an empty `Ready` batch. +#[tokio::test] +async fn test_retrieve_caught_up_peer_without_new_entries_gets_no_batch() { + assert_eq!(indices(select_batch(false, &[], 100, 21, 1)), None); +} + +/// A next index equal to the oldest kept entry is still readable; only below it needs a snapshot. +/// +/// # Test Purpose +/// Off-by-one guard on the purge boundary. first_index 11, next 11, cap 100 -> 11..=23. +#[tokio::test] +async fn test_retrieve_next_index_at_first_index_reads_log_instead_of_snapshot() { + let got = indices(select_batch(true, &new_21_to_23(), 100, 11, 11)); + assert_eq!(got, Some((11..=23).collect())); +} + +/// A failed log read is reported as storage corruption, never as a snapshot or an empty batch. +/// +/// # Test Purpose +/// The caller must not hide a broken log behind a snapshot, and must not send the follower +/// a batch that silently lacks the entries it asked for. +#[tokio::test] +async fn test_retrieve_log_read_error_is_reported_as_corrupt_gap() { + let mut context = setup_mock_replication_test_context(1); + Arc::get_mut(&mut context.raft_log) + .unwrap() + .expect_get_entries_range() + .returning(|_| Err(crate::StorageError::LogStorage("disk unreadable".into()).into())); + let handler = ReplicationHandler::::new(1); + + let result = handler.retrieve_to_be_synced_logs_for_peers( + &new_21_to_23(), + 20, + 100, + &HashMap::from([(2, 5_u64)]), + &context.raft_log, + 1, + &HashMap::new(), + ); + + assert_eq!( + result.get(&2), + Some(&crate::PeerEntriesResult::CorruptGap { + start: 5, + end: 20, + first_seen: None, + last_seen: None, + count: 0, + }) + ); +} + +/// Each follower gets its own interval; one peer's state never changes another's batch. +/// +/// # Scenario +/// Peer 2 lags (next 1, cap 5); peer 3 is caught up (next 21). +/// +/// # Test Purpose +/// Peer 2 is cut to 1..=5 without new entries, while peer 3 still receives 21..=23. +#[tokio::test] +async fn test_retrieve_peers_are_decided_independently() { + let mut context = setup_mock_replication_test_context(1); + let log = mock_insert_log_entries((1..=20).collect(), 1, 1); + mock_log_entries_exist(Arc::get_mut(&mut context.raft_log).unwrap(), log); + let handler = ReplicationHandler::::new(1); + + let mut result = handler.retrieve_to_be_synced_logs_for_peers( + &new_21_to_23(), + 20, + 5, + &HashMap::from([(2, 1_u64), (3, 21_u64)]), + &context.raft_log, + 1, + &HashMap::new(), + ); + + assert_eq!(indices(result.remove(&2)), Some((1..=5).collect())); + assert_eq!(indices(result.remove(&3)), Some(vec![21, 22, 23])); +} + +/// A batch with a hole inside must never be turned into a request. +/// +/// # Scenario +/// - Leader log 1..=20; peer 2 is recorded at next_index 10 +/// - The entries handed to `build_append_request` are 10, 11 and 13 (12 is missing) +/// +/// # Test Purpose +/// The follower checks only `prev_log_index` against its own log. It would accept this batch +/// and store the hole, then acknowledge up to 13. The leader is the last place that can still +/// refuse, so a non-contiguous batch must fail loudly instead of being sent. +#[tokio::test] +async fn test_build_request_rejects_batch_with_a_hole() { + let mut context = setup_mock_replication_test_context(1); + let handler = ReplicationHandler::::new(1); + let peer_id = 2; + + let log = mock_insert_log_entries((1..=20).collect(), 1, 1); + let raft_log_mut = Arc::get_mut(&mut context.raft_log).unwrap(); + mock_log_entries_exist(raft_log_mut, log.clone()); + raft_log_mut.expect_first_entry_id().returning(|| 1); + + let holed: Vec = [10_u64, 11, 13].iter().map(|&i| log[i as usize - 1].clone()).collect(); + let mut holed = holed; + holed[2].index = 13; // keep the payload of entry 13 but make the gap visible + + let data = crate::ReplicationData { + leader_last_index_before: 20, + current_term: 1, + commit_index: 0, + peer_next_indices: HashMap::from([(peer_id, 10_u64)]), + }; + let mut entries_per_peer = HashMap::from([(peer_id, holed)]); + + let result = + handler.build_append_request(&context.raft_log, peer_id, &mut entries_per_peer, &data); + + assert!( + result.is_err(), + "a batch with indices 10, 11, 13 must be refused, got {:?}", + result.map(|(_, outcome)| matches!(outcome, crate::BuildAppendOutcome::Append { .. })) ); } @@ -524,3 +938,168 @@ async fn test_gated_peer_gets_no_entries_while_ungated_peer_does() { "an ungated peer at the same position must still get its entries" ); } + +// --------------------------------------------------------------------------- +// plan_peer_interval: which indices one follower gets this round. +// +// Fixture: the leader's log is 1..=10 (`old_last`), this round adds 11..=13 (`new_last`), +// the oldest kept entry is 1 and the cap is 100, unless a test says otherwise. +// --------------------------------------------------------------------------- + +fn plan( + peer_next_id: u64, + first_index: u64, + cap: u64, +) -> PeerInterval { + plan_peer_interval(peer_next_id, 10, 13, first_index, cap) +} + +/// A follower that already has the whole old log gets only this round's entries. +#[test] +fn test_plan_interval_caught_up_peer_gets_only_new_entries() { + assert!(matches!(plan(11, 1, 100), PeerInterval::Send { + start: 11, + end: 13 + })); +} + +/// A follower behind but within the cap gets the old entries it lacks plus this round's, +/// as one run. +#[test] +fn test_plan_interval_lagging_within_cap_gets_old_and_new_entries() { + assert!(matches!(plan(8, 1, 100), PeerInterval::Send { + start: 8, + end: 13 + })); +} + +/// When the cap stops the catch-up short of the old tail, this round's entries are left out. +/// Adding them would put a hole between the catch-up and the new entries. +#[test] +fn test_plan_interval_cap_cuts_catchup_and_drops_new_entries() { + // 1..=10 lacking, cap 4: catch-up 1..=4 only. + assert!(matches!(plan(1, 1, 4), PeerInterval::Send { + start: 1, + end: 4 + })); +} + +/// Boundary: the cap lands exactly on the old tail, so this round's entries are included; +/// one fewer and they are not. +#[test] +fn test_plan_interval_cap_reaching_old_tail_exactly_includes_new_entries() { + // next=7, cap=4 -> 7..=10 reaches old_last. + assert!(matches!(plan(7, 1, 4), PeerInterval::Send { + start: 7, + end: 13 + })); + // next=6, cap=4 -> 6..=9 stops one short. + assert!(matches!(plan(6, 1, 4), PeerInterval::Send { + start: 6, + end: 9 + })); +} + +/// A follower whose next index falls inside this round's entries gets the rest of them. +#[test] +fn test_plan_interval_peer_inside_new_entries_gets_the_rest() { + assert!(matches!(plan(12, 1, 100), PeerInterval::Send { + start: 12, + end: 13 + })); +} + +/// A recorded next index past the log end is clamped, and then there is nothing to send. +#[test] +fn test_plan_interval_peer_past_log_end_gets_nothing() { + assert!(matches!(plan(50, 1, 100), PeerInterval::Nothing)); + // Next is exactly one past the log end. + assert!(matches!(plan(14, 1, 100), PeerInterval::Nothing)); +} + +/// No new entries this round and the follower is caught up: heartbeat only. +#[test] +fn test_plan_interval_no_new_entries_and_caught_up_gets_nothing() { + assert!(matches!( + plan_peer_interval(11, 10, 10, 1, 100), + PeerInterval::Nothing + )); +} + +/// Cap 0 allows no catch-up: a lagging follower gets nothing, a caught-up one still gets +/// this round's entries. +#[test] +fn test_plan_interval_cap_zero_blocks_catchup_but_not_new_entries() { + assert!(matches!(plan(5, 1, 0), PeerInterval::Nothing)); + assert!(matches!(plan(11, 1, 0), PeerInterval::Send { + start: 11, + end: 13 + })); +} + +/// A follower below the oldest kept entry can only be caught up by a snapshot, even when the +/// cap would allow no catch-up at all. +#[test] +fn test_plan_interval_purged_peer_needs_snapshot_regardless_of_cap() { + assert!(matches!(plan(3, 5, 100), PeerInterval::NeedSnapshot)); + assert!(matches!(plan(3, 5, 0), PeerInterval::NeedSnapshot)); +} + +/// Boundary: next index equal to the oldest kept entry is still servable from the log. +#[test] +fn test_plan_interval_peer_at_first_index_is_not_purged() { + assert!(matches!(plan(5, 5, 100), PeerInterval::Send { + start: 5, + end: 13 + })); +} + +/// A huge cap must not overflow when added to the start index. +#[test] +fn test_plan_interval_max_cap_does_not_overflow() { + assert!(matches!(plan(1, 1, u64::MAX), PeerInterval::Send { + start: 1, + end: 13 + })); +} + +/// Over every small combination, the chosen interval keeps the properties replication relies on: +/// it ends within the log, respects the cap on the catch-up part, never skips indices between +/// catch-up and new entries, and never starts below the oldest kept entry. +#[test] +fn test_plan_interval_properties_hold_for_all_small_inputs() { + for old_last in 0u64..=6 { + for new_count in 0u64..=3 { + let new_last = old_last + new_count; + for first_index in 1u64..=old_last + 1 { + for peer_next in 1u64..=new_last + 3 { + for cap in [0u64, 1, 2, 5, u64::MAX] { + let ctx = format!( + "old_last={old_last} new_last={new_last} first={first_index} \ + next={peer_next} cap={cap}" + ); + match plan_peer_interval(peer_next, old_last, new_last, first_index, cap) { + PeerInterval::NeedSnapshot => { + assert!(peer_next <= old_last && peer_next < first_index, "{ctx}"); + } + PeerInterval::Nothing => {} + PeerInterval::Send { start, end } => { + assert!(start <= end && end <= new_last, "{ctx}"); + assert_eq!(start, peer_next.min(new_last + 1), "{ctx}"); + if start <= old_last { + assert!(start >= first_index, "{ctx}"); + // Catch-up part is capped. + assert!(end.min(old_last) - start < cap.max(1), "{ctx}"); + // New entries only follow a catch-up that reached the tail. + if end > old_last { + assert!(start.saturating_add(cap) > old_last, "{ctx}"); + } + } + } + } + } + } + } + } + } +} diff --git a/d-engine-core/src/replication/replication_handler_test/mod.rs b/d-engine-core/src/replication/replication_handler_test/mod.rs deleted file mode 100644 index 18eb92ff..00000000 --- a/d-engine-core/src/replication/replication_handler_test/mod.rs +++ /dev/null @@ -1,46 +0,0 @@ -//! Replication Handler Test Module -//! -//! This module contains comprehensive unit tests for `ReplicationHandler` and its -//! interaction with leader state batch processing. Tests are organized by functional -//! domains for better maintainability. -//! -//! ## Test Organization -//! -//! - `basic_scenarios_test`: Cluster topology tests (single/multi-node) -//! - `quorum_calculation_test`: Quorum achievement and failure scenarios -//! - `command_handling_test`: Command processing and batching -//! - `learner_management_test`: Learner replication and progress tracking -//! - `index_tracking_test`: Match index updates and conflict handling -//! - `edge_cases_test`: Network partitions, timeouts, large batches -//! - `log_retrieval_test`: Log synchronization and retrieval logic -//! - `helper_functions_test`: Internal helper function unit tests - -#[cfg(test)] -mod basic_scenarios_test; - -#[cfg(test)] -mod command_handling_test; - -#[cfg(test)] -mod edge_cases_test; - -#[cfg(test)] -mod helper_functions_test; - -#[cfg(test)] -mod index_tracking_test; - -#[cfg(test)] -mod learner_management_test; - -#[cfg(test)] -mod log_retrieval_test; - -#[cfg(test)] -mod quorum_calculation_test; - -#[cfg(test)] -mod snapshot_trigger_test; - -#[cfg(test)] -mod heartbeat_match_index_test; diff --git a/d-engine-core/src/storage/raft_log_core.rs b/d-engine-core/src/storage/raft_log_core.rs index d606dc29..e6a27c6f 100644 --- a/d-engine-core/src/storage/raft_log_core.rs +++ b/d-engine-core/src/storage/raft_log_core.rs @@ -30,6 +30,7 @@ use crate::LogStore; use crate::MetaStore; use crate::NetworkError; use crate::RaftLog; +use crate::ReplicationError; use crate::Result; use crate::StorageEngine; use crate::TypeConfig; @@ -303,6 +304,8 @@ where ) -> Result> { let _timer = ScopedTimer::new("filter_out_conflicts_and_append"); + Self::ensure_contiguous_from_prev(prev_log_index, &new_entries)?; + // prev_log_index==0 has no real entry to compare against, not a reset signal let is_virtual_log_start = prev_log_index == 0 && prev_log_term == 0; if !is_virtual_log_start { @@ -1048,6 +1051,34 @@ where self.entries.read().is_empty() } + /// Rejects a batch that is not consecutive indices starting at `prev_log_index + 1`. + /// Checked before any state is touched, so the log never gets a hole. + /// O(1): looks only at the first and last entry. + fn ensure_contiguous_from_prev( + prev_log_index: u64, + entries: &[Entry], + ) -> Result<()> { + let (Some(first), Some(last)) = (entries.first(), entries.last()) else { + return Ok(()); + }; + let contiguous = prev_log_index.checked_add(1) == Some(first.index) + && last.index.checked_sub(first.index).map(|d| d + 1) == Some(entries.len() as u64); + if contiguous { + Ok(()) + } else { + metrics::counter!("core.raft.follower.append_rejected_total", "reason" => "non_contiguous") + .increment(1); + + Err(ReplicationError::NonContiguousAppend { + prev_log_index, + first_index: first.index, + last_index: last.index, + len: entries.len(), + } + .into()) + } + } + /// Returns reference to next_id atomic for test verification (test-only). #[cfg(test)] pub fn next_id(&self) -> &std::sync::atomic::AtomicU64 { diff --git a/d-engine-core/src/storage/raft_log_core_test/basic_operations_test.rs b/d-engine-core/src/storage/raft_log_core_test/basic_operations_test.rs index ced99bf8..4977f942 100644 --- a/d-engine-core/src/storage/raft_log_core_test/basic_operations_test.rs +++ b/d-engine-core/src/storage/raft_log_core_test/basic_operations_test.rs @@ -152,9 +152,10 @@ async fn test_filter_conflicts_handles_multiple_scenarios() { term: 2, payload: None, }]; + // prev is entry 3 (term 3 after Case 1): the batch starts right after it. let result = ctx .raft_log - .filter_out_conflicts_and_append(1, 1, new_entries) + .filter_out_conflicts_and_append(3, 3, new_entries) .await .unwrap() .unwrap(); diff --git a/d-engine-core/src/storage/raft_log_core_test/pipeline_overlap_test.rs b/d-engine-core/src/storage/raft_log_core_test/pipeline_overlap_test.rs index fd9d2b61..14d1b52c 100644 --- a/d-engine-core/src/storage/raft_log_core_test/pipeline_overlap_test.rs +++ b/d-engine-core/src/storage/raft_log_core_test/pipeline_overlap_test.rs @@ -493,3 +493,187 @@ async fn test_replace_range_and_submit_delegates_to_replace_range_not_truncate() "truncate() must not be called separately — replace_range() owns the full operation" ); } + +// --------------------------------------------------------------------------- +// Follower must never store a hole. +// +// `filter_out_conflicts_and_append` verifies only `prev_log_index`/`prev_log_term`. It does not +// check that the first incoming entry sits at `prev_log_index + 1`, nor that the batch has no +// hole inside. A leader that builds such a batch (a bug, or an older leader during a rolling +// upgrade) makes the follower store the entries and acknowledge the highest index in the batch, +// which the leader then counts towards the majority. +// +// The assertions state the invariant, not an error shape: after the call the log is contiguous +// and the follower does not claim an index it does not hold. They stay valid whichever way the +// fix rejects the batch. +// --------------------------------------------------------------------------- + +/// Follower holding `1..=last` in `term`. +async fn follower_with_log( + name: &str, + last: u64, + term: u64, +) -> RaftLogCoreTestContext { + let ctx = RaftLogCoreTestContext::new(name); + ctx.append_entries(1, last, term).await; + assert_eq!(ctx.raft_log.last_entry_id(), last, "precondition"); + ctx +} + +/// Every index between the first and last stored entry must be readable. +fn assert_log_has_no_hole(ctx: &RaftLogCoreTestContext) { + let first = ctx.raft_log.first_entry_id(); + let last = ctx.raft_log.last_entry_id(); + for i in first..=last { + assert!( + ctx.raft_log.entry(i).unwrap().is_some(), + "hole at index {i} in a log that spans {first}..={last}" + ); + } +} + +/// The index returned to the leader as "matched" must be one the follower really holds. +fn assert_ack_is_backed_by_log( + ctx: &RaftLogCoreTestContext, + result: crate::Result>, +) { + if let Ok(Some(id)) = result { + for i in 1..=id.index { + assert!( + ctx.raft_log.entry(i).unwrap().is_some(), + "follower acknowledged index {} but does not hold index {i}", + id.index + ); + } + } +} + +/// Follower has 1..=10. The leader's batch claims `prev = 10` but starts at 20. +/// Business view: the follower must not store 20..=22 and ack 22 while 11..=19 never arrived. +#[tokio::test] +async fn test_filter_conflicts_first_entry_past_prev_plus_one_leaves_no_hole() { + let ctx = follower_with_log("batch_first_entry_past_prev", 10, 1).await; + + let result = ctx + .raft_log + .filter_out_conflicts_and_append(10, 1, vec![entry(20, 1), entry(21, 1), entry(22, 1)]) + .await; + + assert_log_has_no_hole(&ctx); + assert_ack_is_backed_by_log(&ctx, result); +} + +/// Same as above, but the hole is inside the batch: 11, 12, then 14, 15. +/// This is the shape the leader produced when a capped catch-up run was joined to the new +/// entries of the round. +#[tokio::test] +async fn test_filter_conflicts_hole_inside_batch_leaves_no_hole() { + let ctx = follower_with_log("batch_hole_inside", 10, 1).await; + + let result = ctx + .raft_log + .filter_out_conflicts_and_append(10, 1, vec![ + entry(11, 1), + entry(12, 1), + entry(14, 1), + entry(15, 1), + ]) + .await; + + assert_log_has_no_hole(&ctx); + assert_ack_is_backed_by_log(&ctx, result); +} + +/// Slow path: the incoming batch overlaps the follower's tail with a different term, so the +/// follower truncates and replaces. A hole in the replacement tail must still not be stored. +#[tokio::test] +async fn test_filter_conflicts_hole_after_term_conflict_leaves_no_hole() { + let ctx = RaftLogCoreTestContext::new("batch_hole_after_conflict"); + ctx.append_entries(1, 8, 1).await; + ctx.append_entries(9, 2, 2).await; // 9..=10 in term 2 + assert_eq!(ctx.raft_log.last_entry_id(), 10, "precondition"); + + // New leader (term 3) overwrites 9..=10 and then skips 11. + let result = ctx + .raft_log + .filter_out_conflicts_and_append(8, 1, vec![entry(9, 3), entry(10, 3), entry(12, 3)]) + .await; + + assert_log_has_no_hole(&ctx); + assert_ack_is_backed_by_log(&ctx, result); +} + +/// Control: a correct contiguous batch is still accepted. Guards against a fix that rejects +/// everything. +#[tokio::test] +async fn test_filter_conflicts_contiguous_batch_is_accepted() { + let ctx = follower_with_log("batch_contiguous_control", 10, 1).await; + + let result = ctx + .raft_log + .filter_out_conflicts_and_append(10, 1, vec![entry(11, 1), entry(12, 1), entry(13, 1)]) + .await + .unwrap(); + + assert_eq!(result.unwrap().index, 13); + assert_eq!(ctx.raft_log.last_entry_id(), 13); + assert_log_has_no_hole(&ctx); +} + +/// The first entry sits at or below `prev_log_index`: the batch overlaps the prefix the +/// leader just vouched for, which is not the shape of any valid request. +#[tokio::test] +async fn test_filter_conflicts_first_entry_not_after_prev_is_rejected() { + let ctx = follower_with_log("batch_first_entry_not_after_prev", 10, 1).await; + + let result = ctx + .raft_log + .filter_out_conflicts_and_append(10, 1, vec![entry(10, 1), entry(11, 1)]) + .await; + + assert!( + result.is_err(), + "a batch starting at prev_log_index must be rejected" + ); + assert_eq!( + ctx.raft_log.last_entry_id(), + 10, + "the log must be untouched" + ); +} + +/// Indices that go backwards must not pass the span check by wrapping around. +#[tokio::test] +async fn test_filter_conflicts_descending_batch_is_rejected() { + let ctx = follower_with_log("batch_descending", 10, 1).await; + + let result = ctx + .raft_log + .filter_out_conflicts_and_append(10, 1, vec![entry(11, 1), entry(10, 1)]) + .await; + + assert!(result.is_err(), "a descending batch must be rejected"); + assert_eq!( + ctx.raft_log.last_entry_id(), + 10, + "the log must be untouched" + ); +} + +/// `prev_log_index + 1` would overflow: the request must be rejected, not panic. +#[tokio::test] +async fn test_filter_conflicts_prev_index_at_u64_max_is_rejected_without_overflow() { + let ctx = follower_with_log("batch_prev_u64_max", 10, 1).await; + + let result = ctx + .raft_log + .filter_out_conflicts_and_append(u64::MAX, 1, vec![entry(1, 1)]) + .await; + + assert!(result.is_err()); + assert_eq!( + ctx.raft_log.last_entry_id(), + 10, + "the log must be untouched" + ); +} diff --git a/d-engine-core/src/storage/storage_engine.rs b/d-engine-core/src/storage/storage_engine.rs index 5feedb5c..398b8833 100644 --- a/d-engine-core/src/storage/storage_engine.rs +++ b/d-engine-core/src/storage/storage_engine.rs @@ -77,6 +77,13 @@ pub trait LogStore: Send + Sync + 'static { /// Atomically truncate from `from_index` and persist `new_entries` in one write. /// + /// # Contract + /// The caller guarantees `new_entries[i].index == from_index + i`: contiguous, starting at + /// `from_index`. Implementations do not re-check it; the Raft log rejects a malformed + /// AppendEntries batch before it reaches storage. + /// Where `from_index` sits relative to the existing log (below the purge boundary, + /// beyond `last_index + 1`) is the caller's responsibility; behavior there is unspecified. + /// /// Default implementation calls `truncate` then `persist_entries` sequentially /// (non-atomic). Override with a single WriteBatch for true crash atomicity. /// Returns the highest log index on disk after the operation: the last of diff --git a/d-engine-core/src/storage/storage_engine_test.rs b/d-engine-core/src/storage/storage_engine_test.rs index f669b121..42f4e3ab 100644 --- a/d-engine-core/src/storage/storage_engine_test.rs +++ b/d-engine-core/src/storage/storage_engine_test.rs @@ -44,6 +44,8 @@ impl StorageEngineTestSuite { Self::test_batch_persistence(builder.build().await?).await?; Self::test_purge_logs(builder.build().await?).await?; Self::test_truncation(builder.build().await?).await?; + Self::test_replace_range(builder.build().await?).await?; + Self::test_replace_range_empty_entries_truncates(builder.build().await?).await?; Self::test_reset_operation(builder.build().await?).await?; Self::test_edge_cases(builder.build().await?).await?; Self::test_purge_empty_range(builder.build().await?).await?; @@ -172,6 +174,65 @@ impl StorageEngineTestSuite { Ok(()) } + /// `replace_range` drops the old tail from `from_index` and stores the new entries. + /// + /// Raft requirement: on a term conflict a follower replaces its divergent suffix in one + /// step; nothing of the old suffix may survive past the new entries. + pub async fn test_replace_range(engine: Arc) -> Result<(), Error> + where + E: StorageEngine + Send + Sync + 'static, + E::LogStore: Send + Sync, + E::MetaStore: Send + Sync, + { + let log_store = engine.log_store(); + log_store.reset().await?; + log_store.persist_entries(create_test_entries(1..=10)).await?; + + let mut replacement = create_test_entries(6..=7); + for e in &mut replacement { + e.term = 99; + } + let last = log_store.replace_range(6, replacement.clone()).await?; + + assert_eq!(last, 7, "replace_range must return the last replaced index"); + assert_eq!(log_store.last_index(), 7); + assert_eq!(log_store.entry(6).await?.map(|e| e.term), Some(99)); + assert_eq!(log_store.entry(7).await?.map(|e| e.term), Some(99)); + assert!(log_store.entry(8).await?.is_none(), "old tail must be gone"); + assert!( + log_store.entry(10).await?.is_none(), + "old tail must be gone" + ); + assert_eq!( + log_store.entry(5).await?.map(|e| e.term), + Some(5), + "prefix is untouched" + ); + Ok(()) + } + + /// `replace_range` with no new entries is a plain truncate and returns `from_index - 1`. + /// + /// This is the documented behavior of the trait, so every engine must agree on it. + pub async fn test_replace_range_empty_entries_truncates(engine: Arc) -> Result<(), Error> + where + E: StorageEngine + Send + Sync + 'static, + E::LogStore: Send + Sync, + E::MetaStore: Send + Sync, + { + let log_store = engine.log_store(); + log_store.reset().await?; + log_store.persist_entries(create_test_entries(1..=10)).await?; + + let last = log_store.replace_range(6, Vec::new()).await?; + + assert_eq!(last, 5); + assert_eq!(log_store.last_index(), 5); + assert!(log_store.entry(6).await?.is_none()); + assert!(log_store.entry(5).await?.is_some()); + Ok(()) + } + /// Test storage reset functionality pub async fn test_reset_operation(engine: Arc) -> Result<(), Error> where diff --git a/d-engine-server/tests/embedded_client/embedded_client_operations.rs b/d-engine-server/tests/embedded_client/embedded_client_operations.rs index 191e79af..7e74ac02 100644 --- a/d-engine-server/tests/embedded_client/embedded_client_operations.rs +++ b/d-engine-server/tests/embedded_client/embedded_client_operations.rs @@ -56,6 +56,11 @@ async fn create_test_engine(test_name: &str) -> (DefaultEmbeddedEngine, TempDir) [cluster] listen_address = "127.0.0.1:{}" single_node = true + +[raft] +# The default (50ms) is the client write deadline; a write waits for fdatasync, which +# on a loaded 2-vCPU CI runner can exceed it. These tests check CRUD results, not latency. +general_raft_timeout_duration_in_ms = 3000 "#, port ); diff --git a/d-engine-server/tests/storage_raft_log/performance_test.rs b/d-engine-server/tests/storage_raft_log/performance_test.rs index d770beff..b16b08e5 100644 --- a/d-engine-server/tests/storage_raft_log/performance_test.rs +++ b/d-engine-server/tests/storage_raft_log/performance_test.rs @@ -62,7 +62,7 @@ mod filter_out_conflicts_and_append_performance_tests { 0, // prev_log_index 0, // prev_log_term vec![Entry { - index: 501, + index: 1, term: 1, payload: Some(EntryPayload::command(Bytes::from(vec![1; 256]))), }], @@ -79,7 +79,7 @@ mod filter_out_conflicts_and_append_performance_tests { "Duration {duration}ms exceeds max {max_duration_ms}ms for {idle_flush_interval_ms}ms interval" ); - // Verify correctness: index=501 term=1 already exists (populated above), so this + // Verify correctness: index=1 term=1 already exists (populated above), so this // prev_log_index=0 resend is a pure duplicate — must be a no-op, not a reset. assert_eq!( log.last_entry_id(), diff --git a/d-engine/src/docs/examples/three-nodes-standalone.md b/d-engine/src/docs/examples/three-nodes-standalone.md index 1f29881a..f1921f2b 100644 --- a/d-engine/src/docs/examples/three-nodes-standalone.md +++ b/d-engine/src/docs/examples/three-nodes-standalone.md @@ -219,7 +219,7 @@ LOG_LEVEL=info make start-node1 **Cluster won't start:** -- Verify `target/release/demo` exists (run `make build`) +- Verify `target/release/three-nodes-standalone` exists (run `make build`) - Check port availability: `lsof -i :9081` (9082, 9083) - Ensure no stale processes: `pkill -f demo` diff --git a/d-engine/src/docs/quick-start-standalone.md b/d-engine/src/docs/quick-start-standalone.md index cf4b5770..c2284757 100644 --- a/d-engine/src/docs/quick-start-standalone.md +++ b/d-engine/src/docs/quick-start-standalone.md @@ -36,7 +36,7 @@ make start-cluster ```bash # Check processes -ps aux | grep "target/release/demo" # Should see 3 processes +ps aux | grep "target/release/three-nodes-standalone" # Should see 3 processes # Check logs for leader election tail -f logs/1/demo.log # Look for ">>> switch to Leader" diff --git a/examples/README.md b/examples/README.md index 5a0df578..13c73eac 100644 --- a/examples/README.md +++ b/examples/README.md @@ -59,9 +59,10 @@ Usage examples demonstrating different d-engine integration modes. ## Storage Backend Examples -| Example | Description | -| ----------------------------- | --------------------------------------- | -| [sled-cluster](sled-cluster/) | Alternative with Sled embedded database | +| Example | Description | +| ----------------------------------- | ----------------------------------------------------------------------------------- | +| [sled-cluster](sled-cluster/) | Alternative with Sled embedded database | +| [raft-engine-log](raft-engine-log/) | 3-node embedded cluster with the Raft log in raft-engine instead of the RocksDB WAL (experimental) | --- diff --git a/examples/raft-engine-log/.gitignore b/examples/raft-engine-log/.gitignore new file mode 100644 index 00000000..49149f1b --- /dev/null +++ b/examples/raft-engine-log/.gitignore @@ -0,0 +1,3 @@ +/target +/data +/db diff --git a/examples/raft-engine-log/Cargo.lock b/examples/raft-engine-log/Cargo.lock new file mode 100644 index 00000000..5e72c6b6 --- /dev/null +++ b/examples/raft-engine-log/Cargo.lock @@ -0,0 +1,2898 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arc-swap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "astral-tokio-tar" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" +dependencies = [ + "futures-core", + "libc", + "portable-atomic", + "rustc-hash", + "rustix", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "async-compression" +version = "0.4.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core 0.5.6", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +dependencies = [ + "serde", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "cc" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "compression-codecs" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + +[[package]] +name = "config" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68578f196d2a33ff61b27fae256c3164f65e36382648e30666dde05b8cc9dfdf" +dependencies = [ + "nom", + "pathdiff", + "serde", + "toml", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e71406cd8807725f7ac2f999a4cdd32e98f829fdf65f528343cebf945e41df1e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "d-engine" +version = "0.2.4" +dependencies = [ + "d-engine-server", +] + +[[package]] +name = "d-engine-core" +version = "0.2.4" +dependencies = [ + "astral-tokio-tar", + "async-compression", + "async-stream", + "async-trait", + "bincode", + "bytes", + "config", + "crc32fast", + "crossbeam-skiplist", + "d-engine-proto", + "dashmap", + "futures", + "http-body", + "http-body-util", + "lru", + "memmap2", + "metrics", + "mockall", + "parking_lot", + "prost", + "rand 0.9.5", + "serde", + "sha2", + "tempfile", + "thiserror", + "tokio", + "tokio-stream", + "tonic", + "tonic-health", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "d-engine-proto" +version = "0.2.4" +dependencies = [ + "bytes", + "prost", + "serde", + "tonic", + "tonic-build", + "tracing", + "vergen", +] + +[[package]] +name = "d-engine-server" +version = "0.2.4" +dependencies = [ + "arc-swap", + "async-trait", + "bincode", + "bytes", + "config", + "crc32fast", + "crossbeam", + "crossbeam-skiplist", + "d-engine-core", + "d-engine-proto", + "dashmap", + "futures", + "http-body", + "http-body-util", + "metrics", + "parking_lot", + "prost", + "rcgen", + "rust-rocksdb", + "serde", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tonic-health", + "tracing", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fail" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5e43d0f78a42ad591453aedb1d7ae631ce7ee445c7643691055a9ed8d3b01c" +dependencies = [ + "log", + "once_cell", + "rand 0.8.8", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fragile" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8878864ba14bb86e818a412bfd6f18f9eabd4ec0f008a28e8f7eb61db532fcf9" +dependencies = [ + "futures-core", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "libc", + "pin-project-lite", + "socket2 0.6.5", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "if_chain" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd62e6b5e86ea8eeeb8db1de02880a6abc01a397b2ebb64b5d74ac255318f5cb" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "libz-sys" +version = "1.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f710a23e6dbf193214fd46ca56a9d6864e550abe86202184532ae7275e46de19" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mockall" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43766c2b5203b10de348ffe19f7e54564b64f3d6018ff7648d1e2d6d3a0f0a48" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cbce79ec385a1d4f54baa90a76401eb15d9cab93685f62e7e9f942aa00ae2" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", + "pin-utils", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_threads" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cabcc654086861f9e1bc5115258de62ef92c79b20f4615a1b0cc57c4e82027d4" +dependencies = [ + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bee6c73da26345c729282832b60b0363cf3dd9f4bfd81d8551b7a1c889a113" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "predicates-core", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror", +] + +[[package]] +name = "prometheus-static-metric" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8f30cdb09c39930b8fa5e0f23cbb895ab3f766b187403a0ba0956fc1ef4f0e5" +dependencies = [ + "lazy_static", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +dependencies = [ + "heck", + "itertools 0.14.0", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "raft-engine" +version = "0.4.2" +source = "git+https://github.com/JoshuaChi/raft-engine.git?rev=e5ef9956540552cccc4b2a1bde1d2c5a1fb2ba9c#e5ef9956540552cccc4b2a1bde1d2c5a1fb2ba9c" +dependencies = [ + "byteorder", + "crc32fast", + "crossbeam", + "fail", + "fs2", + "hashbrown 0.14.5", + "hex", + "if_chain", + "lazy_static", + "libc", + "log", + "lz4-sys", + "nix", + "num-derive", + "num-traits", + "parking_lot", + "prometheus", + "prometheus-static-metric", + "protobuf", + "rayon", + "scopeguard", + "serde", + "serde_repr", + "strum", + "thiserror", +] + +[[package]] +name = "raft-engine-log" +version = "0.1.0" +dependencies = [ + "async-trait", + "axum 0.7.9", + "bincode", + "clap", + "d-engine", + "prometheus", + "prost", + "protobuf", + "raft-engine", + "serde", + "serde_json", + "tempfile", + "tokio", + "toml", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rapidhash" +version = "4.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da7e78a036ce858e8d55b7e7dc8ba3a88b78350fd2155d3591bbd966b58589e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rust-librocksdb-sys" +version = "0.42.0+10.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6522f8dfc02acd618477cd9aa94234f6a823ce578bba66215f15df8cb09a97d5" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", + "zstd-sys", +] + +[[package]] +name = "rust-rocksdb" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5573cd9e65281eb049123284fb8e161192f34ff76a7d635945d5fedbb7a59db7" +dependencies = [ + "libc", + "parking_lot", + "rust-librocksdb-sys", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tokio" +version = "1.53.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.5", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "axum 0.8.9", + "base64", + "bytes", + "flate2", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2 0.5.10", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tonic-health" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb87334d340313fefa513b6e60794d44a86d5f039b523229c99c323e4e19ca4b" +dependencies = [ + "prost", + "tokio", + "tokio-stream", + "tonic", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vergen" +version = "8.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2990d9ea5967266ea0ccf413a4aa5c42a93dbcfda9cb49a97de6931726b12566" +dependencies = [ + "anyhow", + "cfg-if", + "rustversion", + "time", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/examples/raft-engine-log/Cargo.toml b/examples/raft-engine-log/Cargo.toml new file mode 100644 index 00000000..4fd61d8a --- /dev/null +++ b/examples/raft-engine-log/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "raft-engine-log" +version = "0.1.0" +edition = "2024" + +[dependencies] +d-engine = { path = "../../d-engine" } +# raft-engine's log entries are rust-protobuf messages (`protobuf::Message`), not prost. +# default-features = false drops the `scripting` feature (rhai -> smartstring), which +# fails the license/advisory policy and is unused here. +# +# Pinned to a commit of a fork of the upstream crate, for two reasons: +# - the published 0.4.2 has a no-op `Engine::sync()` (fixed upstream in an unreleased commit); +# - upstream master pins `lz4-sys = "=1.9.5"`, which cannot coexist with the RocksDB build +# (`lz4-sys ^1.11`; both link the native lz4 library). The fork relaxes that pin to "1.9". +# Go back to the crates.io release once it contains the sync fix and no longer pins lz4-sys. +raft-engine = { git = "https://github.com/JoshuaChi/raft-engine.git", rev = "e5ef9956540552cccc4b2a1bde1d2c5a1fb2ba9c", default-features = false } + +# Needed to implement `protobuf::Message` for `EntryRecord`. +protobuf = "2" +# d-engine's `Entry`/`LogId` are prost; needed to encode/decode them. +prost = "0.13" +async-trait = "0.1" +bincode = "1.3" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "sync", "signal"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +clap = { version = "4", features = ["derive"] } +axum = "0.7" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +toml = "0.8" + +[dev-dependencies] +# `__test_support` exposes d_engine::storage_engine_test, the shared contract suite. +d-engine = { path = "../../d-engine", features = ["__test_support"] } +tempfile = "3" +# Same major as the one raft-engine registers its metrics in, so both share one registry. +prometheus = "0.13" diff --git a/examples/raft-engine-log/Makefile b/examples/raft-engine-log/Makefile new file mode 100644 index 00000000..4ec50be2 --- /dev/null +++ b/examples/raft-engine-log/Makefile @@ -0,0 +1,126 @@ +# Makefile for DEngine cluster operations + +.PHONY: build start-cluster clean help \ + start-node1 start-node2 start-node3 +.DEFAULT_GOAL := help + + +# ============================================================================ +# CONFIGURATION SECTION +# ============================================================================ + +# Cargo executable +CARGO ?= cargo + +# On macOS with Homebrew: auto-detect compression lib paths to skip bundled C++ +# compilation of RocksDB dependencies, which fails under macOS 26 + Xcode 26 +# (Clang 16 lacks __builtin_ctzg/__builtin_clzg from LLVM 18+ SDK headers). +# brew --prefix resolves correctly on both Apple Silicon (/opt/homebrew) and +# Intel Mac (/usr/local). Silently no-ops when brew or a lib is absent. +SNAPPY_PREFIX := $(shell brew --prefix snappy 2>/dev/null) +LZ4_PREFIX := $(shell brew --prefix lz4 2>/dev/null) +ZSTD_PREFIX := $(shell brew --prefix zstd 2>/dev/null) +BREW_ROCKSDB_ENV := + +ifneq ($(SNAPPY_PREFIX),) +ifneq ($(wildcard $(SNAPPY_PREFIX)/lib),) + BREW_ROCKSDB_ENV += SNAPPY_LIB_DIR=$(SNAPPY_PREFIX)/lib +endif +endif +ifneq ($(LZ4_PREFIX),) + BREW_ROCKSDB_ENV += LZ4_LIB_DIR=$(LZ4_PREFIX)/lib +endif +ifneq ($(ZSTD_PREFIX),) + BREW_ROCKSDB_ENV += ZSTD_LIB_DIR=$(ZSTD_PREFIX)/lib +endif +# Applied directly in recipes below (not folded into CARGO): a command-line +# `make CARGO=... build` override would otherwise silently skip these vars. + +# =============================== +# Global Variables +# =============================== +LOG_LEVEL ?= warn +BIN := ./target/release/raft-engine-log +# =============================== +# Build Targets +# =============================== +build: + @echo "Building release binary..." + $(BREW_ROCKSDB_ENV) $(CARGO) build --release --jobs 4 + +# =============================== +# Cluster Management (Normal Mode) +# =============================== +start-node1: + @echo "🚀 Starting Node 1..." + @trap 'echo "[Node1] Received shutdown signal"; kill -INT $$PID 2>/dev/null; wait $$PID' INT TERM; \ + RUST_LOG=raft-engine-log=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_BACKTRACE=1 \ + $(BIN) \ + --port 8081 \ + --health-port 10001 \ + --config-path "config/n1.toml" \ + --data-dir "db/1" \ + 2>&1 | sed 's/^/[Node1] /' & \ + PID=$$!; \ + wait $$PID; \ + EXIT_CODE=$$?; \ + exit $$EXIT_CODE + +start-node2: + @echo "🚀 Starting Node 2..." + @trap 'echo "[Node2] Received shutdown signal"; kill -INT $$PID 2>/dev/null; wait $$PID' INT TERM; \ + RUST_LOG=raft-engine-log=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_BACKTRACE=1 \ + $(BIN) \ + --port 8082 \ + --health-port 10002 \ + --config-path "config/n2.toml" \ + --data-dir "db/2" \ + 2>&1 | sed 's/^/[Node2] /' & \ + PID=$$!; \ + wait $$PID; \ + EXIT_CODE=$$?; \ + exit $$EXIT_CODE + +start-node3: + @echo "🚀 Starting Node 3..." + @trap 'echo "[Node3] Received shutdown signal"; kill -INT $$PID 2>/dev/null; wait $$PID' INT TERM; \ + RUST_LOG=raft-engine-log=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_BACKTRACE=1 \ + $(BIN) \ + --port 8083 \ + --health-port 10003 \ + --config-path "config/n3.toml" \ + --data-dir "db/3" \ + 2>&1 | sed 's/^/[Node3] /' & \ + PID=$$!; \ + wait $$PID; \ + EXIT_CODE=$$?; \ + exit $$EXIT_CODE + + +start-cluster: build + @echo "Starting 3-node cluster in parallel..." + $(MAKE) -j3 start-node1 start-node2 start-node3 + + +# =============================== +# Utilities +# =============================== +clean: + @echo "Cleaning build artifacts..." + $(BREW_ROCKSDB_ENV) $(CARGO) clean + rm -rf logs/* + rm -rf db/* + rm -rf snapshots/* + rm -rf samply_reports/* + + +help: + @echo "DEngine Cluster Management Commands:" + @echo " build - Build release binary" + @echo " start-node1..3 - Start individual cluster nodes" + @echo " start-cluster - Start all 3 nodes in parallel" + @echo " clean - Remove build artifacts, logs, and profiles" + @echo " help - Show this help message" diff --git a/examples/raft-engine-log/README.md b/examples/raft-engine-log/README.md new file mode 100644 index 00000000..a27fb248 --- /dev/null +++ b/examples/raft-engine-log/README.md @@ -0,0 +1,98 @@ +# raft-engine-log + +A 3-node embedded d-engine cluster whose Raft log is stored in +[raft-engine](https://github.com/tikv/raft-engine) instead of the RocksDB WAL. +Raft consensus and the RocksDB state machine are unchanged; only the log store +and the hard-state store are replaced. + +**Status: experimental.** The storage adapter is covered by unit tests and a +3-node smoke run, but it has not been benchmarked on the target disk yet. It is +not a production recommendation. + +## Why + +Since the log is fsynced before a write is acknowledged, the cost of one +`fdatasync` dominates commit latency on network disks. raft-engine writes into +preallocated segment files and overwrites them, so a sync does not have to +commit file-length metadata the way appending to a growing file does. This +example measures whether that helps. + +## Run + +```sh +make start-cluster # builds, then starts 3 nodes +``` + +| Node | API port | Health port | Data dir | +| ---- | -------- | ----------- | -------- | +| 1 | 8081 | 10001 | `db/1` | +| 2 | 8082 | 10002 | `db/2` | +| 3 | 8083 | 10003 | `db/3` | + +Write through the leader (`GET /primary` on a health port returns 200 on the +leader), read from any node: + +```sh +curl -X POST localhost:8083/kv -H 'content-type: application/json' \ + -d '{"key":"k1","value":"v1"}' +curl localhost:8081/kv/k1 +``` + +## Configuration + +Log-store settings go in an optional `[raft_engine]` table of the same node +config file you pass to d-engine (`--config-path`). d-engine ignores the table; +the adapter reads it. Every key is optional. + +```toml +[raft_engine] +target_file_size_mib = 64 # size at which the active log file is rotated +purge_threshold_mib = 512 # reclaim log files once the retained log exceeds this +prefill_limit_mib = 256 # zero-filled recycle pool prepared at startup +purge_interval_secs = 10 # how often expired log files are reclaimed +``` + +- A misspelled key, a zero value, or `purge_threshold_mib` below + `target_file_size_mib` makes startup fail with a message naming the problem. +- The recycle pool holds `prefill_limit_mib / target_file_size_mib` files. A + write into a recycled file overwrites blocks that are already allocated, + which is what avoids the extra metadata commit on sync. The first log file is + created fresh and does not have that property, so a run shorter than one + segment never reaches a recycled file; use a small `target_file_size_mib` + when benchmarking. +- Embedded code reads the table with `RaftEngineOptions::from_toml_file(path)` + and opens the store with `RaftEngineStorage::open_with(dir, &options)`. + +## Test + +```sh +cargo nextest run --all-features --nocapture +``` + +The tests include the shared `StorageEngine` contract suite from +`d_engine::storage_engine_test`, plus restart recovery, tail replacement and +truncation, `reset` keeping the metadata, purge-boundary and hard-state +persistence, and purge-thread shutdown. + +## Behavior to know + +- raft-engine has no suffix-truncate command, so `truncate` re-appends the entry + just before the cut, which drops everything after it in one atomic write. + `replace_range` overwrites the tail the same way. +- Log entries and the metadata (hard state, purge boundary) live in separate + raft-engine regions, so `reset` clears the log without touching the metadata. +- `persist_entries` and `save_hard_state` write without syncing. `flush()` runs + `fdatasync`, matching the RocksDB adapter's drain-then-fsync model, so + `durable_index` advances only after the physical sync. +- Entries are stored as d-engine's own encoded `Entry` bytes (see + `src/entry_record.rs`), with no wrapper message. +- A background thread reclaims log files every 10 seconds. raft-engine never + deletes files on its own. +- Each node preallocates `prefill_limit_mib` of zero-filled log files at startup + (256 MiB by default; about 380 MB on disk with the default segment size). + +## Not covered yet + +- Tail corruption after a crash. +- Single-node failure and leader change under write load. +- Latency measurements on the target disk. diff --git a/examples/raft-engine-log/config/n1.toml b/examples/raft-engine-log/config/n1.toml new file mode 100644 index 00000000..d86d3a15 --- /dev/null +++ b/examples/raft-engine-log/config/n1.toml @@ -0,0 +1,18 @@ +[cluster] +node_id = 1 +listen_address = "0.0.0.0:9081" +initial_cluster = [ + { id = 1, address = "0.0.0.0:9081", role = 1, status = 3 }, + { id = 2, address = "0.0.0.0:9082", role = 1, status = 3 }, + { id = 3, address = "0.0.0.0:9083", role = 1, status = 3 }, +] + +[raft.snapshot] +enable = true + +# Log store settings; every key is optional and the values below are the defaults. +[raft_engine] +target_file_size_mib = 64 # size at which the active log file is rotated +purge_threshold_mib = 512 # reclaim log files once the retained log exceeds this +prefill_limit_mib = 256 # zero-filled recycle pool prepared at startup +purge_interval_secs = 10 # how often expired log files are reclaimed diff --git a/examples/raft-engine-log/config/n2.toml b/examples/raft-engine-log/config/n2.toml new file mode 100644 index 00000000..d27a20d3 --- /dev/null +++ b/examples/raft-engine-log/config/n2.toml @@ -0,0 +1,18 @@ +[cluster] +node_id = 2 +listen_address = "0.0.0.0:9082" +initial_cluster = [ + { id = 1, address = "0.0.0.0:9081", role = 1, status = 3 }, + { id = 2, address = "0.0.0.0:9082", role = 1, status = 3 }, + { id = 3, address = "0.0.0.0:9083", role = 1, status = 3 }, +] + +[raft.snapshot] +enable = true + +# Log store settings; every key is optional and the values below are the defaults. +[raft_engine] +target_file_size_mib = 64 # size at which the active log file is rotated +purge_threshold_mib = 512 # reclaim log files once the retained log exceeds this +prefill_limit_mib = 256 # zero-filled recycle pool prepared at startup +purge_interval_secs = 10 # how often expired log files are reclaimed diff --git a/examples/raft-engine-log/config/n3.toml b/examples/raft-engine-log/config/n3.toml new file mode 100644 index 00000000..f8976ca0 --- /dev/null +++ b/examples/raft-engine-log/config/n3.toml @@ -0,0 +1,18 @@ +[cluster] +node_id = 3 +listen_address = "0.0.0.0:9083" +initial_cluster = [ + { id = 1, address = "0.0.0.0:9081", role = 1, status = 3 }, + { id = 2, address = "0.0.0.0:9082", role = 1, status = 3 }, + { id = 3, address = "0.0.0.0:9083", role = 1, status = 3 }, +] + +[raft.snapshot] +enable = true + +# Log store settings; every key is optional and the values below are the defaults. +[raft_engine] +target_file_size_mib = 64 # size at which the active log file is rotated +purge_threshold_mib = 512 # reclaim log files once the retained log exceeds this +prefill_limit_mib = 256 # zero-filled recycle pool prepared at startup +purge_interval_secs = 10 # how often expired log files are reclaimed diff --git a/examples/raft-engine-log/src/entry_record.rs b/examples/raft-engine-log/src/entry_record.rs new file mode 100644 index 00000000..2b7e1e88 --- /dev/null +++ b/examples/raft-engine-log/src/entry_record.rs @@ -0,0 +1,110 @@ +//! Adapts d-engine's prost `Entry` to raft-engine's `protobuf::Message` bound. +//! +//! raft-engine serializes log entries through `write_to_vec` and parses them back +//! through `parse_from_bytes`. Delegating both to prost stores d-engine's own +//! `Entry` bytes on disk, with no wrapper message and no second encoding. + +use std::any::Any; +use std::fmt; +use std::io::{self, Read}; +use std::sync::OnceLock; + +use d_engine::common::Entry; +use prost::Message as _; +use protobuf::reflect::MessageDescriptor; +use protobuf::{ + Clear, CodedInputStream, CodedOutputStream, Message, ProtobufError, ProtobufResult, + UnknownFields, +}; + +#[derive(Clone, PartialEq, Default)] +pub struct EntryRecord { + pub entry: Entry, + unknown: UnknownFields, +} + +impl From for EntryRecord { + fn from(entry: Entry) -> Self { + Self { + entry, + unknown: UnknownFields::default(), + } + } +} + +impl fmt::Debug for EntryRecord { + fn fmt( + &self, + f: &mut fmt::Formatter<'_>, + ) -> fmt::Result { + self.entry.fmt(f) + } +} + +impl Clear for EntryRecord { + fn clear(&mut self) { + self.entry = Entry::default(); + } +} + +impl Message for EntryRecord { + // Reflection is never used on the raft-engine read/write path. + fn descriptor(&self) -> &'static MessageDescriptor { + unreachable!("reflection is not used on the log path") + } + + fn descriptor_static() -> &'static MessageDescriptor { + unreachable!("reflection is not used on the log path") + } + + fn is_initialized(&self) -> bool { + true + } + + fn merge_from( + &mut self, + is: &mut CodedInputStream, + ) -> ProtobufResult<()> { + let mut buf = Vec::new(); + is.read_to_end(&mut buf)?; + self.entry = Entry::decode(buf.as_slice()) + .map_err(|e| ProtobufError::IoError(io::Error::new(io::ErrorKind::InvalidData, e)))?; + Ok(()) + } + + fn write_to_with_cached_sizes( + &self, + os: &mut CodedOutputStream, + ) -> ProtobufResult<()> { + os.write_raw_bytes(&self.entry.encode_to_vec()) + } + + fn compute_size(&self) -> u32 { + self.entry.encoded_len() as u32 + } + + fn get_cached_size(&self) -> u32 { + self.entry.encoded_len() as u32 + } + + fn get_unknown_fields(&self) -> &UnknownFields { + &self.unknown + } + + fn mut_unknown_fields(&mut self) -> &mut UnknownFields { + &mut self.unknown + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn new() -> Self { + Self::default() + } + + fn default_instance() -> &'static Self { + static DEFAULT: OnceLock = OnceLock::new(); + DEFAULT.get_or_init(EntryRecord::default) + } +} diff --git a/examples/raft-engine-log/src/lib.rs b/examples/raft-engine-log/src/lib.rs new file mode 100644 index 00000000..d5412bc8 --- /dev/null +++ b/examples/raft-engine-log/src/lib.rs @@ -0,0 +1,17 @@ +//! raft-engine-backed `StorageEngine` for d-engine. +//! +//! The log and the hard state live in raft-engine, which writes into +//! preallocated segment files instead of appending to a growing file. + +mod entry_record; +mod options; +mod raft_engine_storage; +#[cfg(test)] +mod raft_engine_storage_test; +#[cfg(test)] +mod storage_engine_suite_test; +#[cfg(test)] +mod options_test; + +pub use options::RaftEngineOptions; +pub use raft_engine_storage::{RaftEngineLogStore, RaftEngineMetaStore, RaftEngineStorage}; diff --git a/examples/raft-engine-log/src/main.rs b/examples/raft-engine-log/src/main.rs new file mode 100644 index 00000000..1896dadf --- /dev/null +++ b/examples/raft-engine-log/src/main.rs @@ -0,0 +1,249 @@ + +use clap::Parser; +use std::sync::Arc; +use std::time::Duration; + +use axum::{ + Json, Router, + extract::{Path, State}, + http::StatusCode, + routing::{get, post}, +}; +use d_engine::{ClientApiError, EmbeddedEngine, ErrorCode, RocksDBStateMachine}; +use raft_engine_log::{RaftEngineOptions, RaftEngineStorage}; +use serde::{Deserialize, Serialize}; +use tokio::sync::watch; + +type Engine = EmbeddedEngine; + +#[derive(Debug, Deserialize)] +struct PutRequest { + key: String, + value: String, +} + +#[derive(Debug, Serialize)] +struct GetResponse { + value: Option, +} + +#[derive(Debug, clap::Parser)] +struct Cli { + #[clap(long, default_value = "8080")] + port: u16, + #[clap(long, default_value = "8008")] + health_port: u16, + #[clap(long)] + config_path: String, + #[clap(long)] + data_dir: String, +} + +#[tokio::main] +async fn main() { + // ============================================================ + // Application Setup (Your Code) + // ============================================================ + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .init(); + + let config_path = std::env::var("CONFIG_PATH").ok(); + let mut cli = Cli::parse(); + if let Some(path) = config_path { + cli.config_path = path; + } + + println!( + "starting (api::{} health::{} config::{})", + cli.port, cli.health_port, cli.config_path + ); + + // ============================================================ + // d-engine Integration (Start & Wait for Leader Election) + // ============================================================ + let options = RaftEngineOptions::from_toml_file(&cli.config_path).expect("read [raft_engine]"); + let storage = Arc::new( + RaftEngineStorage::open_with(format!("{}/log", cli.data_dir), &options) + .expect("open log"), + ); + let sm = + Arc::new(RocksDBStateMachine::new(format!("{}/state", cli.data_dir)).expect("open state")); + + let engine = Arc::new( + Engine::start_custom(&cli.data_dir, storage, sm, Some(cli.config_path.as_str())) + .await + .expect("Failed to start engine"), + ); + + let leader_info = match engine.wait_ready(Duration::from_secs(5)).await { + Ok(info) => info, + Err(err) => { + eprintln!("✗ node failed to become ready: {err}"); + std::process::exit(1); + } + }; + + println!( + "✓ node {} ready — leader {}", + engine.node_id(), + leader_info.leader_id, + ); + + // ============================================================ + // Application Layer: Start HTTP Services + // ============================================================ + let (shutdown_tx, shutdown_rx) = watch::channel(()); + + // Health check server (for load balancers) + let mut health_handle = tokio::spawn({ + let engine = engine.clone(); + let shutdown_rx = shutdown_rx.clone(); + async move { + start_health_check_server(engine, cli.health_port, shutdown_rx).await; + } + }); + + // Business API server (your KV service) + let mut business_handle = tokio::spawn({ + let engine = engine.clone(); + let shutdown_rx = shutdown_rx.clone(); + async move { + start_business_server(engine, cli.port, shutdown_rx).await; + } + }); + + // ============================================================ + // Graceful Shutdown Handling (Application Responsibility) + // ============================================================ + tokio::select! { + res = tokio::signal::ctrl_c() => { + if let Err(e) = res { + eprintln!("✗ failed to install Ctrl+C handler: {e}"); + } + println!("shutting down..."); + } + _ = &mut health_handle => { + eprintln!("✗ health check server task exited unexpectedly"); + } + _ = &mut business_handle => { + eprintln!("✗ business server task exited unexpectedly"); + } + } + + // Stop HTTP servers + let _ = shutdown_tx.send(()); + let _ = tokio::join!(health_handle, business_handle); + + // ============================================================ + // d-engine Cleanup (Stop & Flush Data) + // ============================================================ + match Arc::try_unwrap(engine) { + Ok(engine) => { + if let Err(e) = engine.stop().await { + eprintln!("✗ error during shutdown: {e}"); + } + } + Err(_) => { + eprintln!("✗ warning: cannot stop engine - references still exist"); + } + } + + println!("✓ shutdown complete"); +} + +async fn start_health_check_server( + engine: Arc, + port: u16, + mut shutdown_rx: watch::Receiver<()>, +) { + let app = Router::new() + .route("/primary", get(health_primary)) + .route("/replica", get(health_replica)) + .with_state(engine); + + let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}")) + .await + .expect("Failed to bind health check server"); + + axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.changed().await; + }) + .await + .expect("Health check server failed"); +} + +async fn health_primary(State(engine): State>) -> StatusCode { + if engine.is_leader() { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } +} + +async fn health_replica(State(engine): State>) -> StatusCode { + if !engine.is_leader() { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE + } +} + +async fn start_business_server( + engine: Arc, + port: u16, + mut shutdown_rx: watch::Receiver<()>, +) { + let app = Router::new() + .route("/kv", post(handle_put)) + .route("/kv/:key", get(handle_get)) + .with_state(engine); + + let listener = tokio::net::TcpListener::bind(format!("0.0.0.0:{port}")) + .await + .expect("Failed to bind business server"); + + axum::serve(listener, app) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.changed().await; + }) + .await + .expect("Business server failed"); +} + +fn is_not_leader(e: &ClientApiError) -> bool { + e.code() == ErrorCode::NotLeader +} + +async fn handle_put( + State(engine): State>, + Json(req): Json, +) -> (StatusCode, Json) { + match engine.client().put(req.key.into_bytes(), req.value.into_bytes()).await { + Ok(_) => (StatusCode::OK, Json(serde_json::json!({}))), + Err(e) if is_not_leader(&e) => ( + StatusCode::SERVICE_UNAVAILABLE, + Json(serde_json::json!({ + "error": "not_leader", + "hint": "Use GET /primary on the health port to find the leader, then retry the write there." + })), + ), + Err(e) => ( + StatusCode::INTERNAL_SERVER_ERROR, + Json(serde_json::json!({ "error": format!("{e}") })), + ), + } +} + +async fn handle_get( + State(engine): State>, + Path(key): Path, +) -> Result, StatusCode> { + match engine.client().get_eventual(key.into_bytes()).await { + Ok(value) => Ok(Json(GetResponse { + value: value.map(|v| String::from_utf8_lossy(&v).to_string()), + })), + Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR), + } +} diff --git a/examples/raft-engine-log/src/options.rs b/examples/raft-engine-log/src/options.rs new file mode 100644 index 00000000..ee85fa9f --- /dev/null +++ b/examples/raft-engine-log/src/options.rs @@ -0,0 +1,89 @@ +//! Tunables for the raft-engine log store. +//! +//! They are read from an optional `[raft_engine]` table in the node's config +//! file, next to the d-engine settings. d-engine itself ignores that table. + +use std::path::Path; + +use d_engine::{Error, StorageError}; +use serde::Deserialize; + +/// Table name in the node config file. +const TABLE: &str = "raft_engine"; + +/// Log-store settings. Every field is optional in the file; a missing field (or +/// a missing table) keeps its default. A misspelled key is rejected. +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct RaftEngineOptions { + /// Size at which the active log file is closed and the next one is started. + pub target_file_size_mib: u64, + /// Log files are reclaimed once the retained log grows past this size. + pub purge_threshold_mib: u64, + /// Zero-filled files prepared at startup for reuse as future log files. + /// Writes into these files overwrite already-allocated blocks. + pub prefill_limit_mib: u64, + /// How often expired log files are reclaimed. + pub purge_interval_secs: u64, +} + +impl Default for RaftEngineOptions { + fn default() -> Self { + Self { + target_file_size_mib: 64, + purge_threshold_mib: 512, + prefill_limit_mib: 256, + purge_interval_secs: 10, + } + } +} + +impl RaftEngineOptions { + /// Reads the `[raft_engine]` table of the config file at `path`. + pub fn from_toml_file(path: impl AsRef) -> Result { + let path = path.as_ref(); + let text = std::fs::read_to_string(path).map_err(StorageError::IoError)?; + let doc: toml::Table = toml::from_str(&text).map_err(|e| { + StorageError::LogStorage(format!("invalid config file {}: {e}", path.display())) + })?; + let options = match doc.get(TABLE) { + None => Self::default(), + Some(table) => table.clone().try_into().map_err(|e| { + StorageError::LogStorage(format!( + "invalid [{TABLE}] in {}: {e}", + path.display() + )) + })?, + }; + options.validate()?; + Ok(options) + } + + /// Rejects values that would make the store misbehave instead of failing + /// later in a less obvious place. + pub fn validate(&self) -> Result<(), Error> { + let positive = [ + ("target_file_size_mib", self.target_file_size_mib), + ("purge_threshold_mib", self.purge_threshold_mib), + ("prefill_limit_mib", self.prefill_limit_mib), + // A zero interval would make the purge thread spin. + ("purge_interval_secs", self.purge_interval_secs), + ]; + for (name, value) in positive { + if value == 0 { + return Err(StorageError::LogStorage(format!( + "[{TABLE}] {name} must be greater than 0" + )) + .into()); + } + } + if self.purge_threshold_mib < self.target_file_size_mib { + return Err(StorageError::LogStorage(format!( + "[{TABLE}] purge_threshold_mib ({}) must not be smaller than target_file_size_mib ({})", + self.purge_threshold_mib, self.target_file_size_mib + )) + .into()); + } + Ok(()) + } +} diff --git a/examples/raft-engine-log/src/options_test.rs b/examples/raft-engine-log/src/options_test.rs new file mode 100644 index 00000000..8b3f2890 --- /dev/null +++ b/examples/raft-engine-log/src/options_test.rs @@ -0,0 +1,139 @@ +use std::path::Path; + +use tempfile::TempDir; + +use crate::options::RaftEngineOptions; +use crate::raft_engine_storage::RaftEngineStorage; + +fn write_config( + dir: &TempDir, + body: &str, +) -> std::path::PathBuf { + let path = dir.path().join("node.toml"); + std::fs::write(&path, body).unwrap(); + path +} + +fn count_log_files(dir: &Path) -> usize { + std::fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().ends_with(".raftlog")) + .count() +} + +/// A node config with no `[raft_engine]` table is the common case and must keep +/// the defaults, while the d-engine settings around it are not an error. +#[test] +fn test_options_missing_table_uses_defaults() { + let dir = TempDir::new().unwrap(); + let path = write_config(&dir, "[cluster]\nnode_id = 1\n"); + + assert_eq!( + RaftEngineOptions::from_toml_file(path).unwrap(), + RaftEngineOptions::default() + ); +} + +/// Only the keys present in the file change; the rest stay at their defaults. +#[test] +fn test_options_partial_table_overrides_only_given_keys() { + let dir = TempDir::new().unwrap(); + let path = write_config(&dir, "[cluster]\nnode_id = 1\n[raft_engine]\ntarget_file_size_mib = 8\n"); + + let options = RaftEngineOptions::from_toml_file(path).unwrap(); + + assert_eq!(options.target_file_size_mib, 8); + assert_eq!( + options.purge_threshold_mib, + RaftEngineOptions::default().purge_threshold_mib + ); +} + +/// A misspelled key must fail loudly; silently ignoring it would run the +/// benchmark with a segment size nobody asked for. +#[test] +fn test_options_unknown_key_is_rejected() { + let dir = TempDir::new().unwrap(); + let path = write_config(&dir, "[raft_engine]\ntarget_file_size = 8\n"); + + assert!(RaftEngineOptions::from_toml_file(path).is_err()); +} + +/// Values that would stall or break the store are refused at load time. +#[test] +fn test_options_invalid_values_are_rejected() { + let dir = TempDir::new().unwrap(); + for body in [ + "[raft_engine]\ntarget_file_size_mib = 0\n", + "[raft_engine]\npurge_interval_secs = 0\n", + "[raft_engine]\ntarget_file_size_mib = 64\npurge_threshold_mib = 32\n", + ] { + let path = write_config(&dir, body); + assert!( + RaftEngineOptions::from_toml_file(path).is_err(), + "accepted: {body}" + ); + } +} + +/// The configured segment size is the one raft-engine rotates at: a 1 MiB +/// segment must be rotated after about 3 MiB of entries, whereas the 64 MiB +/// default keeps them all in one file. +#[tokio::test] +async fn test_options_target_file_size_controls_rotation() { + use d_engine::common::{Entry, EntryPayload, entry_payload}; + use d_engine::{LogStore, StorageEngine}; + use prost::bytes::Bytes; + + let entries = |range: std::ops::RangeInclusive| -> Vec { + range + .map(|i| Entry { + index: i, + term: 1, + payload: Some(EntryPayload { + payload: Some(entry_payload::Payload::Command(Bytes::from(vec![7u8; 1024]))), + }), + }) + .collect() + }; + + let small_dir = TempDir::new().unwrap(); + let small = RaftEngineStorage::open_with( + small_dir.path().join("log"), + &RaftEngineOptions { + target_file_size_mib: 1, + purge_threshold_mib: 8, + prefill_limit_mib: 4, + ..Default::default() + }, + ) + .unwrap(); + for chunk in 0..3 { + small + .log_store() + .persist_entries(entries(chunk * 1000 + 1..=chunk * 1000 + 1000)) + .await + .unwrap(); + } + + let default_dir = TempDir::new().unwrap(); + let default = RaftEngineStorage::open_with( + default_dir.path().join("log"), + &RaftEngineOptions { + prefill_limit_mib: 4, + ..Default::default() + }, + ) + .unwrap(); + for chunk in 0..3 { + default + .log_store() + .persist_entries(entries(chunk * 1000 + 1..=chunk * 1000 + 1000)) + .await + .unwrap(); + } + + assert!(count_log_files(&small_dir.path().join("log")) >= 2); + assert_eq!(count_log_files(&default_dir.path().join("log")), 1); +} diff --git a/examples/raft-engine-log/src/raft_engine_storage.rs b/examples/raft-engine-log/src/raft_engine_storage.rs new file mode 100644 index 00000000..26993ba2 --- /dev/null +++ b/examples/raft-engine-log/src/raft_engine_storage.rs @@ -0,0 +1,406 @@ +//! raft-engine-backed `StorageEngine` for spike #460. +//! +//! Replaces the RocksDB WAL log store with raft-engine (preallocated, +//! overwrite-based segments) while keeping d-engine's Raft core and state +//! machine untouched. The single raft group lives under a fixed region id. + +use std::ops::RangeInclusive; +use std::path::Path; +use std::sync::Arc; +use std::sync::mpsc::{self, RecvTimeoutError}; +use std::thread::JoinHandle; +use std::time::Duration; + +use crate::entry_record::EntryRecord; +use crate::options::RaftEngineOptions; +use async_trait::async_trait; +use d_engine::common::{Entry, LogId}; +use d_engine::{Error, HardState, LogStore, MetaStore, StorageEngine, StorageError}; +use prost::Message as _; +use raft_engine::{Command, Config, Engine, LogBatch, MessageExt, ReadableSize}; + +/// Raft log entries. +const LOG_REGION_ID: u64 = 1; +/// Hard state and purge boundary. Kept apart from the entries because `reset` +/// clears every key of a region and must leave these untouched. +const META_REGION_ID: u64 = 2; + +const HARD_STATE_KEY: &[u8] = b"hard_state"; +const PURGE_BOUNDARY_KEY: &[u8] = b"purge_boundary"; + +/// Bridges d-engine's prost `Entry` to raft-engine's rust-protobuf entry type. +struct EntryAdapter; + +impl MessageExt for EntryAdapter { + type Entry = EntryRecord; + + fn index(e: &Self::Entry) -> u64 { + e.entry.index + } +} + +/// Reclaims log files whose entries have all been compacted. +/// +/// raft-engine never deletes files by itself: `LogStore::purge` only compacts the +/// in-memory index, and the files are released when `purge_expired_files` runs. +/// That call deletes files and may rewrite older ones, so it runs on a dedicated +/// thread and never on the Raft loop. A failed round is logged and retried on the +/// next tick: the log stays durable, only disk space is delayed. +struct PurgeWorker { + stop: Option>, + handle: Option>, +} + +impl PurgeWorker { + fn spawn( + engine: Arc, + interval: Duration, + ) -> Result { + let (stop, stop_rx) = mpsc::channel::<()>(); + let handle = std::thread::Builder::new() + .name("raft-engine-purge".into()) + .spawn(move || { + // Only a timeout means "run a round"; a message or a closed + // channel means "stop". + while let Err(RecvTimeoutError::Timeout) = stop_rx.recv_timeout(interval) { + match engine.purge_expired_files() { + Ok(regions) => tracing::debug!(?regions, "purged expired log files"), + Err(e) => tracing::error!("failed to purge expired log files: {e}"), + } + } + }) + .map_err(|e| StorageError::LogStorage(format!("failed to spawn purge thread: {e}")))?; + Ok(Self { + stop: Some(stop), + handle: Some(handle), + }) + } +} + +impl Drop for PurgeWorker { + fn drop(&mut self) { + // Closing the channel wakes the thread out of its wait immediately. + self.stop.take(); + if let Some(handle) = self.handle.take() + && handle.join().is_err() + { + tracing::error!("raft-engine purge thread panicked"); + } + } +} + +pub struct RaftEngineLogStore { + engine: Arc, + _purge_worker: PurgeWorker, +} + +pub struct RaftEngineMetaStore { + engine: Arc, +} + +pub struct RaftEngineStorage { + log_store: Arc, + meta_store: Arc, + path: String, +} + +impl RaftEngineStorage { + /// Opens raft-engine at `path` with the default options. + pub fn open(path: impl AsRef) -> Result { + Self::open_with(path, &RaftEngineOptions::default()) + } + + /// Opens raft-engine at `path` with the given options. + pub fn open_with( + path: impl AsRef, + options: &RaftEngineOptions, + ) -> Result { + options.validate()?; + let path = path.as_ref(); + // raft-engine creates only the last path component, so a nested data + // directory (e.g. `db/1/log`) must exist before it opens. + std::fs::create_dir_all(path).map_err(StorageError::IoError)?; + let config = Config { + dir: path.to_string_lossy().into_owned(), + target_file_size: ReadableSize::mb(options.target_file_size_mib), + purge_threshold: ReadableSize::mb(options.purge_threshold_mib), + prefill_for_recycle: true, + prefill_limit: Some(ReadableSize::mb(options.prefill_limit_mib)), + batch_compression_threshold: ReadableSize::kb(0), + ..Default::default() + }; + let engine = + Arc::new(Engine::open(config).map_err(|e| StorageError::LogStorage(e.to_string()))?); + let purge_worker = PurgeWorker::spawn( + engine.clone(), + Duration::from_secs(options.purge_interval_secs), + )?; + Ok(Self { + log_store: Arc::new(RaftEngineLogStore { + engine: engine.clone(), + _purge_worker: purge_worker, + }), + meta_store: Arc::new(RaftEngineMetaStore { engine }), + path: path.to_string_lossy().into_owned(), + }) + } +} + +impl std::fmt::Debug for RaftEngineStorage { + fn fmt( + &self, + f: &mut std::fmt::Formatter<'_>, + ) -> std::fmt::Result { + f.debug_struct("RaftEngineStorage").field("path", &self.path).finish() + } +} + +impl StorageEngine for RaftEngineStorage { + type LogStore = RaftEngineLogStore; + type MetaStore = RaftEngineMetaStore; + + fn log_store(&self) -> Arc { + self.log_store.clone() + } + + fn meta_store(&self) -> Arc { + self.meta_store.clone() + } +} + +#[async_trait] +impl LogStore for RaftEngineLogStore { + async fn persist_entries( + &self, + entries: Vec, + ) -> Result<(), Error> { + let mut batch = LogBatch::default(); + let raft_entries: Vec = entries.into_iter().map(EntryRecord::from).collect(); + + batch + .add_entries::(LOG_REGION_ID, &raft_entries) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + // sync=false: lands in the OS page cache; the IO thread's flush() + // (`Engine::sync`) makes it durable, mirroring the RocksDB adapter's + // drain-then-fsync model. + self.engine + .write(&mut batch, false) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + Ok(()) + } + + async fn entry( + &self, + index: u64, + ) -> Result, Error> { + match self + .engine + .get_entry::(LOG_REGION_ID, index) + .map_err(|e| StorageError::LogStorage(e.to_string()))? + { + Some(r) => Ok(Some(r.entry)), + None => Ok(None), + } + } + + fn get_entries( + &self, + range: RangeInclusive, + ) -> Result, Error> { + let Some(first) = self.engine.first_index(LOG_REGION_ID) else { + return Ok(Vec::new()); + }; + let last = self.engine.last_index(LOG_REGION_ID).unwrap_or(first); + let begin = (*range.start()).max(first); + let end = (*range.end()).min(last); + if begin > end { + return Ok(Vec::new()); + } + // fetch_entries_to is [begin, end); the trait range is inclusive. + let mut raw = Vec::new(); + self.engine + .fetch_entries_to::(LOG_REGION_ID, begin, end + 1, None, &mut raw) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + Ok(raw.into_iter().map(|r| r.entry).collect()) + } + + async fn purge( + &self, + cutoff_index: LogId, + ) -> Result<(), Error> { + // Compact the prefix and record the boundary in one batch, so a reader + // never sees entries gone before the boundary is persisted (#442). + let mut batch = LogBatch::default(); + batch.add_command( + LOG_REGION_ID, + Command::Compact { + // raft-engine compacts entries strictly below `index`, while the + // cutoff is inclusive. + index: cutoff_index.index + 1, + }, + ); + batch + .put( + META_REGION_ID, + PURGE_BOUNDARY_KEY.to_vec(), + cutoff_index.encode_to_vec(), + ) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + self.engine + .write(&mut batch, false) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + Ok(()) + } + + async fn truncate( + &self, + from_index: u64, + ) -> Result<(), Error> { + let (Some(first), Some(last)) = ( + self.engine.first_index(LOG_REGION_ID), + self.engine.last_index(LOG_REGION_ID), + ) else { + return Ok(()); + }; + if from_index > last { + return Ok(()); + } + + let mut batch = LogBatch::default(); + if from_index > first { + // raft-engine has no suffix-truncate command, but appending an index + // that is already present drops every entry after it. Re-appending the + // entry just before `from_index` therefore cuts the tail in one write. + let keep = self + .engine + .get_entry::(LOG_REGION_ID, from_index - 1) + .map_err(|e| StorageError::LogStorage(e.to_string()))? + .ok_or_else(|| { + StorageError::LogStorage(format!("entry {} missing", from_index - 1)) + })?; + batch + .add_entries::(LOG_REGION_ID, &[keep]) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + } else { + // Nothing below `from_index` survives: compact past the last entry. + batch.add_command(LOG_REGION_ID, Command::Compact { index: last + 1 }); + } + self.engine + .write(&mut batch, false) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + Ok(()) + } + + async fn replace_range( + &self, + from_index: u64, + new_entries: Vec, + ) -> Result { + if new_entries.is_empty() { + self.truncate(from_index).await?; + return Ok(from_index.saturating_sub(1)); + } + + // Guard the two append panic conditions in raft-engine's MemTable. An empty + // log (never written, or fully purged) has no span to overwrite or leave a + // hole in, so only a non-empty log is checked. + if let Some(first) = self.engine.first_index(LOG_REGION_ID) + && from_index < first + { + return Err(StorageError::LogStorage( + "replace_range would overwrite compacted history".into(), + ) + .into()); + } + if let Some(last) = self.engine.last_index(LOG_REGION_ID) + && from_index > last + 1 + { + return Err( + StorageError::LogStorage("replace_range would create a hole".into()).into(), + ); + } + let mut batch = LogBatch::default(); + let last = new_entries.last().map(|e| e.index); + let raft_entries: Vec = + new_entries.into_iter().map(EntryRecord::from).collect(); + batch + .add_entries::(LOG_REGION_ID, &raft_entries) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + self.engine + .write(&mut batch, false) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + Ok(last.unwrap_or(from_index.saturating_sub(1))) + } + + fn is_write_durable(&self) -> bool { + false + } + + fn flush(&self) -> Result<(), Error> { + self.engine.sync().map_err(|e| StorageError::LogStorage(e.to_string()))?; + Ok(()) + } + + async fn flush_async(&self) -> Result<(), Error> { + self.flush() + } + + async fn reset(&self) -> Result<(), Error> { + let mut batch = LogBatch::default(); + batch.add_command(LOG_REGION_ID, Command::Clean); + self.engine + .write(&mut batch, false) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + Ok(()) + } + + fn last_index(&self) -> u64 { + self.engine.last_index(LOG_REGION_ID).unwrap_or(0) + } + + fn load_purge_boundary(&self) -> Result, Error> { + match self.engine.get(META_REGION_ID, PURGE_BOUNDARY_KEY) { + Some(bytes) => LogId::decode(&*bytes) + .map(Some) + .map_err(|e| StorageError::SerializationError(e.to_string()).into()), + None => Ok(None), + } + } +} + +#[async_trait] +impl MetaStore for RaftEngineMetaStore { + fn save_hard_state( + &self, + state: &HardState, + ) -> Result<(), Error> { + let bytes = bincode::serialize(state).map_err(StorageError::BincodeError)?; + let mut batch = LogBatch::default(); + batch + .put(META_REGION_ID, HARD_STATE_KEY.to_vec(), bytes) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + // Not synced here: the caller flushes the meta store right after this call. + self.engine + .write(&mut batch, false) + .map_err(|e| StorageError::LogStorage(e.to_string()))?; + Ok(()) + } + + fn load_hard_state(&self) -> Result, Error> { + match self.engine.get(META_REGION_ID, HARD_STATE_KEY) { + Some(bytes) => Ok(Some( + bincode::deserialize(&bytes).map_err(StorageError::BincodeError)?, + )), + None => Ok(None), + } + } + + fn flush(&self) -> Result<(), Error> { + self.engine.sync().map_err(|e| StorageError::LogStorage(e.to_string()))?; + Ok(()) + } + + async fn flush_async(&self) -> Result<(), Error> { + self.flush() + } +} diff --git a/examples/raft-engine-log/src/raft_engine_storage_test.rs b/examples/raft-engine-log/src/raft_engine_storage_test.rs new file mode 100644 index 00000000..9267511c --- /dev/null +++ b/examples/raft-engine-log/src/raft_engine_storage_test.rs @@ -0,0 +1,441 @@ +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use d_engine::common::{Entry, EntryPayload, LogId, entry_payload}; +use d_engine::{HardState, LogStore, MetaStore, StorageEngine}; +use prost::bytes::Bytes; + +use crate::options::RaftEngineOptions; +use crate::raft_engine_storage::RaftEngineStorage; + +/// Unique scratch directory, removed on drop. +struct TestDir(PathBuf); + +impl TestDir { + fn new(name: &str) -> Self { + static SEQ: AtomicU64 = AtomicU64::new(0); + let dir = std::env::temp_dir().join(format!( + "raft-engine-log-{}-{}-{name}", + std::process::id(), + SEQ.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + Self(dir) + } + + fn path(&self) -> &Path { + &self.0 + } +} + +impl Drop for TestDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn entry( + index: u64, + term: u64, + cmd: &str, +) -> Entry { + Entry { + index, + term, + payload: Some(EntryPayload { + payload: Some(entry_payload::Payload::Command(Bytes::from(cmd.to_owned()))), + }), + } +} + +fn entries( + range: std::ops::RangeInclusive, + term: u64, +) -> Vec { + range.map(|i| entry(i, term, &format!("cmd-{i}"))).collect() +} + +/// Persist entries and make them durable, as the fsync worker does. +async fn append_durable( + storage: &RaftEngineStorage, + batch: Vec, +) { + let log = storage.log_store(); + log.persist_entries(batch).await.unwrap(); + log.flush().unwrap(); +} + +/// Entries that were persisted and flushed must be fully readable after the +/// engine is closed and reopened, including their payload bytes. +#[tokio::test] +async fn test_log_flushed_entries_survive_reopen() { + let dir = TestDir::new("reopen"); + let written = entries(1..=10, 3); + { + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + append_durable(&storage, written.clone()).await; + } + + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + assert_eq!(log.last_index(), 10); + assert_eq!(log.get_entries(1..=10).unwrap(), written); + assert_eq!(log.entry(7).await.unwrap(), Some(written[6].clone())); +} + +/// An entry without a payload must round-trip unchanged: the on-disk bytes are +/// the entry's own encoding, so an empty field must not turn into a default. +#[tokio::test] +async fn test_log_entry_without_payload_roundtrips() { + let dir = TestDir::new("nopayload"); + let bare = Entry { + index: 1, + term: 2, + payload: None, + }; + { + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + append_durable(&storage, vec![bare.clone()]).await; + } + + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + assert_eq!(storage.log_store().entry(1).await.unwrap(), Some(bare)); +} + +/// A term conflict replaces the tail from `from_index` in one atomic write: the +/// old suffix beyond the new entries must be gone, before and after a restart. +#[tokio::test] +async fn test_log_replace_range_drops_old_tail_after_reopen() { + let dir = TestDir::new("replace"); + let replacement = vec![entry(5, 2, "new-5"), entry(6, 2, "new-6")]; + { + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + append_durable(&storage, entries(1..=10, 1)).await; + let log = storage.log_store(); + + let last = log.replace_range(5, replacement.clone()).await.unwrap(); + log.flush().unwrap(); + + assert_eq!(last, 6); + assert_eq!(log.last_index(), 6); + assert_eq!(log.entry(7).await.unwrap(), None); + } + + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + assert_eq!(log.last_index(), 6); + assert_eq!(log.entry(4).await.unwrap(), Some(entry(4, 1, "cmd-4"))); + assert_eq!(log.get_entries(5..=6).unwrap(), replacement); + assert_eq!(log.entry(7).await.unwrap(), None); + assert_eq!(log.entry(10).await.unwrap(), None); +} + +/// `replace_range` must refuse the cases that would corrupt the index space +/// instead of reaching raft-engine's internal panics. +#[tokio::test] +async fn test_log_replace_range_invalid_input_returns_error() { + let dir = TestDir::new("replace-invalid"); + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + append_durable(&storage, entries(1..=5, 1)).await; + log.purge(LogId { term: 1, index: 2 }).await.unwrap(); + + // Starts inside the purged prefix. + assert!(log.replace_range(2, vec![entry(2, 2, "x")]).await.is_err()); + // Leaves a hole after the last index. + assert!(log.replace_range(8, vec![entry(8, 2, "x")]).await.is_err()); + // The failed calls left the log untouched. + assert_eq!(log.last_index(), 5); +} + +/// Appending exactly at `last + 1` is a plain extension, not a hole. +#[tokio::test] +async fn test_log_replace_range_at_last_plus_one_extends_log() { + let dir = TestDir::new("replace-extend"); + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + append_durable(&storage, entries(1..=5, 1)).await; + + let tail = vec![entry(6, 2, "n6"), entry(7, 2, "n7")]; + assert_eq!(log.replace_range(6, tail.clone()).await.unwrap(), 7); + + assert_eq!(log.last_index(), 7); + assert_eq!(log.get_entries(6..=7).unwrap(), tail); + assert_eq!(log.entry(5).await.unwrap(), Some(entry(5, 1, "cmd-5"))); +} + +/// After every entry has been purged the log is empty but the index space has +/// moved on: continuing at purged_last + 1 is legitimate and must not be +/// reported as a hole. +#[tokio::test] +async fn test_log_replace_range_after_full_purge_continues_at_next_index() { + let dir = TestDir::new("replace-after-purge"); + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + append_durable(&storage, entries(1..=5, 1)).await; + log.purge(LogId { term: 1, index: 5 }).await.unwrap(); + + let tail = vec![entry(6, 2, "n6")]; + assert_eq!(log.replace_range(6, tail.clone()).await.unwrap(), 6); + assert_eq!(log.get_entries(6..=6).unwrap(), tail); +} + +/// The purge boundary is written together with the compaction, so after a +/// restart both the removed prefix and the boundary must be visible. +#[tokio::test] +async fn test_log_purge_boundary_survives_reopen() { + let dir = TestDir::new("purge"); + let boundary = LogId { term: 4, index: 5 }; + { + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + append_durable(&storage, entries(1..=10, 4)).await; + let log = storage.log_store(); + assert_eq!(log.load_purge_boundary().unwrap(), None); + + log.purge(boundary).await.unwrap(); + log.flush().unwrap(); + } + + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + assert_eq!(log.load_purge_boundary().unwrap(), Some(boundary)); + assert_eq!(log.entry(5).await.unwrap(), None); + assert_eq!(log.entry(6).await.unwrap(), Some(entry(6, 4, "cmd-6"))); + assert_eq!(log.last_index(), 10); +} + +/// Reads past either end of the stored range are clamped, never an error. +#[tokio::test] +async fn test_log_get_entries_clamps_to_stored_range() { + let dir = TestDir::new("clamp"); + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + assert!(log.get_entries(1..=5).unwrap().is_empty()); + + append_durable(&storage, entries(3..=6, 1)).await; + let got = log.get_entries(1..=100).unwrap(); + assert_eq!(got.iter().map(|e| e.index).collect::>(), vec![3, 4, 5, 6]); + assert!(log.get_entries(7..=9).unwrap().is_empty()); +} + +/// `reset` drops every entry and returns `last_index` to 0. +#[tokio::test] +async fn test_log_reset_clears_all_entries() { + let dir = TestDir::new("reset"); + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + append_durable(&storage, entries(1..=5, 1)).await; + + log.reset().await.unwrap(); + + assert_eq!(log.last_index(), 0); + assert_eq!(log.entry(1).await.unwrap(), None); +} + +/// Truncating a suffix must drop it durably: after a restart the log still ends +/// at the entry before `from_index`, and the kept prefix is unchanged. +#[tokio::test] +async fn test_log_truncate_drops_suffix_after_reopen() { + let dir = TestDir::new("truncate"); + { + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + append_durable(&storage, entries(1..=10, 1)).await; + let log = storage.log_store(); + + log.truncate(6).await.unwrap(); + log.flush().unwrap(); + + assert_eq!(log.last_index(), 5); + } + + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + assert_eq!(log.last_index(), 5); + assert_eq!(log.get_entries(1..=10).unwrap(), entries(1..=5, 1)); + assert_eq!(log.entry(6).await.unwrap(), None); +} + +/// After a truncate the log accepts new entries from `from_index` again, which +/// is how a follower resumes after a conflict. +#[tokio::test] +async fn test_log_truncate_then_append_continues_from_cut() { + let dir = TestDir::new("truncate-append"); + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + append_durable(&storage, entries(1..=10, 1)).await; + + log.truncate(4).await.unwrap(); + // Checked before the append: appending index 4 would also drop the old tail + // by itself, so only this assertion proves `truncate` did the cut. + assert_eq!(log.last_index(), 3); + append_durable(&storage, vec![entry(4, 2, "new-4")]).await; + + assert_eq!(log.last_index(), 4); + assert_eq!(log.entry(4).await.unwrap(), Some(entry(4, 2, "new-4"))); + assert_eq!(log.entry(3).await.unwrap(), Some(entry(3, 1, "cmd-3"))); +} + +/// Truncating from the first stored index empties the log, including across a +/// restart, and a later append starts the log again. +#[tokio::test] +async fn test_log_truncate_from_first_index_empties_log() { + let dir = TestDir::new("truncate-all"); + { + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + append_durable(&storage, entries(1..=5, 1)).await; + let log = storage.log_store(); + + log.truncate(1).await.unwrap(); + log.flush().unwrap(); + assert_eq!(log.last_index(), 0); + } + + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + assert_eq!(log.last_index(), 0); + assert_eq!(log.entry(1).await.unwrap(), None); + + append_durable(&storage, entries(1..=2, 3)).await; + assert_eq!(log.last_index(), 2); +} + +/// Truncating at or past the end changes nothing. +#[tokio::test] +async fn test_log_truncate_beyond_last_index_is_noop() { + let dir = TestDir::new("truncate-noop"); + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let log = storage.log_store(); + assert!(log.truncate(1).await.is_ok()); + + append_durable(&storage, entries(1..=3, 1)).await; + log.truncate(4).await.unwrap(); + log.truncate(100).await.unwrap(); + + assert_eq!(log.get_entries(1..=3).unwrap(), entries(1..=3, 1)); +} + +/// `reset` clears the log only. The hard state and the purge boundary live in +/// the same engine and must survive it, in memory and after a restart. +#[tokio::test] +async fn test_log_reset_keeps_hard_state_and_purge_boundary() { + let dir = TestDir::new("reset-meta"); + let boundary = LogId { term: 2, index: 3 }; + { + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + append_durable(&storage, entries(1..=5, 2)).await; + let meta = storage.meta_store(); + meta.save_hard_state(&HardState { + current_term: 9, + voted_for: None, + }) + .unwrap(); + let log = storage.log_store(); + log.purge(boundary).await.unwrap(); + + log.reset().await.unwrap(); + log.flush().unwrap(); + + assert_eq!(log.last_index(), 0); + assert_eq!(meta.load_hard_state().unwrap().unwrap().current_term, 9); + assert_eq!(log.load_purge_boundary().unwrap(), Some(boundary)); + } + + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + assert_eq!(storage.log_store().last_index(), 0); + assert_eq!(storage.meta_store().load_hard_state().unwrap().unwrap().current_term, 9); + assert_eq!(storage.log_store().load_purge_boundary().unwrap(), Some(boundary)); +} + +/// `persist_entries` only reaches the OS cache. The store must report that, so +/// the core runs `flush()` before it advances `durable_index`; a wrong `true` +/// would acknowledge writes that a power loss can still discard. +#[tokio::test] +async fn test_log_write_is_not_reported_durable_without_flush() { + let dir = TestDir::new("durable-flag"); + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + assert!(!storage.log_store().is_write_durable()); +} + +/// The hard state saved before a restart is the one loaded after it. +#[tokio::test] +async fn test_meta_hard_state_survives_reopen() { + let dir = TestDir::new("hard-state"); + { + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let meta = storage.meta_store(); + assert!(meta.load_hard_state().unwrap().is_none()); + + meta.save_hard_state(&HardState { + current_term: 42, + voted_for: None, + }) + .unwrap(); + meta.flush().unwrap(); + } + + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + let loaded = storage.meta_store().load_hard_state().unwrap().unwrap(); + assert_eq!(loaded.current_term, 42); + assert!(loaded.voted_for.is_none()); +} + +/// Log entries and hard state share one engine; writing one must not disturb +/// the other across a restart. +#[tokio::test] +async fn test_log_and_hard_state_are_independent_after_reopen() { + let dir = TestDir::new("shared-engine"); + { + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + append_durable(&storage, entries(1..=3, 7)).await; + let meta = storage.meta_store(); + meta.save_hard_state(&HardState { + current_term: 7, + voted_for: None, + }) + .unwrap(); + meta.flush().unwrap(); + storage.log_store().purge(LogId { term: 7, index: 1 }).await.unwrap(); + storage.log_store().flush().unwrap(); + } + + let storage = RaftEngineStorage::open(dir.path()).unwrap(); + assert_eq!(storage.meta_store().load_hard_state().unwrap().unwrap().current_term, 7); + assert_eq!(storage.log_store().last_index(), 3); +} + +/// Dropping the storage must stop the purge thread right away. A worker that +/// sleeps out its interval would make shutdown hang for that whole interval. +#[tokio::test] +async fn test_purge_worker_stops_promptly_on_drop() { + let dir = TestDir::new("purge-stop"); + let storage = RaftEngineStorage::open_with( + dir.path(), + &RaftEngineOptions { + purge_interval_secs: 3600, + ..Default::default() + }, + ) + .unwrap(); + + let started = std::time::Instant::now(); + drop(storage); + + assert!( + started.elapsed() < std::time::Duration::from_secs(5), + "drop waited for the purge interval: {:?}", + started.elapsed() + ); +} + +/// A nested data directory that does not exist yet must be created, since a +/// fresh node is started with a path like `db/1/log` and no setup step. +#[tokio::test] +async fn test_open_creates_missing_nested_directory() { + let dir = TestDir::new("nested"); + let nested = dir.path().join("a").join("b").join("log"); + + let storage = RaftEngineStorage::open(&nested).unwrap(); + + assert!(nested.is_dir()); + drop(storage); +} diff --git a/examples/raft-engine-log/src/storage_engine_suite_test.rs b/examples/raft-engine-log/src/storage_engine_suite_test.rs new file mode 100644 index 00000000..8a003742 --- /dev/null +++ b/examples/raft-engine-log/src/storage_engine_suite_test.rs @@ -0,0 +1,74 @@ +//! Runs d-engine's shared `StorageEngine` contract suite against the raft-engine +//! adapter, so it is held to the same durability and ordering rules as the +//! built-in engines. + +use std::sync::Arc; + +use async_trait::async_trait; +use d_engine::Error; +use d_engine::common::Entry; +use d_engine::storage_engine_test::{StorageEngineBuilder, StorageEngineTestSuite}; +use d_engine::{LogStore, StorageEngine}; +use tempfile::TempDir; + +use crate::raft_engine_storage::RaftEngineStorage; + +/// Every `build()` reopens the same directory, which is how the suite simulates +/// a restart: the previous engine is dropped before the next one is built. +struct RaftEngineStorageBuilder { + dir: TempDir, +} + +impl RaftEngineStorageBuilder { + fn new() -> Self { + Self { + dir: TempDir::new().unwrap(), + } + } +} + +#[async_trait] +impl StorageEngineBuilder for RaftEngineStorageBuilder { + type Engine = RaftEngineStorage; + + async fn build(&self) -> Result, Error> { + Ok(Arc::new(RaftEngineStorage::open(self.dir.path().join("log"))?)) + } + + async fn cleanup(&self) -> Result<(), Error> { + Ok(()) + } +} + +/// The adapter must satisfy the whole shared contract, including durability +/// across a reopen without an explicit flush. +#[tokio::test] +async fn test_raft_engine_storage_engine_passes_contract_suite() -> Result<(), Error> { + StorageEngineTestSuite::run_all_tests(RaftEngineStorageBuilder::new()).await +} + +/// Persisting 10,000 entries in one call must stay within the throughput the +/// storage guide promises for a log store. +#[tokio::test] +async fn test_raft_engine_persist_10k_entries_within_one_second() -> Result<(), Error> { + let builder = RaftEngineStorageBuilder::new(); + let engine = builder.build().await?; + let log_store = engine.log_store(); + let entries: Vec = (1..=10_000) + .map(|i| Entry { + index: i, + term: 1, + payload: None, + }) + .collect(); + + let start = std::time::Instant::now(); + log_store.persist_entries(entries).await?; + + assert!( + start.elapsed().as_millis() < 1000, + "persisting 10k entries took {:?}", + start.elapsed() + ); + Ok(()) +} diff --git a/examples/raft-engine-log/tests/crash_recovery_test.rs b/examples/raft-engine-log/tests/crash_recovery_test.rs new file mode 100644 index 00000000..8fc35ccc --- /dev/null +++ b/examples/raft-engine-log/tests/crash_recovery_test.rs @@ -0,0 +1,340 @@ +//! Recovery after a crash that loses what `flush()` had not yet made durable. +//! +//! Dropping the store and reopening it does not model a power loss: the page cache survives. Each +//! test therefore drops the store and then edits the files the way a power loss leaves them (the +//! unsynced tail torn or missing, a never-synced file gone), reopens, and checks that every entry +//! acknowledged by `flush()` is still there, byte for byte, and that the store accepts new writes. + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use d_engine::common::{Entry, EntryPayload, LogId, entry_payload}; +use d_engine::{LogStore, StorageEngine}; +use prost::bytes::Bytes; +use raft_engine_log::{RaftEngineOptions, RaftEngineStorage}; +use tempfile::TempDir; + +const PAYLOAD_LEN: usize = 1024; +/// Entries per `persist_entries` call. raft-engine starts a new log file only between calls. +const BATCH: u64 = 10; + +fn options() -> RaftEngineOptions { + RaftEngineOptions { + target_file_size_mib: 1, + purge_threshold_mib: 2, + prefill_limit_mib: 1, + purge_interval_secs: 1, + } +} + +fn entry(index: u64, marker: u8) -> Entry { + Entry { + index, + term: 1, + payload: Some(EntryPayload { + payload: Some(entry_payload::Payload::Command(Bytes::from(vec![marker; PAYLOAD_LEN]))), + }), + } +} + +fn marker_of(entry: &Entry) -> u8 { + match entry.payload.as_ref().and_then(|p| p.payload.as_ref()) { + Some(entry_payload::Payload::Command(bytes)) => { + assert!(bytes.iter().all(|b| *b == bytes[0]), "payload of {} is mixed", entry.index); + bytes[0] + } + other => panic!("entry {} has no command payload: {other:?}", entry.index), + } +} + +/// Appends entries `from..=to` with `marker` in `BATCH`-sized calls and makes each call durable. +async fn write_acked( + log: &impl LogStore, + from: u64, + to: u64, + marker: u8, +) { + let mut next = from; + while next <= to { + let end = (next + BATCH - 1).min(to); + log.persist_entries((next..=end).map(|i| entry(i, marker)).collect()).await.unwrap(); + log.flush().unwrap(); + next = end + 1; + } +} + +/// Every index in `from..=to` is present, carries `expected(index)`, and nothing follows `to`. +async fn assert_log( + log: &impl LogStore, + from: u64, + to: u64, + expected: impl Fn(u64) -> u8, +) { + assert_eq!(log.last_index(), to, "last_index after recovery"); + for index in from..=to { + let e = log + .entry(index) + .await + .unwrap() + .unwrap_or_else(|| panic!("acknowledged entry {index} is missing after recovery")); + assert_eq!(marker_of(&e), expected(index), "entry {index} has the wrong payload"); + } + assert!(log.entry(to + 1).await.unwrap().is_none(), "entry {} must not exist", to + 1); +} + +/// `.raftlog` files, oldest first. +fn log_files(dir: &Path) -> Vec<(u64, PathBuf)> { + let mut files: Vec<(u64, PathBuf)> = std::fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok()) + .filter_map(|e| { + let name = e.file_name().to_string_lossy().into_owned(); + let seq = name.strip_suffix(".raftlog")?.parse().ok()?; + Some((seq, e.path())) + }) + .collect(); + files.sort(); + files +} + +fn active_file(dir: &Path) -> PathBuf { + log_files(dir).pop().expect("no log file").1 +} + +fn open(dir: &Path) -> RaftEngineStorage { + RaftEngineStorage::open_with(dir, &options()).unwrap() +} + +/// Entries 1..=50 are acknowledged, 51..=60 were written but not flushed, and the power loss +/// kept only part of the unflushed bytes: the file ends in the middle of an entry. Recovery must +/// keep exactly the acknowledged entries, and a rewritten tail must survive one more restart. +#[tokio::test] +async fn test_recovery_after_torn_unflushed_tail_keeps_acked_entries() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("log"); + let acked_len; + { + let storage = open(&path); + let log = storage.log_store(); + write_acked(&*log, 1, 50, 1).await; + acked_len = std::fs::metadata(active_file(&path)).unwrap().len(); + log.persist_entries((51..=60).map(|i| entry(i, 2)).collect()).await.unwrap(); + } + let file = active_file(&path); + assert!(std::fs::metadata(&file).unwrap().len() > acked_len + 2_000, "unflushed bytes missing"); + std::fs::OpenOptions::new().write(true).open(&file).unwrap().set_len(acked_len + 1_500).unwrap(); + + { + let storage = open(&path); + let log = storage.log_store(); + assert_log(&*log, 1, 50, |_| 1).await; + write_acked(&*log, 51, 55, 3).await; + } + let storage = open(&path); + assert_log(&*storage.log_store(), 1, 55, |i| if i <= 50 { 1 } else { 3 }).await; +} + +/// Same unflushed entries, but the disk wrote their later blocks and not the earlier ones: the +/// file keeps its full length and a stretch inside the unflushed region is garbage. +#[tokio::test] +async fn test_recovery_after_garbage_in_unflushed_tail_keeps_acked_entries() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("log"); + let acked_len; + { + let storage = open(&path); + let log = storage.log_store(); + write_acked(&*log, 1, 50, 1).await; + acked_len = std::fs::metadata(active_file(&path)).unwrap().len() as usize; + log.persist_entries((51..=60).map(|i| entry(i, 2)).collect()).await.unwrap(); + } + let file = active_file(&path); + let mut bytes = std::fs::read(&file).unwrap(); + assert!(bytes.len() > acked_len + 2_000, "unflushed bytes missing"); + bytes[acked_len + 100..acked_len + 600].fill(0xAB); + std::fs::write(&file, bytes).unwrap(); + + { + let storage = open(&path); + let log = storage.log_store(); + assert_log(&*log, 1, 50, |_| 1).await; + write_acked(&*log, 51, 55, 3).await; + } + let storage = open(&path); + assert_log(&*storage.log_store(), 1, 55, |i| if i <= 50 { 1 } else { 3 }).await; +} + +/// A log file taken from the reuse pool still holds the records of its previous life after the +/// new header and the first new entries. Recovery must stop at the end of the new entries and not +/// read those old records as part of the log. +#[tokio::test] +async fn test_recovery_in_reused_file_ignores_stale_records_after_the_tail() { + const STALE: u8 = 0xEE; + const FRESH: u8 = 0x11; + let dir = TempDir::new().unwrap(); + let path = dir.path().join("log"); + let mut last_acked; + { + let storage = open(&path); + let log = storage.log_store(); + // First life: ~6 MiB, then release the oldest files into the reuse pool. + write_acked(&*log, 1, 6_000, STALE).await; + let files_before = log_files(&path).len(); + log.purge(LogId { term: 1, index: 5_900 }).await.unwrap(); + log.flush().unwrap(); + let deadline = Instant::now() + Duration::from_secs(20); + while log_files(&path).len() >= files_before { + assert!(Instant::now() < deadline, "purge thread did not release any log file"); + tokio::time::sleep(Duration::from_millis(200)).await; + } + + // Second life: write until a new log file whose bytes after the header are stale. + last_acked = 6_000; + let started_at = log_files(&path).last().unwrap().0; + let mut stale_found = false; + while last_acked < 6_000 + 12_000 { + write_acked(&*log, last_acked + 1, last_acked + BATCH, FRESH).await; + last_acked += BATCH; + let (seq, file) = log_files(&path).pop().unwrap(); + if seq > started_at { + stale_found = contains_run(&file, STALE); + if stale_found { + break; + } + } + } + assert!(stale_found, "precondition not met: no log file with stale records after its tail"); + write_acked(&*log, last_acked + 1, last_acked + 5, FRESH).await; + last_acked += 5; + } + let storage = open(&path); + let log = storage.log_store(); + assert_log(&*log, 5_901, last_acked, |i| if i <= 6_000 { STALE } else { FRESH }).await; + write_acked(&*log, last_acked + 1, last_acked + 5, FRESH).await; +} + +/// True when the file holds a run of `marker` bytes. The fresh entries carry a different marker, +/// so such a run is a record left over from the file's previous life. +fn contains_run( + file: &Path, + marker: u8, +) -> bool { + std::fs::read(file).unwrap().windows(512).any(|w| w.iter().all(|b| *b == marker)) +} + +/// How the newest log file looks after a power loss right after it was created. +#[derive(Clone, Copy, Debug)] +enum NewestFile { + /// The directory entry never reached the disk. + Missing, + /// The entry exists but nothing was written into it. + Empty, + /// Only the first bytes of the header were written. + TornHeader, +} + +/// Writes acknowledged batches until the log starts a new file, then damages that file as a power +/// loss between creating it and syncing its first entries would. The batch that opened the file was +/// not acknowledged. Recovery must keep every earlier entry and accept new writes. +async fn recover_after_new_file_crash(newest: NewestFile) { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("log"); + let mut acked = 0; + { + let storage = open(&path); + let log = storage.log_store(); + let files = log_files(&path).len(); + loop { + let batch_end = acked + BATCH; + log.persist_entries((acked + 1..=batch_end).map(|i| entry(i, 1)).collect()).await.unwrap(); + if log_files(&path).len() > files { + break; // this batch opened the new file and stays unacknowledged + } + log.flush().unwrap(); + acked = batch_end; + assert!(acked < 5_000, "log never started a second file"); + } + } + let (_, file) = log_files(&path).pop().unwrap(); + match newest { + NewestFile::Missing => std::fs::remove_file(&file).unwrap(), + NewestFile::Empty => { + std::fs::OpenOptions::new().write(true).open(&file).unwrap().set_len(0).unwrap() + } + NewestFile::TornHeader => { + std::fs::OpenOptions::new().write(true).open(&file).unwrap().set_len(7).unwrap() + } + } + + { + let storage = open(&path); + let log = storage.log_store(); + assert_log(&*log, 1, acked, |_| 1).await; + // Enough new data to open further files on top of the recovered state. + write_acked(&*log, acked + 1, acked + 3_000, 2).await; + } + let storage = open(&path); + assert_log(&*storage.log_store(), 1, acked + 3_000, |i| if i <= acked { 1 } else { 2 }).await; +} + +#[tokio::test] +async fn test_recovery_when_newest_log_file_is_missing_keeps_acked_entries() { + recover_after_new_file_crash(NewestFile::Missing).await; +} + +#[tokio::test] +async fn test_recovery_when_newest_log_file_is_empty_keeps_acked_entries() { + recover_after_new_file_crash(NewestFile::Empty).await; +} + +#[tokio::test] +async fn test_recovery_when_newest_log_file_has_torn_header_keeps_acked_entries() { + recover_after_new_file_crash(NewestFile::TornHeader).await; +} + +/// The zero-filled files prepared at startup are never synced, so after a power loss they can be +/// absent or empty. They hold no entries; the log must still recover, and later rotations (which +/// would reuse them) must still work. +async fn recover_after_prefilled_files_lost(delete: bool) { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("log"); + { + let storage = open(&path); + let log = storage.log_store(); + write_acked(&*log, 1, 50, 1).await; + } + let spare: Vec = std::fs::read_dir(&path) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .filter(|p| p.to_string_lossy().ends_with(".raftlog.reserved")) + .collect(); + assert!(!spare.is_empty(), "precondition not met: no prefilled file on disk"); + for file in &spare { + if delete { + std::fs::remove_file(file).unwrap(); + } else { + std::fs::OpenOptions::new().write(true).open(file).unwrap().set_len(0).unwrap(); + } + } + + { + let storage = open(&path); + let log = storage.log_store(); + assert_log(&*log, 1, 50, |_| 1).await; + // About 4 MiB: several rotations, which take prefilled files first when there are any. + write_acked(&*log, 51, 4_050, 2).await; + } + let storage = open(&path); + assert_log(&*storage.log_store(), 1, 4_050, |i| if i <= 50 { 1 } else { 2 }).await; +} + +#[tokio::test] +async fn test_recovery_when_prefilled_files_are_deleted_keeps_acked_entries() { + recover_after_prefilled_files_lost(true).await; +} + +#[tokio::test] +async fn test_recovery_when_prefilled_files_are_empty_keeps_acked_entries() { + recover_after_prefilled_files_lost(false).await; +} diff --git a/examples/raft-engine-log/tests/flush_syncs_test.rs b/examples/raft-engine-log/tests/flush_syncs_test.rs new file mode 100644 index 00000000..c4189acb --- /dev/null +++ b/examples/raft-engine-log/tests/flush_syncs_test.rs @@ -0,0 +1,67 @@ +//! `flush()` must reach the disk. This lives in its own test binary because the +//! counter it reads is process-wide: another test syncing in parallel could +//! otherwise satisfy the assertion and hide a store that never syncs. + +use d_engine::common::Entry; +use d_engine::{LogStore, MetaStore, StorageEngine}; +use raft_engine_log::RaftEngineStorage; +use tempfile::TempDir; + +/// Samples recorded by raft-engine's own histogram of physical log syncs. +fn physical_syncs() -> u64 { + prometheus::gather() + .iter() + .filter(|family| family.get_name() == "raft_engine_sync_log_duration_seconds") + .flat_map(|family| family.get_metric().iter()) + .map(|metric| metric.get_histogram().get_sample_count()) + .sum() +} + +fn entry(index: u64) -> Entry { + Entry { + index, + term: 1, + payload: None, + } +} + +/// Every `flush()` after a write must perform a physical sync, for the log and +/// for the hard state (acknowledged after `MetaStore::flush`). A store whose +/// `flush()` returns without syncing still passes restart tests (the page cache +/// survives a reopen) while acknowledging commits that a power loss would drop. +/// +/// One test only: the counter is process-wide, so two tests running in parallel +/// in this binary could satisfy each other's assertions. +#[tokio::test] +async fn test_flush_after_write_performs_physical_sync() { + use d_engine::HardState; + + let dir = TempDir::new().unwrap(); + let storage = RaftEngineStorage::open(dir.path().join("log")).unwrap(); + let log = storage.log_store(); + let meta = storage.meta_store(); + + let mut previous = physical_syncs(); + for index in 1..=5 { + log.persist_entries(vec![entry(index)]).await.unwrap(); + log.flush().unwrap(); + + let now = physical_syncs(); + assert!( + now > previous, + "log flush #{index} did not sync (sync samples stayed at {previous})" + ); + previous = now; + } + + meta.save_hard_state(&HardState { + current_term: 3, + voted_for: None, + }) + .unwrap(); + meta.flush().unwrap(); + assert!( + physical_syncs() > previous, + "meta flush did not sync (sync samples stayed at {previous})" + ); +} diff --git a/examples/raft-engine-log/tests/purge_reclaims_files_test.rs b/examples/raft-engine-log/tests/purge_reclaims_files_test.rs new file mode 100644 index 00000000..cff5ff08 --- /dev/null +++ b/examples/raft-engine-log/tests/purge_reclaims_files_test.rs @@ -0,0 +1,87 @@ +//! The background purge thread must actually release log files once their +//! entries have been purged. raft-engine never does this by itself, and the unit +//! tests only prove that the thread stops promptly, not that it does its job. + +use std::path::Path; +use std::time::{Duration, Instant}; + +use d_engine::common::{Entry, EntryPayload, LogId, entry_payload}; +use d_engine::{LogStore, StorageEngine}; +use prost::bytes::Bytes; +use raft_engine_log::{RaftEngineOptions, RaftEngineStorage}; +use tempfile::TempDir; + +/// Sequence numbers of the log files on disk (`.raftlog`), oldest first. A purged file that is +/// kept for reuse keeps its name, so only the number of files says how much space was released. +fn log_file_seqs(dir: &Path) -> Vec { + let mut seqs: Vec = std::fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok()) + .filter_map(|e| { + let name = e.file_name().to_string_lossy().into_owned(); + name.strip_suffix(".raftlog").and_then(|stem| stem.parse().ok()) + }) + .collect(); + seqs.sort_unstable(); + seqs +} + +fn entry(index: u64) -> Entry { + Entry { + index, + term: 1, + payload: Some(EntryPayload { + payload: Some(entry_payload::Payload::Command(Bytes::from(vec![7u8; 1024]))), + }), + } +} + +/// About 6 MiB of entries span several 1 MiB files. After `purge` covers all but +/// the last 100 entries, the oldest files hold nothing live, and the purge thread +/// (1 s interval, retained log above the 2 MiB threshold) must release them: files +/// beyond the recycle capacity are deleted, so fewer files remain on disk. +#[tokio::test] +async fn test_purge_thread_releases_files_after_purge() { + let dir = TempDir::new().unwrap(); + let log_dir = dir.path().join("log"); + let storage = RaftEngineStorage::open_with( + &log_dir, + &RaftEngineOptions { + target_file_size_mib: 1, + purge_threshold_mib: 2, + prefill_limit_mib: 4, + purge_interval_secs: 1, + }, + ) + .unwrap(); + let log = storage.log_store(); + + for chunk in 0..60u64 { + let batch: Vec = (1..=100).map(|i| entry(chunk * 100 + i)).collect(); + log.persist_entries(batch).await.unwrap(); + log.flush().unwrap(); + } + let before = log_file_seqs(&log_dir); + assert!(before.len() >= 4, "expected several log files, got {before:?}"); + + log.purge(LogId { + term: 1, + index: 5_900, + }) + .await + .unwrap(); + log.flush().unwrap(); + + let deadline = Instant::now() + Duration::from_secs(20); + loop { + let after = log_file_seqs(&log_dir); + if after.len() < before.len() { + return; + } + assert!( + Instant::now() < deadline, + "purge thread did not release any file: before {before:?}, after {after:?}" + ); + tokio::time::sleep(Duration::from_millis(500)).await; + } +} diff --git a/examples/single-node-expansion/Cargo.lock b/examples/single-node-expansion/Cargo.lock index 5a759431..ad3e580c 100644 --- a/examples/single-node-expansion/Cargo.lock +++ b/examples/single-node-expansion/Cargo.lock @@ -634,21 +634,6 @@ dependencies = [ "parking_lot_core", ] -[[package]] -name = "demo" -version = "0.2.0" -dependencies = [ - "console-subscriber", - "d-engine", - "metrics", - "metrics-exporter-prometheus", - "tokio", - "tokio-metrics", - "tracing", - "tracing-appender", - "tracing-subscriber", -] - [[package]] name = "deranged" version = "0.5.8" @@ -2041,6 +2026,21 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "single-node-expansion" +version = "0.2.0" +dependencies = [ + "console-subscriber", + "d-engine", + "metrics", + "metrics-exporter-prometheus", + "tokio", + "tokio-metrics", + "tracing", + "tracing-appender", + "tracing-subscriber", +] + [[package]] name = "sketches-ddsketch" version = "0.3.1" diff --git a/examples/single-node-expansion/Cargo.toml b/examples/single-node-expansion/Cargo.toml index 2d3be6e3..7b96df77 100644 --- a/examples/single-node-expansion/Cargo.toml +++ b/examples/single-node-expansion/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "demo" +name = "single-node-expansion" version = "0.2.0" edition = "2024" diff --git a/examples/single-node-expansion/Makefile b/examples/single-node-expansion/Makefile index a00218c4..5ddadec2 100644 --- a/examples/single-node-expansion/Makefile +++ b/examples/single-node-expansion/Makefile @@ -49,9 +49,9 @@ start-node1: DB_PATH="./db/1" \ LOG_DIR="./logs/1" \ METRICS_PORT=8081 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_LOG=single_node_expansion=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ RUST_BACKTRACE=1 \ - ./target/release/demo + ./target/release/single-node-expansion # =============================== # Dynamic Expansion - Add Node 2 @@ -70,9 +70,9 @@ join-node2: DB_PATH="./db/2" \ LOG_DIR="./logs/2" \ METRICS_PORT=8082 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_LOG=single_node_expansion=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ RUST_BACKTRACE=1 \ - ./target/release/demo + ./target/release/single-node-expansion # =============================== # Dynamic Expansion - Add Node 3 @@ -91,9 +91,9 @@ join-node3: DB_PATH="./db/3" \ LOG_DIR="./logs/3" \ METRICS_PORT=8083 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_LOG=single_node_expansion=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ RUST_BACKTRACE=1 \ - ./target/release/demo + ./target/release/single-node-expansion # =============================== # Log Monitoring diff --git a/examples/single-node-expansion/README.md b/examples/single-node-expansion/README.md index 275bb3ea..48441be3 100644 --- a/examples/single-node-expansion/README.md +++ b/examples/single-node-expansion/README.md @@ -163,7 +163,7 @@ make clean-logs # Remove logs and DB only **Node 2/3 can't join:** -- Verify Node 1 is running: `ps aux | grep demo` +- Verify Node 1 is running: `ps aux | grep single-node-expansion` - Check addresses in config match actual IPs **Learner never promotes:** diff --git a/examples/three-nodes-standalone/.gitignore b/examples/three-nodes-standalone/.gitignore index 7c7dbb52..c0a2ff34 100644 --- a/examples/three-nodes-standalone/.gitignore +++ b/examples/three-nodes-standalone/.gitignore @@ -37,3 +37,6 @@ docker/jepsen/store/*/*/* cobertura.xml .DS_Store + +target-raft-engine/ +db-raft-engine/ diff --git a/examples/three-nodes-standalone/Cargo.toml b/examples/three-nodes-standalone/Cargo.toml index 07b6ec09..3fda2db6 100644 --- a/examples/three-nodes-standalone/Cargo.toml +++ b/examples/three-nodes-standalone/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "demo" +name = "three-nodes-standalone" version = "0.2.0" edition = "2024" @@ -19,3 +19,9 @@ metrics = "0.24" # For console support console-subscriber = "0.4" + +# Optional: run the Raft log in raft-engine instead of the RocksDB WAL. +raft-engine-log = { path = "../raft-engine-log", optional = true } + +[features] +raft-engine = ["dep:raft-engine-log"] diff --git a/examples/three-nodes-standalone/Makefile b/examples/three-nodes-standalone/Makefile index 7f23d971..4c0670ff 100644 --- a/examples/three-nodes-standalone/Makefile +++ b/examples/three-nodes-standalone/Makefile @@ -12,6 +12,15 @@ # =============================== LOG_LEVEL ?= warn +# Storage engine. Default: Raft log in raft-engine (experimental). +# ENGINE=rocksdb builds and runs the stock RocksDB-WAL binary. Each engine has its own +# target dir and its own data dir, so switching never mixes their files. +ENGINE ?= raft-engine +$(if $(filter raft-engine rocksdb,$(ENGINE)),,$(error ENGINE must be raft-engine or rocksdb, got '$(ENGINE)')) +TARGET_DIR := $(if $(filter raft-engine,$(ENGINE)),target-raft-engine,target) +CARGO_FEATURES := $(if $(filter raft-engine,$(ENGINE)),--features raft-engine,) +DB_ROOT := $(if $(filter raft-engine,$(ENGINE)),./db-raft-engine,./db) + # On macOS with Homebrew: auto-detect compression lib paths to skip bundled C++ # compilation of RocksDB dependencies, which fails under macOS 26 + Xcode 26 # (Clang 16 lacks __builtin_ctzg/__builtin_clzg from LLVM 18+ SDK headers). @@ -38,17 +47,18 @@ endif # Build Targets # =============================== build: - @echo "Building release binary..." - $(BREW_ROCKSDB_ENV) cargo build --release --jobs 4 + @echo "Building release binary (engine: $(ENGINE))..." + $(BREW_ROCKSDB_ENV) cargo build --release --jobs 4 $(CARGO_FEATURES) --target-dir $(TARGET_DIR) # =============================== # Cluster Management (Normal Mode) # =============================== # Overridable so start-ram-cluster can point these at /Volumes/RAMDiskN # without duplicating the node targets. -DB_PATH_1 ?= ./db/1 -DB_PATH_2 ?= ./db/2 -DB_PATH_3 ?= ./db/3 +BIN ?= ./$(TARGET_DIR)/release/three-nodes-standalone +DB_PATH_1 ?= $(DB_ROOT)/1 +DB_PATH_2 ?= $(DB_ROOT)/2 +DB_PATH_3 ?= $(DB_ROOT)/3 start-node1: @echo "🚀 Starting Node 1..." @@ -56,9 +66,9 @@ start-node1: DB_PATH="$(DB_PATH_1)" \ LOG_DIR="./logs/1" \ METRICS_PORT=8081 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_LOG=three_nodes_standalone=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ RUST_BACKTRACE=1 \ - ./target/release/demo 2>&1 | sed 's/^/[Node1] /' + $(BIN) 2>&1 | sed 's/^/[Node1] /' start-node2: @echo "🚀 Starting Node 2..." @@ -66,8 +76,8 @@ start-node2: DB_PATH="$(DB_PATH_2)" \ LOG_DIR="./logs/2" \ METRICS_PORT=8082 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ - ./target/release/demo 2>&1 | sed 's/^/[Node2] /' + RUST_LOG=three_nodes_standalone=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + $(BIN) 2>&1 | sed 's/^/[Node2] /' start-node3: @echo "🚀 Starting Node 3..." @@ -75,8 +85,8 @@ start-node3: DB_PATH="$(DB_PATH_3)" \ LOG_DIR="./logs/3" \ METRICS_PORT=8083 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ - ./target/release/demo 2>&1 | sed 's/^/[Node3] /' + RUST_LOG=three_nodes_standalone=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + $(BIN) 2>&1 | sed 's/^/[Node3] /' start-cluster: @@ -108,10 +118,10 @@ ramdisk-create: [ -d "/Volumes/$$v" ] || diskutil erasevolume HFS+ $$v `hdiutil attach -nomount ram://$(RAMDISK_SECTORS)` >/dev/null; \ done -# Wipes node data off the RAM disks; keeps the volumes mounted so the next -# start-ram-cluster doesn't pay to recreate them. +# Wipes this example's node directories off the RAM disks; keeps the volumes mounted so the +# next start-ram-cluster doesn't pay to recreate them. Nothing else on the volumes is touched. clean-ram-log-db: - @for v in $(RAMDISK_VOLUMES); do rm -rf /Volumes/$$v/*; done + @rm -rf /Volumes/RAMDisk1/n1 /Volumes/RAMDisk2/n2 /Volumes/RAMDisk3/n3 # Unmounts the RAM disks entirely, releasing the memory back to the OS. ramdisk-release: @@ -126,77 +136,77 @@ ramdisk-release: perf-node1: @echo "Profiling Node 1 with samply..." CONFIG_PATH=config/n1 \ - DB_PATH="./db/1" \ + DB_PATH="$(DB_PATH_1)" \ LOG_DIR="./logs/1" \ METRICS_PORT=8081 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_LOG=three_nodes_standalone=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ RUST_BACKTRACE=1 \ - samply record -o ./samply_reports/1.profile.json -P 3001 ./target/release/demo + samply record -o ./samply_reports/1.profile.json -P 3001 $(BIN) perf-node2: @echo "Profiling Node 2 with samply..." CONFIG_PATH=config/n2 \ - DB_PATH="./db/2" \ + DB_PATH="$(DB_PATH_2)" \ LOG_DIR="./logs/2" \ METRICS_PORT=8082 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ - samply record -o ./samply_reports/2.profile.json -P 3002 ./target/release/demo + RUST_LOG=three_nodes_standalone=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + samply record -o ./samply_reports/2.profile.json -P 3002 $(BIN) perf-node3: @echo "Profiling Node 3 with samply..." CONFIG_PATH=config/n3 \ - DB_PATH="./db/3" \ + DB_PATH="$(DB_PATH_3)" \ LOG_DIR="./logs/3" \ METRICS_PORT=8083 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ - samply record -o ./samply_reports/3.profile.json -P 3003 ./target/release/demo + RUST_LOG=three_nodes_standalone=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + samply record -o ./samply_reports/3.profile.json -P 3003 $(BIN) -perf-cluster: clean build +perf-cluster: clean-log-db build @echo "Starting 3-node cluster under samply profiling..." $(MAKE) -j3 perf-node1 perf-node2 perf-node3 build-tokio-console: - $(BREW_ROCKSDB_ENV) RUSTFLAGS="--cfg tokio_unstable" cargo build --release --jobs 4 + $(BREW_ROCKSDB_ENV) RUSTFLAGS="--cfg tokio_unstable" cargo build --release --jobs 4 $(CARGO_FEATURES) --target-dir $(TARGET_DIR) tokio-console-node1: @echo "Starting Node 1 with tokio console monitoring..." TOKIO_CONSOLE=1 \ TOKIO_CONSOLE_PORT=6670 \ CONFIG_PATH=config/n1 \ - DB_PATH="./db/1" \ + DB_PATH="$(DB_PATH_1)" \ LOG_DIR="./logs/1" \ METRICS_PORT=8081 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_LOG=three_nodes_standalone=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ RUST_BACKTRACE=1 \ - ./target/release/demo + $(BIN) tokio-console-node2: @echo "Starting Node 2 with tokio console monitoring..." TOKIO_CONSOLE=1 \ TOKIO_CONSOLE_PORT=6671 \ CONFIG_PATH=config/n2 \ - DB_PATH="./db/2" \ + DB_PATH="$(DB_PATH_2)" \ LOG_DIR="./logs/2" \ METRICS_PORT=8082 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_LOG=three_nodes_standalone=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ RUST_BACKTRACE=1 \ - ./target/release/demo + $(BIN) tokio-console-node3: @echo "Starting Node 3 with tokio console monitoring..." TOKIO_CONSOLE=1 \ TOKIO_CONSOLE_PORT=6672 \ CONFIG_PATH=config/n3 \ - DB_PATH="./db/3" \ + DB_PATH="$(DB_PATH_3)" \ LOG_DIR="./logs/3" \ METRICS_PORT=8083 \ - RUST_LOG=demo=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ + RUST_LOG=three_nodes_standalone=$(LOG_LEVEL),d_engine=$(LOG_LEVEL),timing=$(LOG_LEVEL) \ RUST_BACKTRACE=1 \ - ./target/release/demo + $(BIN) -tokio-console-cluster: clean build-tokio-console +tokio-console-cluster: clean-log-db build-tokio-console @echo "Starting 3-node cluster under tokio console monitoring ..." $(MAKE) -j3 tokio-console-node1 tokio-console-node2 tokio-console-node3 @@ -204,11 +214,15 @@ tokio-console-cluster: clean build-tokio-console # =============================== # Utilities # =============================== +# Removes both engines' build output and local data. RAM disk data is wiped separately with +# `make clean-ram-log-db`. clean: @echo "Cleaning build artifacts..." - cargo clean + cargo clean --target-dir target + cargo clean --target-dir target-raft-engine rm -rf logs/* rm -rf db/* + rm -rf db-raft-engine/* rm -rf snapshots/* rm -rf samply_reports/* @@ -216,12 +230,14 @@ clean-log-db: @echo "Cleaning db and logs..." rm -rf logs/* rm -rf db/* + rm -rf db-raft-engine/* help: @echo "DEngine Cluster Management Commands:" @echo " build - Build release binary" @echo " start-node1..3 - Start individual cluster nodes" @echo " start-cluster - Start all 3 nodes in parallel" + @echo " ENGINE=rocksdb - Any target: use the stock RocksDB-WAL build (default: raft-engine, experimental)" @echo " perf-node1..3 - Run individual nodes under samply profiler" @echo " perf-cluster - Run full 3-node cluster under samply profiling" @echo " tokio-console-cluster - Run full 3-node cluster under tokio console monitoring" @@ -229,6 +245,6 @@ help: @echo " ramdisk-create - (macOS) Create RAMDisk1/2/3 if not already mounted" @echo " clean-ram-log-db - Wipe node data off the RAM disks (keeps them mounted)" @echo " ramdisk-release - Unmount RAMDisk1/2/3, freeing the memory" - @echo " clean - Remove build artifacts, logs, and profiles" + @echo " clean - Remove both engines' builds, data, logs and profiles (RAM disks: use clean-ram-log-db)" @echo " clean-log-db - Remove only logs and database files" @echo " help - Show this help message" diff --git a/examples/three-nodes-standalone/README.md b/examples/three-nodes-standalone/README.md index bca07a23..1a12fc52 100644 --- a/examples/three-nodes-standalone/README.md +++ b/examples/three-nodes-standalone/README.md @@ -28,9 +28,9 @@ For manual control, run these in separate terminal sessions: CONFIG_PATH=config/n1 \ DB_PATH="./db/1" \ LOG_DIR="./logs/1" \ -RUST_LOG=demo=debug,d_engine=debug \ +RUST_LOG=three_nodes_standalone=debug,d_engine=debug \ RUST_BACKTRACE=1 \ -target/release/demo +target/release/three-nodes-standalone ``` **Node 2:** @@ -39,8 +39,8 @@ target/release/demo CONFIG_PATH=config/n2 \ DB_PATH="./db/2" \ LOG_DIR="./logs/2" \ -RUST_LOG=demo=debug,d_engine=debug \ -target/release/demo +RUST_LOG=three_nodes_standalone=debug,d_engine=debug \ +target/release/three-nodes-standalone ``` **Node 3:** @@ -49,8 +49,8 @@ target/release/demo CONFIG_PATH=config/n3 \ DB_PATH="./db/3" \ LOG_DIR="./logs/3" \ -RUST_LOG=demo=debug,d_engine=debug \ -target/release/demo +RUST_LOG=three_nodes_standalone=debug,d_engine=debug \ +target/release/three-nodes-standalone ``` ## Prerequisites @@ -91,7 +91,9 @@ config/ ### Log Monitoring ```bash -tail -f logs/1/demo.log# Watch node 1 logstail -f logs/2/demo.log# Watch node 2 logstail -f logs/3/demo.log# Watch node 3 logs +tail -f logs/1/d.log # Watch node 1 logs +tail -f logs/2/d.log # Watch node 2 logs +tail -f logs/3/d.log # Watch node 3 logs ``` ## Important Notes diff --git a/examples/three-nodes-standalone/config/n1.toml b/examples/three-nodes-standalone/config/n1.toml index 431f4e09..0866c1bb 100644 --- a/examples/three-nodes-standalone/config/n1.toml +++ b/examples/three-nodes-standalone/config/n1.toml @@ -83,3 +83,17 @@ max_pending_accept_reset_streams = 2000 # Higher pending stream limit for Rapid [storage] unified_db = false + +# == Raft log store: read only by the `raft-engine` build (`make start-cluster`, the default build); +# the RocksDB build ignores this section. Values suit a local run (about 12 MiB of recycle pool per node); +# the library defaults are 64 / 512 / 256 / 10 and the AWS benchmark scripts write their own values. +# Log files are only released after d-engine purges the log, which happens after a snapshot. Snapshots +# are disabled above, as in the benchmark configs, so nothing is released in this setup; to watch files +# being released locally, set [raft.snapshot] enable = true (5000 entries is about 1.6 MiB of log). +# purge_threshold_mib also caps the recycle pool at 1.5 x purge_threshold_mib / target_file_size_mib +# files, so prefill_limit_mib above purge_threshold_mib has no effect. +[raft_engine] +target_file_size_mib = 1 # size at which the active log file is rotated +purge_threshold_mib = 8 # reclaim log files once the retained log exceeds this +prefill_limit_mib = 8 # zero-filled recycle pool prepared at startup: 1.5 x this many files +purge_interval_secs = 2 # how often expired log files are reclaimed diff --git a/examples/three-nodes-standalone/config/n2.toml b/examples/three-nodes-standalone/config/n2.toml index 0d100281..cbb9e8b3 100644 --- a/examples/three-nodes-standalone/config/n2.toml +++ b/examples/three-nodes-standalone/config/n2.toml @@ -83,3 +83,17 @@ max_pending_accept_reset_streams = 2000 # Higher pending stream limit for Rapid [storage] unified_db = false + +# == Raft log store: read only by the `raft-engine` build (`make start-cluster`, the default build); +# the RocksDB build ignores this section. Values suit a local run (about 12 MiB of recycle pool per node); +# the library defaults are 64 / 512 / 256 / 10 and the AWS benchmark scripts write their own values. +# Log files are only released after d-engine purges the log, which happens after a snapshot. Snapshots +# are disabled above, as in the benchmark configs, so nothing is released in this setup; to watch files +# being released locally, set [raft.snapshot] enable = true (5000 entries is about 1.6 MiB of log). +# purge_threshold_mib also caps the recycle pool at 1.5 x purge_threshold_mib / target_file_size_mib +# files, so prefill_limit_mib above purge_threshold_mib has no effect. +[raft_engine] +target_file_size_mib = 1 # size at which the active log file is rotated +purge_threshold_mib = 8 # reclaim log files once the retained log exceeds this +prefill_limit_mib = 8 # zero-filled recycle pool prepared at startup: 1.5 x this many files +purge_interval_secs = 2 # how often expired log files are reclaimed diff --git a/examples/three-nodes-standalone/config/n3.toml b/examples/three-nodes-standalone/config/n3.toml index 5e3d80d2..8688add7 100644 --- a/examples/three-nodes-standalone/config/n3.toml +++ b/examples/three-nodes-standalone/config/n3.toml @@ -83,3 +83,17 @@ max_pending_accept_reset_streams = 2000 # Higher pending stream limit for Rapid [storage] unified_db = false + +# == Raft log store: read only by the `raft-engine` build (`make start-cluster`, the default build); +# the RocksDB build ignores this section. Values suit a local run (about 12 MiB of recycle pool per node); +# the library defaults are 64 / 512 / 256 / 10 and the AWS benchmark scripts write their own values. +# Log files are only released after d-engine purges the log, which happens after a snapshot. Snapshots +# are disabled above, as in the benchmark configs, so nothing is released in this setup; to watch files +# being released locally, set [raft.snapshot] enable = true (5000 entries is about 1.6 MiB of log). +# purge_threshold_mib also caps the recycle pool at 1.5 x purge_threshold_mib / target_file_size_mib +# files, so prefill_limit_mib above purge_threshold_mib has no effect. +[raft_engine] +target_file_size_mib = 1 # size at which the active log file is rotated +purge_threshold_mib = 8 # reclaim log files once the retained log exceeds this +prefill_limit_mib = 8 # zero-filled recycle pool prepared at startup: 1.5 x this many files +purge_interval_secs = 2 # how often expired log files are reclaimed diff --git a/examples/three-nodes-standalone/docker/Dockerfile b/examples/three-nodes-standalone/docker/Dockerfile index f38d2e0a..20589759 100644 --- a/examples/three-nodes-standalone/docker/Dockerfile +++ b/examples/three-nodes-standalone/docker/Dockerfile @@ -36,7 +36,9 @@ WORKDIR /build/examples/three-nodes-standalone RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/build/examples/three-nodes-standalone/target \ cargo build --release && \ - cp target/release/demo /usr/local/bin/demo + # Installed as /usr/local/bin/demo on purpose: the Jepsen harness (d-engine-jepsen) starts, kills + # and looks for the node by that name, and its compose file uses the image name demo:. + cp target/release/three-nodes-standalone /usr/local/bin/demo # ------------------------ # Runtime Stage (ARM64 Optimized) @@ -92,4 +94,4 @@ RUN echo "$GIT_SHA" > /etc/d-engine-commit WORKDIR /app -CMD ["sh", "-c", "/usr/sbin/sshd -D & CONFIG_PATH=$CONFIG_PATH LOG_DIR=$LOG_DIR METRICS_PORT=$METRICS_PORT DB_PATH=/app/db/$ID RUST_LOG=demo=$LOG_LEVEL,d_engine=$LOG_LEVEL,hyper=warn,sled=warn demo & promtail --config.file=/etc/promtail/config.yml > /app/logs/promtail.log 2>&1"] +CMD ["sh", "-c", "/usr/sbin/sshd -D & CONFIG_PATH=$CONFIG_PATH LOG_DIR=$LOG_DIR METRICS_PORT=$METRICS_PORT DB_PATH=/app/db/$ID RUST_LOG=three_nodes_standalone=$LOG_LEVEL,d_engine=$LOG_LEVEL,hyper=warn,sled=warn demo & promtail --config.file=/etc/promtail/config.yml > /app/logs/promtail.log 2>&1"] diff --git a/examples/three-nodes-standalone/src/main.rs b/examples/three-nodes-standalone/src/main.rs index 7e86e4d2..322dfcd5 100644 --- a/examples/three-nodes-standalone/src/main.rs +++ b/examples/three-nodes-standalone/src/main.rs @@ -103,9 +103,8 @@ async fn start_dengine_server( println!("║ Config: {config_path:<28} ║"); println!("╚════════════════════════════════════════╝"); - // StandaloneEngine with explicit data_dir and config path // Blocks until shutdown signal received - if let Err(e) = StandaloneEngine::run_with(&data_dir, &config_path, graceful_rx).await { + if let Err(e) = run_engine(&data_dir, &config_path, graceful_rx).await { error!("Server stopped with error: {:?}", e); } else { info!("Server stopped gracefully"); @@ -114,6 +113,40 @@ async fn start_dengine_server( println!("Exiting program."); } +/// Default build: d-engine's stock RocksDB engine, explicit data_dir and config path. +#[cfg(not(feature = "raft-engine"))] +async fn run_engine( + data_dir: &str, + config_path: &str, + graceful_rx: watch::Receiver<()>, +) -> Result<(), d_engine::Error> { + StandaloneEngine::run_with(data_dir, config_path, graceful_rx).await +} + +/// `raft-engine` feature: only the Raft log and the hard state move to raft-engine. The state +/// machine, the config file and the data layout are the same as the default build, so the two +/// builds differ by the log store alone. +#[cfg(feature = "raft-engine")] +async fn run_engine( + data_dir: &str, + config_path: &str, + graceful_rx: watch::Receiver<()>, +) -> Result<(), d_engine::Error> { + use std::sync::Arc; + + std::fs::create_dir_all(data_dir) + .map_err(|e| d_engine::Error::Fatal(format!("create {data_dir}: {e}")))?; + let options = raft_engine_log::RaftEngineOptions::from_toml_file(config_path)?; + let storage = Arc::new(raft_engine_log::RaftEngineStorage::open_with( + format!("{data_dir}/storage"), + &options, + )?); + let sm = Arc::new(d_engine::RocksDBStateMachine::new(format!( + "{data_dir}/state_machine" + ))?); + StandaloneEngine::run_custom(data_dir, storage, sm, graceful_rx, Some(config_path)).await +} + async fn graceful_shutdown(graceful_tx: watch::Sender<()>) { info!("Monitoring shutdown signal ..."); let mut sigint = signal(SignalKind::interrupt()).unwrap();